1
2
3
4
5
6
7
8
9
10
11#include <linux/module.h>
12
13#include <linux/stringify.h>
14#include <linux/kernel.h>
15#include <linux/timer.h>
16#include <linux/errno.h>
17#include <linux/ioport.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
20#include <linux/interrupt.h>
21#include <linux/pci.h>
22#include <linux/netdevice.h>
23#include <linux/etherdevice.h>
24#include <linux/skbuff.h>
25#include <linux/dma-mapping.h>
26#include <linux/bitops.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/delay.h>
30#include <asm/byteorder.h>
31#include <asm/page.h>
32#include <linux/time.h>
33#include <linux/mii.h>
34#include <linux/mdio.h>
35#include <linux/if.h>
36#include <linux/if_vlan.h>
37#include <linux/if_bridge.h>
38#include <linux/rtc.h>
39#include <linux/bpf.h>
40#include <net/ip.h>
41#include <net/tcp.h>
42#include <net/udp.h>
43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
45#include <net/udp_tunnel.h>
46#include <linux/workqueue.h>
47#include <linux/prefetch.h>
48#include <linux/cache.h>
49#include <linux/log2.h>
50#include <linux/aer.h>
51#include <linux/bitmap.h>
52#include <linux/ptp_clock_kernel.h>
53#include <linux/timecounter.h>
54#include <linux/cpu_rmap.h>
55#include <linux/cpumask.h>
56#include <net/pkt_cls.h>
57#include <linux/hwmon.h>
58#include <linux/hwmon-sysfs.h>
59#include <net/page_pool.h>
60
61#include "bnxt_hsi.h"
62#include "bnxt.h"
63#include "bnxt_ulp.h"
64#include "bnxt_sriov.h"
65#include "bnxt_ethtool.h"
66#include "bnxt_dcb.h"
67#include "bnxt_xdp.h"
68#include "bnxt_ptp.h"
69#include "bnxt_vfr.h"
70#include "bnxt_tc.h"
71#include "bnxt_devlink.h"
72#include "bnxt_debugfs.h"
73
74#define BNXT_TX_TIMEOUT (5 * HZ)
75#define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \
76 NETIF_MSG_TX_ERR)
77
78MODULE_LICENSE("GPL");
79MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
80
81#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
82#define BNXT_RX_DMA_OFFSET NET_SKB_PAD
83#define BNXT_RX_COPY_THRESH 256
84
85#define BNXT_TX_PUSH_THRESH 164
86
87enum board_idx {
88 BCM57301,
89 BCM57302,
90 BCM57304,
91 BCM57417_NPAR,
92 BCM58700,
93 BCM57311,
94 BCM57312,
95 BCM57402,
96 BCM57404,
97 BCM57406,
98 BCM57402_NPAR,
99 BCM57407,
100 BCM57412,
101 BCM57414,
102 BCM57416,
103 BCM57417,
104 BCM57412_NPAR,
105 BCM57314,
106 BCM57417_SFP,
107 BCM57416_SFP,
108 BCM57404_NPAR,
109 BCM57406_NPAR,
110 BCM57407_SFP,
111 BCM57407_NPAR,
112 BCM57414_NPAR,
113 BCM57416_NPAR,
114 BCM57452,
115 BCM57454,
116 BCM5745x_NPAR,
117 BCM57508,
118 BCM57504,
119 BCM57502,
120 BCM57508_NPAR,
121 BCM57504_NPAR,
122 BCM57502_NPAR,
123 BCM58802,
124 BCM58804,
125 BCM58808,
126 NETXTREME_E_VF,
127 NETXTREME_C_VF,
128 NETXTREME_S_VF,
129 NETXTREME_C_VF_HV,
130 NETXTREME_E_VF_HV,
131 NETXTREME_E_P5_VF,
132 NETXTREME_E_P5_VF_HV,
133};
134
135
136static const struct {
137 char *name;
138} board_info[] = {
139 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
140 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
141 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
142 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
143 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
144 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
145 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
146 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
147 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
148 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
149 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
150 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
151 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
152 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
153 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
154 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
155 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
156 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
157 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
158 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
159 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
160 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
161 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
162 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
163 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
164 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
165 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
166 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
167 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
168 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
169 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
170 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
171 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
172 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
173 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
174 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
175 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
176 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
177 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
178 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
179 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
180 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
181 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
182 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
183 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
184};
185
186static const struct pci_device_id bnxt_pci_tbl[] = {
187 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
188 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
189 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
190 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
191 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
192 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
193 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
194 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
195 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
196 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
197 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
198 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
199 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
200 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
201 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
202 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
203 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
204 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
205 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
206 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
207 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
208 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
209 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
210 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
211 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
212 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
213 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
214 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
215 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
216 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
217 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
218 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
219 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
220 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
221 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
222 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
223 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
224 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
225 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
226 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
227 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
228 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
229 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
230 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
231 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
232 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
233#ifdef CONFIG_BNXT_SRIOV
234 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
235 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
236 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
237 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
238 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
239 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
240 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
241 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
242 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
243 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
244 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
245 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
246 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
247 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
248 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
249 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
250 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
251 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
252 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
253 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
254 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
255#endif
256 { 0 }
257};
258
259MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
260
261static const u16 bnxt_vf_req_snif[] = {
262 HWRM_FUNC_CFG,
263 HWRM_FUNC_VF_CFG,
264 HWRM_PORT_PHY_QCFG,
265 HWRM_CFA_L2_FILTER_ALLOC,
266};
267
268static const u16 bnxt_async_events_arr[] = {
269 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
270 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
271 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
272 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
273 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
274 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
275 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
276 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
277 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
278 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
279 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
280 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
281};
282
283static struct workqueue_struct *bnxt_pf_wq;
284
285static bool bnxt_vf_pciid(enum board_idx idx)
286{
287 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
288 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
289 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
290 idx == NETXTREME_E_P5_VF_HV);
291}
292
293#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
294#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
295#define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
296
297#define BNXT_CP_DB_IRQ_DIS(db) \
298 writel(DB_CP_IRQ_DIS_FLAGS, db)
299
300#define BNXT_DB_CQ(db, idx) \
301 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
302
303#define BNXT_DB_NQ_P5(db, idx) \
304 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
305
306#define BNXT_DB_CQ_ARM(db, idx) \
307 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
308
309#define BNXT_DB_NQ_ARM_P5(db, idx) \
310 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
311
312static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
313{
314 if (bp->flags & BNXT_FLAG_CHIP_P5)
315 BNXT_DB_NQ_P5(db, idx);
316 else
317 BNXT_DB_CQ(db, idx);
318}
319
320static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
321{
322 if (bp->flags & BNXT_FLAG_CHIP_P5)
323 BNXT_DB_NQ_ARM_P5(db, idx);
324 else
325 BNXT_DB_CQ_ARM(db, idx);
326}
327
328static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
329{
330 if (bp->flags & BNXT_FLAG_CHIP_P5)
331 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
332 db->doorbell);
333 else
334 BNXT_DB_CQ(db, idx);
335}
336
337const u16 bnxt_lhint_arr[] = {
338 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
339 TX_BD_FLAGS_LHINT_512_TO_1023,
340 TX_BD_FLAGS_LHINT_1024_TO_2047,
341 TX_BD_FLAGS_LHINT_1024_TO_2047,
342 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
343 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
344 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
345 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
346 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
347 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
348 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
349 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
350 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
351 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
352 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
353 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
354 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
355 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
356 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
357};
358
359static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
360{
361 struct metadata_dst *md_dst = skb_metadata_dst(skb);
362
363 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
364 return 0;
365
366 return md_dst->u.port_info.port_id;
367}
368
369static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
370 u16 prod)
371{
372 bnxt_db_write(bp, &txr->tx_db, prod);
373 txr->kick_pending = 0;
374}
375
376static bool bnxt_txr_netif_try_stop_queue(struct bnxt *bp,
377 struct bnxt_tx_ring_info *txr,
378 struct netdev_queue *txq)
379{
380 netif_tx_stop_queue(txq);
381
382
383
384
385
386
387 smp_mb();
388 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh) {
389 netif_tx_wake_queue(txq);
390 return false;
391 }
392
393 return true;
394}
395
396static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
397{
398 struct bnxt *bp = netdev_priv(dev);
399 struct tx_bd *txbd;
400 struct tx_bd_ext *txbd1;
401 struct netdev_queue *txq;
402 int i;
403 dma_addr_t mapping;
404 unsigned int length, pad = 0;
405 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
406 u16 prod, last_frag;
407 struct pci_dev *pdev = bp->pdev;
408 struct bnxt_tx_ring_info *txr;
409 struct bnxt_sw_tx_bd *tx_buf;
410 __le32 lflags = 0;
411
412 i = skb_get_queue_mapping(skb);
413 if (unlikely(i >= bp->tx_nr_rings)) {
414 dev_kfree_skb_any(skb);
415 atomic_long_inc(&dev->tx_dropped);
416 return NETDEV_TX_OK;
417 }
418
419 txq = netdev_get_tx_queue(dev, i);
420 txr = &bp->tx_ring[bp->tx_ring_map[i]];
421 prod = txr->tx_prod;
422
423 free_size = bnxt_tx_avail(bp, txr);
424 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
425
426 if (net_ratelimit() && txr->kick_pending)
427 netif_warn(bp, tx_err, dev,
428 "bnxt: ring busy w/ flush pending!\n");
429 if (bnxt_txr_netif_try_stop_queue(bp, txr, txq))
430 return NETDEV_TX_BUSY;
431 }
432
433 length = skb->len;
434 len = skb_headlen(skb);
435 last_frag = skb_shinfo(skb)->nr_frags;
436
437 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
438
439 txbd->tx_bd_opaque = prod;
440
441 tx_buf = &txr->tx_buf_ring[prod];
442 tx_buf->skb = skb;
443 tx_buf->nr_frags = last_frag;
444
445 vlan_tag_flags = 0;
446 cfa_action = bnxt_xmit_get_cfa_action(skb);
447 if (skb_vlan_tag_present(skb)) {
448 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
449 skb_vlan_tag_get(skb);
450
451
452
453 if (skb->vlan_proto == htons(ETH_P_8021Q))
454 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
455 }
456
457 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
458 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
459
460 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
461 atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
462 if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
463 &ptp->tx_hdr_off)) {
464 if (vlan_tag_flags)
465 ptp->tx_hdr_off += VLAN_HLEN;
466 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
467 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
468 } else {
469 atomic_inc(&bp->ptp_cfg->tx_avail);
470 }
471 }
472 }
473
474 if (unlikely(skb->no_fcs))
475 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
476
477 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
478 !lflags) {
479 struct tx_push_buffer *tx_push_buf = txr->tx_push;
480 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
481 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
482 void __iomem *db = txr->tx_db.doorbell;
483 void *pdata = tx_push_buf->data;
484 u64 *end;
485 int j, push_len;
486
487
488 tx_push->tx_bd_len_flags_type =
489 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
490 TX_BD_TYPE_LONG_TX_BD |
491 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
492 TX_BD_FLAGS_COAL_NOW |
493 TX_BD_FLAGS_PACKET_END |
494 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
495
496 if (skb->ip_summed == CHECKSUM_PARTIAL)
497 tx_push1->tx_bd_hsize_lflags =
498 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
499 else
500 tx_push1->tx_bd_hsize_lflags = 0;
501
502 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
503 tx_push1->tx_bd_cfa_action =
504 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
505
506 end = pdata + length;
507 end = PTR_ALIGN(end, 8) - 1;
508 *end = 0;
509
510 skb_copy_from_linear_data(skb, pdata, len);
511 pdata += len;
512 for (j = 0; j < last_frag; j++) {
513 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
514 void *fptr;
515
516 fptr = skb_frag_address_safe(frag);
517 if (!fptr)
518 goto normal_tx;
519
520 memcpy(pdata, fptr, skb_frag_size(frag));
521 pdata += skb_frag_size(frag);
522 }
523
524 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
525 txbd->tx_bd_haddr = txr->data_mapping;
526 prod = NEXT_TX(prod);
527 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
528 memcpy(txbd, tx_push1, sizeof(*txbd));
529 prod = NEXT_TX(prod);
530 tx_push->doorbell =
531 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
532 txr->tx_prod = prod;
533
534 tx_buf->is_push = 1;
535 netdev_tx_sent_queue(txq, skb->len);
536 wmb();
537
538 push_len = (length + sizeof(*tx_push) + 7) / 8;
539 if (push_len > 16) {
540 __iowrite64_copy(db, tx_push_buf, 16);
541 __iowrite32_copy(db + 4, tx_push_buf + 1,
542 (push_len - 16) << 1);
543 } else {
544 __iowrite64_copy(db, tx_push_buf, push_len);
545 }
546
547 goto tx_done;
548 }
549
550normal_tx:
551 if (length < BNXT_MIN_PKT_SIZE) {
552 pad = BNXT_MIN_PKT_SIZE - length;
553 if (skb_pad(skb, pad))
554
555 goto tx_kick_pending;
556 length = BNXT_MIN_PKT_SIZE;
557 }
558
559 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
560
561 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
562 goto tx_free;
563
564 dma_unmap_addr_set(tx_buf, mapping, mapping);
565 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
566 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
567
568 txbd->tx_bd_haddr = cpu_to_le64(mapping);
569
570 prod = NEXT_TX(prod);
571 txbd1 = (struct tx_bd_ext *)
572 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
573
574 txbd1->tx_bd_hsize_lflags = lflags;
575 if (skb_is_gso(skb)) {
576 u32 hdr_len;
577
578 if (skb->encapsulation)
579 hdr_len = skb_inner_network_offset(skb) +
580 skb_inner_network_header_len(skb) +
581 inner_tcp_hdrlen(skb);
582 else
583 hdr_len = skb_transport_offset(skb) +
584 tcp_hdrlen(skb);
585
586 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
587 TX_BD_FLAGS_T_IPID |
588 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
589 length = skb_shinfo(skb)->gso_size;
590 txbd1->tx_bd_mss = cpu_to_le32(length);
591 length += hdr_len;
592 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
593 txbd1->tx_bd_hsize_lflags |=
594 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
595 txbd1->tx_bd_mss = 0;
596 }
597
598 length >>= 9;
599 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
600 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
601 skb->len);
602 i = 0;
603 goto tx_dma_error;
604 }
605 flags |= bnxt_lhint_arr[length];
606 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
607
608 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
609 txbd1->tx_bd_cfa_action =
610 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
611 for (i = 0; i < last_frag; i++) {
612 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
613
614 prod = NEXT_TX(prod);
615 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
616
617 len = skb_frag_size(frag);
618 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
619 DMA_TO_DEVICE);
620
621 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
622 goto tx_dma_error;
623
624 tx_buf = &txr->tx_buf_ring[prod];
625 dma_unmap_addr_set(tx_buf, mapping, mapping);
626
627 txbd->tx_bd_haddr = cpu_to_le64(mapping);
628
629 flags = len << TX_BD_LEN_SHIFT;
630 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
631 }
632
633 flags &= ~TX_BD_LEN;
634 txbd->tx_bd_len_flags_type =
635 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
636 TX_BD_FLAGS_PACKET_END);
637
638 netdev_tx_sent_queue(txq, skb->len);
639
640 skb_tx_timestamp(skb);
641
642
643 wmb();
644
645 prod = NEXT_TX(prod);
646 txr->tx_prod = prod;
647
648 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
649 bnxt_txr_db_kick(bp, txr, prod);
650 else
651 txr->kick_pending = 1;
652
653tx_done:
654
655 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
656 if (netdev_xmit_more() && !tx_buf->is_push)
657 bnxt_txr_db_kick(bp, txr, prod);
658
659 bnxt_txr_netif_try_stop_queue(bp, txr, txq);
660 }
661 return NETDEV_TX_OK;
662
663tx_dma_error:
664 if (BNXT_TX_PTP_IS_SET(lflags))
665 atomic_inc(&bp->ptp_cfg->tx_avail);
666
667 last_frag = i;
668
669
670 prod = txr->tx_prod;
671 tx_buf = &txr->tx_buf_ring[prod];
672 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
673 skb_headlen(skb), PCI_DMA_TODEVICE);
674 prod = NEXT_TX(prod);
675
676
677 for (i = 0; i < last_frag; i++) {
678 prod = NEXT_TX(prod);
679 tx_buf = &txr->tx_buf_ring[prod];
680 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
681 skb_frag_size(&skb_shinfo(skb)->frags[i]),
682 PCI_DMA_TODEVICE);
683 }
684
685tx_free:
686 dev_kfree_skb_any(skb);
687tx_kick_pending:
688 if (txr->kick_pending)
689 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
690 txr->tx_buf_ring[txr->tx_prod].skb = NULL;
691 atomic_long_inc(&dev->tx_dropped);
692 return NETDEV_TX_OK;
693}
694
695static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
696{
697 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
698 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
699 u16 cons = txr->tx_cons;
700 struct pci_dev *pdev = bp->pdev;
701 int i;
702 unsigned int tx_bytes = 0;
703
704 for (i = 0; i < nr_pkts; i++) {
705 struct bnxt_sw_tx_bd *tx_buf;
706 bool compl_deferred = false;
707 struct sk_buff *skb;
708 int j, last;
709
710 tx_buf = &txr->tx_buf_ring[cons];
711 cons = NEXT_TX(cons);
712 skb = tx_buf->skb;
713 tx_buf->skb = NULL;
714
715 if (tx_buf->is_push) {
716 tx_buf->is_push = 0;
717 goto next_tx_int;
718 }
719
720 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
721 skb_headlen(skb), PCI_DMA_TODEVICE);
722 last = tx_buf->nr_frags;
723
724 for (j = 0; j < last; j++) {
725 cons = NEXT_TX(cons);
726 tx_buf = &txr->tx_buf_ring[cons];
727 dma_unmap_page(
728 &pdev->dev,
729 dma_unmap_addr(tx_buf, mapping),
730 skb_frag_size(&skb_shinfo(skb)->frags[j]),
731 PCI_DMA_TODEVICE);
732 }
733 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
734 if (bp->flags & BNXT_FLAG_CHIP_P5) {
735 if (!bnxt_get_tx_ts_p5(bp, skb))
736 compl_deferred = true;
737 else
738 atomic_inc(&bp->ptp_cfg->tx_avail);
739 }
740 }
741
742next_tx_int:
743 cons = NEXT_TX(cons);
744
745 tx_bytes += skb->len;
746 if (!compl_deferred)
747 dev_kfree_skb_any(skb);
748 }
749
750 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
751 txr->tx_cons = cons;
752
753
754
755
756
757
758 smp_mb();
759
760 if (unlikely(netif_tx_queue_stopped(txq)) &&
761 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
762 READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING)
763 netif_tx_wake_queue(txq);
764}
765
766static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
767 struct bnxt_rx_ring_info *rxr,
768 gfp_t gfp)
769{
770 struct device *dev = &bp->pdev->dev;
771 struct page *page;
772
773 page = page_pool_dev_alloc_pages(rxr->page_pool);
774 if (!page)
775 return NULL;
776
777 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
778 DMA_ATTR_WEAK_ORDERING);
779 if (dma_mapping_error(dev, *mapping)) {
780 page_pool_recycle_direct(rxr->page_pool, page);
781 return NULL;
782 }
783 *mapping += bp->rx_dma_offset;
784 return page;
785}
786
787static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
788 gfp_t gfp)
789{
790 u8 *data;
791 struct pci_dev *pdev = bp->pdev;
792
793 data = kmalloc(bp->rx_buf_size, gfp);
794 if (!data)
795 return NULL;
796
797 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
798 bp->rx_buf_use_size, bp->rx_dir,
799 DMA_ATTR_WEAK_ORDERING);
800
801 if (dma_mapping_error(&pdev->dev, *mapping)) {
802 kfree(data);
803 data = NULL;
804 }
805 return data;
806}
807
808int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
809 u16 prod, gfp_t gfp)
810{
811 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
812 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
813 dma_addr_t mapping;
814
815 if (BNXT_RX_PAGE_MODE(bp)) {
816 struct page *page =
817 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
818
819 if (!page)
820 return -ENOMEM;
821
822 rx_buf->data = page;
823 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
824 } else {
825 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
826
827 if (!data)
828 return -ENOMEM;
829
830 rx_buf->data = data;
831 rx_buf->data_ptr = data + bp->rx_offset;
832 }
833 rx_buf->mapping = mapping;
834
835 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
836 return 0;
837}
838
839void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
840{
841 u16 prod = rxr->rx_prod;
842 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
843 struct rx_bd *cons_bd, *prod_bd;
844
845 prod_rx_buf = &rxr->rx_buf_ring[prod];
846 cons_rx_buf = &rxr->rx_buf_ring[cons];
847
848 prod_rx_buf->data = data;
849 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
850
851 prod_rx_buf->mapping = cons_rx_buf->mapping;
852
853 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
854 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
855
856 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
857}
858
859static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
860{
861 u16 next, max = rxr->rx_agg_bmap_size;
862
863 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
864 if (next >= max)
865 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
866 return next;
867}
868
869static inline int bnxt_alloc_rx_page(struct bnxt *bp,
870 struct bnxt_rx_ring_info *rxr,
871 u16 prod, gfp_t gfp)
872{
873 struct rx_bd *rxbd =
874 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
875 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
876 struct pci_dev *pdev = bp->pdev;
877 struct page *page;
878 dma_addr_t mapping;
879 u16 sw_prod = rxr->rx_sw_agg_prod;
880 unsigned int offset = 0;
881
882 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
883 page = rxr->rx_page;
884 if (!page) {
885 page = alloc_page(gfp);
886 if (!page)
887 return -ENOMEM;
888 rxr->rx_page = page;
889 rxr->rx_page_offset = 0;
890 }
891 offset = rxr->rx_page_offset;
892 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
893 if (rxr->rx_page_offset == PAGE_SIZE)
894 rxr->rx_page = NULL;
895 else
896 get_page(page);
897 } else {
898 page = alloc_page(gfp);
899 if (!page)
900 return -ENOMEM;
901 }
902
903 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
904 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
905 DMA_ATTR_WEAK_ORDERING);
906 if (dma_mapping_error(&pdev->dev, mapping)) {
907 __free_page(page);
908 return -EIO;
909 }
910
911 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
912 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
913
914 __set_bit(sw_prod, rxr->rx_agg_bmap);
915 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
916 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
917
918 rx_agg_buf->page = page;
919 rx_agg_buf->offset = offset;
920 rx_agg_buf->mapping = mapping;
921 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
922 rxbd->rx_bd_opaque = sw_prod;
923 return 0;
924}
925
926static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
927 struct bnxt_cp_ring_info *cpr,
928 u16 cp_cons, u16 curr)
929{
930 struct rx_agg_cmp *agg;
931
932 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
933 agg = (struct rx_agg_cmp *)
934 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
935 return agg;
936}
937
938static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
939 struct bnxt_rx_ring_info *rxr,
940 u16 agg_id, u16 curr)
941{
942 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
943
944 return &tpa_info->agg_arr[curr];
945}
946
947static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
948 u16 start, u32 agg_bufs, bool tpa)
949{
950 struct bnxt_napi *bnapi = cpr->bnapi;
951 struct bnxt *bp = bnapi->bp;
952 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
953 u16 prod = rxr->rx_agg_prod;
954 u16 sw_prod = rxr->rx_sw_agg_prod;
955 bool p5_tpa = false;
956 u32 i;
957
958 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
959 p5_tpa = true;
960
961 for (i = 0; i < agg_bufs; i++) {
962 u16 cons;
963 struct rx_agg_cmp *agg;
964 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
965 struct rx_bd *prod_bd;
966 struct page *page;
967
968 if (p5_tpa)
969 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
970 else
971 agg = bnxt_get_agg(bp, cpr, idx, start + i);
972 cons = agg->rx_agg_cmp_opaque;
973 __clear_bit(cons, rxr->rx_agg_bmap);
974
975 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
976 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
977
978 __set_bit(sw_prod, rxr->rx_agg_bmap);
979 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
980 cons_rx_buf = &rxr->rx_agg_ring[cons];
981
982
983
984
985 page = cons_rx_buf->page;
986 cons_rx_buf->page = NULL;
987 prod_rx_buf->page = page;
988 prod_rx_buf->offset = cons_rx_buf->offset;
989
990 prod_rx_buf->mapping = cons_rx_buf->mapping;
991
992 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
993
994 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
995 prod_bd->rx_bd_opaque = sw_prod;
996
997 prod = NEXT_RX_AGG(prod);
998 sw_prod = NEXT_RX_AGG(sw_prod);
999 }
1000 rxr->rx_agg_prod = prod;
1001 rxr->rx_sw_agg_prod = sw_prod;
1002}
1003
1004static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1005 struct bnxt_rx_ring_info *rxr,
1006 u16 cons, void *data, u8 *data_ptr,
1007 dma_addr_t dma_addr,
1008 unsigned int offset_and_len)
1009{
1010 unsigned int payload = offset_and_len >> 16;
1011 unsigned int len = offset_and_len & 0xffff;
1012 skb_frag_t *frag;
1013 struct page *page = data;
1014 u16 prod = rxr->rx_prod;
1015 struct sk_buff *skb;
1016 int off, err;
1017
1018 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1019 if (unlikely(err)) {
1020 bnxt_reuse_rx_data(rxr, cons, data);
1021 return NULL;
1022 }
1023 dma_addr -= bp->rx_dma_offset;
1024 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
1025 DMA_ATTR_WEAK_ORDERING);
1026 page_pool_release_page(rxr->page_pool, page);
1027
1028 if (unlikely(!payload))
1029 payload = eth_get_headlen(bp->dev, data_ptr, len);
1030
1031 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1032 if (!skb) {
1033 __free_page(page);
1034 return NULL;
1035 }
1036
1037 off = (void *)data_ptr - page_address(page);
1038 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
1039 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1040 payload + NET_IP_ALIGN);
1041
1042 frag = &skb_shinfo(skb)->frags[0];
1043 skb_frag_size_sub(frag, payload);
1044 skb_frag_off_add(frag, payload);
1045 skb->data_len -= payload;
1046 skb->tail += payload;
1047
1048 return skb;
1049}
1050
1051static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1052 struct bnxt_rx_ring_info *rxr, u16 cons,
1053 void *data, u8 *data_ptr,
1054 dma_addr_t dma_addr,
1055 unsigned int offset_and_len)
1056{
1057 u16 prod = rxr->rx_prod;
1058 struct sk_buff *skb;
1059 int err;
1060
1061 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1062 if (unlikely(err)) {
1063 bnxt_reuse_rx_data(rxr, cons, data);
1064 return NULL;
1065 }
1066
1067 skb = build_skb(data, 0);
1068 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1069 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1070 if (!skb) {
1071 kfree(data);
1072 return NULL;
1073 }
1074
1075 skb_reserve(skb, bp->rx_offset);
1076 skb_put(skb, offset_and_len & 0xffff);
1077 return skb;
1078}
1079
1080static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1081 struct bnxt_cp_ring_info *cpr,
1082 struct sk_buff *skb, u16 idx,
1083 u32 agg_bufs, bool tpa)
1084{
1085 struct bnxt_napi *bnapi = cpr->bnapi;
1086 struct pci_dev *pdev = bp->pdev;
1087 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1088 u16 prod = rxr->rx_agg_prod;
1089 bool p5_tpa = false;
1090 u32 i;
1091
1092 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1093 p5_tpa = true;
1094
1095 for (i = 0; i < agg_bufs; i++) {
1096 u16 cons, frag_len;
1097 struct rx_agg_cmp *agg;
1098 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1099 struct page *page;
1100 dma_addr_t mapping;
1101
1102 if (p5_tpa)
1103 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1104 else
1105 agg = bnxt_get_agg(bp, cpr, idx, i);
1106 cons = agg->rx_agg_cmp_opaque;
1107 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1108 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1109
1110 cons_rx_buf = &rxr->rx_agg_ring[cons];
1111 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1112 cons_rx_buf->offset, frag_len);
1113 __clear_bit(cons, rxr->rx_agg_bmap);
1114
1115
1116
1117
1118
1119 mapping = cons_rx_buf->mapping;
1120 page = cons_rx_buf->page;
1121 cons_rx_buf->page = NULL;
1122
1123 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1124 struct skb_shared_info *shinfo;
1125 unsigned int nr_frags;
1126
1127 shinfo = skb_shinfo(skb);
1128 nr_frags = --shinfo->nr_frags;
1129 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1130
1131 dev_kfree_skb(skb);
1132
1133 cons_rx_buf->page = page;
1134
1135
1136
1137
1138 rxr->rx_agg_prod = prod;
1139 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1140 return NULL;
1141 }
1142
1143 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1144 PCI_DMA_FROMDEVICE,
1145 DMA_ATTR_WEAK_ORDERING);
1146
1147 skb->data_len += frag_len;
1148 skb->len += frag_len;
1149 skb->truesize += PAGE_SIZE;
1150
1151 prod = NEXT_RX_AGG(prod);
1152 }
1153 rxr->rx_agg_prod = prod;
1154 return skb;
1155}
1156
1157static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1158 u8 agg_bufs, u32 *raw_cons)
1159{
1160 u16 last;
1161 struct rx_agg_cmp *agg;
1162
1163 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1164 last = RING_CMP(*raw_cons);
1165 agg = (struct rx_agg_cmp *)
1166 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1167 return RX_AGG_CMP_VALID(agg, *raw_cons);
1168}
1169
1170static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1171 unsigned int len,
1172 dma_addr_t mapping)
1173{
1174 struct bnxt *bp = bnapi->bp;
1175 struct pci_dev *pdev = bp->pdev;
1176 struct sk_buff *skb;
1177
1178 skb = napi_alloc_skb(&bnapi->napi, len);
1179 if (!skb)
1180 return NULL;
1181
1182 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1183 bp->rx_dir);
1184
1185 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1186 len + NET_IP_ALIGN);
1187
1188 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1189 bp->rx_dir);
1190
1191 skb_put(skb, len);
1192 return skb;
1193}
1194
1195static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1196 u32 *raw_cons, void *cmp)
1197{
1198 struct rx_cmp *rxcmp = cmp;
1199 u32 tmp_raw_cons = *raw_cons;
1200 u8 cmp_type, agg_bufs = 0;
1201
1202 cmp_type = RX_CMP_TYPE(rxcmp);
1203
1204 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1205 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1206 RX_CMP_AGG_BUFS) >>
1207 RX_CMP_AGG_BUFS_SHIFT;
1208 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1209 struct rx_tpa_end_cmp *tpa_end = cmp;
1210
1211 if (bp->flags & BNXT_FLAG_CHIP_P5)
1212 return 0;
1213
1214 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1215 }
1216
1217 if (agg_bufs) {
1218 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1219 return -EBUSY;
1220 }
1221 *raw_cons = tmp_raw_cons;
1222 return 0;
1223}
1224
1225static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1226{
1227 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1228 return;
1229
1230 if (BNXT_PF(bp))
1231 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1232 else
1233 schedule_delayed_work(&bp->fw_reset_task, delay);
1234}
1235
1236static void bnxt_queue_sp_work(struct bnxt *bp)
1237{
1238 if (BNXT_PF(bp))
1239 queue_work(bnxt_pf_wq, &bp->sp_task);
1240 else
1241 schedule_work(&bp->sp_task);
1242}
1243
1244static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1245{
1246 if (!rxr->bnapi->in_reset) {
1247 rxr->bnapi->in_reset = true;
1248 if (bp->flags & BNXT_FLAG_CHIP_P5)
1249 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1250 else
1251 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1252 bnxt_queue_sp_work(bp);
1253 }
1254 rxr->rx_next_cons = 0xffff;
1255}
1256
1257static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1258{
1259 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1260 u16 idx = agg_id & MAX_TPA_P5_MASK;
1261
1262 if (test_bit(idx, map->agg_idx_bmap))
1263 idx = find_first_zero_bit(map->agg_idx_bmap,
1264 BNXT_AGG_IDX_BMAP_SIZE);
1265 __set_bit(idx, map->agg_idx_bmap);
1266 map->agg_id_tbl[agg_id] = idx;
1267 return idx;
1268}
1269
1270static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1271{
1272 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1273
1274 __clear_bit(idx, map->agg_idx_bmap);
1275}
1276
1277static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1278{
1279 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1280
1281 return map->agg_id_tbl[agg_id];
1282}
1283
1284static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1285 struct rx_tpa_start_cmp *tpa_start,
1286 struct rx_tpa_start_cmp_ext *tpa_start1)
1287{
1288 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1289 struct bnxt_tpa_info *tpa_info;
1290 u16 cons, prod, agg_id;
1291 struct rx_bd *prod_bd;
1292 dma_addr_t mapping;
1293
1294 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1295 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1296 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1297 } else {
1298 agg_id = TPA_START_AGG_ID(tpa_start);
1299 }
1300 cons = tpa_start->rx_tpa_start_cmp_opaque;
1301 prod = rxr->rx_prod;
1302 cons_rx_buf = &rxr->rx_buf_ring[cons];
1303 prod_rx_buf = &rxr->rx_buf_ring[prod];
1304 tpa_info = &rxr->rx_tpa[agg_id];
1305
1306 if (unlikely(cons != rxr->rx_next_cons ||
1307 TPA_START_ERROR(tpa_start))) {
1308 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1309 cons, rxr->rx_next_cons,
1310 TPA_START_ERROR_CODE(tpa_start1));
1311 bnxt_sched_reset(bp, rxr);
1312 return;
1313 }
1314
1315
1316
1317 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1318 prod_rx_buf->data = tpa_info->data;
1319 prod_rx_buf->data_ptr = tpa_info->data_ptr;
1320
1321 mapping = tpa_info->mapping;
1322 prod_rx_buf->mapping = mapping;
1323
1324 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1325
1326 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1327
1328 tpa_info->data = cons_rx_buf->data;
1329 tpa_info->data_ptr = cons_rx_buf->data_ptr;
1330 cons_rx_buf->data = NULL;
1331 tpa_info->mapping = cons_rx_buf->mapping;
1332
1333 tpa_info->len =
1334 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1335 RX_TPA_START_CMP_LEN_SHIFT;
1336 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1337 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1338
1339 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1340 tpa_info->gso_type = SKB_GSO_TCPV4;
1341
1342 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1343 tpa_info->gso_type = SKB_GSO_TCPV6;
1344 tpa_info->rss_hash =
1345 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1346 } else {
1347 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1348 tpa_info->gso_type = 0;
1349 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1350 }
1351 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1352 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1353 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1354 tpa_info->agg_count = 0;
1355
1356 rxr->rx_prod = NEXT_RX(prod);
1357 cons = NEXT_RX(cons);
1358 rxr->rx_next_cons = NEXT_RX(cons);
1359 cons_rx_buf = &rxr->rx_buf_ring[cons];
1360
1361 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1362 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1363 cons_rx_buf->data = NULL;
1364}
1365
1366static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1367{
1368 if (agg_bufs)
1369 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1370}
1371
1372#ifdef CONFIG_INET
1373static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1374{
1375 struct udphdr *uh = NULL;
1376
1377 if (ip_proto == htons(ETH_P_IP)) {
1378 struct iphdr *iph = (struct iphdr *)skb->data;
1379
1380 if (iph->protocol == IPPROTO_UDP)
1381 uh = (struct udphdr *)(iph + 1);
1382 } else {
1383 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1384
1385 if (iph->nexthdr == IPPROTO_UDP)
1386 uh = (struct udphdr *)(iph + 1);
1387 }
1388 if (uh) {
1389 if (uh->check)
1390 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1391 else
1392 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1393 }
1394}
1395#endif
1396
1397static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1398 int payload_off, int tcp_ts,
1399 struct sk_buff *skb)
1400{
1401#ifdef CONFIG_INET
1402 struct tcphdr *th;
1403 int len, nw_off;
1404 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1405 u32 hdr_info = tpa_info->hdr_info;
1406 bool loopback = false;
1407
1408 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1409 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1410 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1411
1412
1413
1414
1415 if (inner_mac_off == 4) {
1416 loopback = true;
1417 } else if (inner_mac_off > 4) {
1418 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1419 ETH_HLEN - 2));
1420
1421
1422
1423
1424
1425 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1426 loopback = true;
1427 }
1428 if (loopback) {
1429
1430 inner_ip_off -= 4;
1431 inner_mac_off -= 4;
1432 outer_ip_off -= 4;
1433 }
1434
1435 nw_off = inner_ip_off - ETH_HLEN;
1436 skb_set_network_header(skb, nw_off);
1437 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1438 struct ipv6hdr *iph = ipv6_hdr(skb);
1439
1440 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1441 len = skb->len - skb_transport_offset(skb);
1442 th = tcp_hdr(skb);
1443 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1444 } else {
1445 struct iphdr *iph = ip_hdr(skb);
1446
1447 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1448 len = skb->len - skb_transport_offset(skb);
1449 th = tcp_hdr(skb);
1450 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1451 }
1452
1453 if (inner_mac_off) {
1454 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1455 ETH_HLEN - 2));
1456
1457 bnxt_gro_tunnel(skb, proto);
1458 }
1459#endif
1460 return skb;
1461}
1462
1463static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1464 int payload_off, int tcp_ts,
1465 struct sk_buff *skb)
1466{
1467#ifdef CONFIG_INET
1468 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1469 u32 hdr_info = tpa_info->hdr_info;
1470 int iphdr_len, nw_off;
1471
1472 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1473 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1474 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1475
1476 nw_off = inner_ip_off - ETH_HLEN;
1477 skb_set_network_header(skb, nw_off);
1478 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1479 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1480 skb_set_transport_header(skb, nw_off + iphdr_len);
1481
1482 if (inner_mac_off) {
1483 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1484 ETH_HLEN - 2));
1485
1486 bnxt_gro_tunnel(skb, proto);
1487 }
1488#endif
1489 return skb;
1490}
1491
1492#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1493#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1494
1495static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1496 int payload_off, int tcp_ts,
1497 struct sk_buff *skb)
1498{
1499#ifdef CONFIG_INET
1500 struct tcphdr *th;
1501 int len, nw_off, tcp_opt_len = 0;
1502
1503 if (tcp_ts)
1504 tcp_opt_len = 12;
1505
1506 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1507 struct iphdr *iph;
1508
1509 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1510 ETH_HLEN;
1511 skb_set_network_header(skb, nw_off);
1512 iph = ip_hdr(skb);
1513 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1514 len = skb->len - skb_transport_offset(skb);
1515 th = tcp_hdr(skb);
1516 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1517 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1518 struct ipv6hdr *iph;
1519
1520 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1521 ETH_HLEN;
1522 skb_set_network_header(skb, nw_off);
1523 iph = ipv6_hdr(skb);
1524 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1525 len = skb->len - skb_transport_offset(skb);
1526 th = tcp_hdr(skb);
1527 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1528 } else {
1529 dev_kfree_skb_any(skb);
1530 return NULL;
1531 }
1532
1533 if (nw_off)
1534 bnxt_gro_tunnel(skb, skb->protocol);
1535#endif
1536 return skb;
1537}
1538
1539static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1540 struct bnxt_tpa_info *tpa_info,
1541 struct rx_tpa_end_cmp *tpa_end,
1542 struct rx_tpa_end_cmp_ext *tpa_end1,
1543 struct sk_buff *skb)
1544{
1545#ifdef CONFIG_INET
1546 int payload_off;
1547 u16 segs;
1548
1549 segs = TPA_END_TPA_SEGS(tpa_end);
1550 if (segs == 1)
1551 return skb;
1552
1553 NAPI_GRO_CB(skb)->count = segs;
1554 skb_shinfo(skb)->gso_size =
1555 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1556 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1557 if (bp->flags & BNXT_FLAG_CHIP_P5)
1558 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1559 else
1560 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1561 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1562 if (likely(skb))
1563 tcp_gro_complete(skb);
1564#endif
1565 return skb;
1566}
1567
1568
1569
1570
1571static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1572{
1573 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1574
1575
1576 return dev ? dev : bp->dev;
1577}
1578
1579static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1580 struct bnxt_cp_ring_info *cpr,
1581 u32 *raw_cons,
1582 struct rx_tpa_end_cmp *tpa_end,
1583 struct rx_tpa_end_cmp_ext *tpa_end1,
1584 u8 *event)
1585{
1586 struct bnxt_napi *bnapi = cpr->bnapi;
1587 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1588 u8 *data_ptr, agg_bufs;
1589 unsigned int len;
1590 struct bnxt_tpa_info *tpa_info;
1591 dma_addr_t mapping;
1592 struct sk_buff *skb;
1593 u16 idx = 0, agg_id;
1594 void *data;
1595 bool gro;
1596
1597 if (unlikely(bnapi->in_reset)) {
1598 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1599
1600 if (rc < 0)
1601 return ERR_PTR(-EBUSY);
1602 return NULL;
1603 }
1604
1605 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1606 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1607 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1608 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1609 tpa_info = &rxr->rx_tpa[agg_id];
1610 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1611 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1612 agg_bufs, tpa_info->agg_count);
1613 agg_bufs = tpa_info->agg_count;
1614 }
1615 tpa_info->agg_count = 0;
1616 *event |= BNXT_AGG_EVENT;
1617 bnxt_free_agg_idx(rxr, agg_id);
1618 idx = agg_id;
1619 gro = !!(bp->flags & BNXT_FLAG_GRO);
1620 } else {
1621 agg_id = TPA_END_AGG_ID(tpa_end);
1622 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1623 tpa_info = &rxr->rx_tpa[agg_id];
1624 idx = RING_CMP(*raw_cons);
1625 if (agg_bufs) {
1626 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1627 return ERR_PTR(-EBUSY);
1628
1629 *event |= BNXT_AGG_EVENT;
1630 idx = NEXT_CMP(idx);
1631 }
1632 gro = !!TPA_END_GRO(tpa_end);
1633 }
1634 data = tpa_info->data;
1635 data_ptr = tpa_info->data_ptr;
1636 prefetch(data_ptr);
1637 len = tpa_info->len;
1638 mapping = tpa_info->mapping;
1639
1640 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1641 bnxt_abort_tpa(cpr, idx, agg_bufs);
1642 if (agg_bufs > MAX_SKB_FRAGS)
1643 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1644 agg_bufs, (int)MAX_SKB_FRAGS);
1645 return NULL;
1646 }
1647
1648 if (len <= bp->rx_copy_thresh) {
1649 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1650 if (!skb) {
1651 bnxt_abort_tpa(cpr, idx, agg_bufs);
1652 return NULL;
1653 }
1654 } else {
1655 u8 *new_data;
1656 dma_addr_t new_mapping;
1657
1658 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1659 if (!new_data) {
1660 bnxt_abort_tpa(cpr, idx, agg_bufs);
1661 return NULL;
1662 }
1663
1664 tpa_info->data = new_data;
1665 tpa_info->data_ptr = new_data + bp->rx_offset;
1666 tpa_info->mapping = new_mapping;
1667
1668 skb = build_skb(data, 0);
1669 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1670 bp->rx_buf_use_size, bp->rx_dir,
1671 DMA_ATTR_WEAK_ORDERING);
1672
1673 if (!skb) {
1674 kfree(data);
1675 bnxt_abort_tpa(cpr, idx, agg_bufs);
1676 return NULL;
1677 }
1678 skb_reserve(skb, bp->rx_offset);
1679 skb_put(skb, len);
1680 }
1681
1682 if (agg_bufs) {
1683 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1684 if (!skb) {
1685
1686 return NULL;
1687 }
1688 }
1689
1690 skb->protocol =
1691 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1692
1693 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1694 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1695
1696 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1697 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1698 __be16 vlan_proto = htons(tpa_info->metadata >>
1699 RX_CMP_FLAGS2_METADATA_TPID_SFT);
1700 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1701
1702 if (eth_type_vlan(vlan_proto)) {
1703 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1704 } else {
1705 dev_kfree_skb(skb);
1706 return NULL;
1707 }
1708 }
1709
1710 skb_checksum_none_assert(skb);
1711 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1712 skb->ip_summed = CHECKSUM_UNNECESSARY;
1713 skb->csum_level =
1714 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1715 }
1716
1717 if (gro)
1718 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1719
1720 return skb;
1721}
1722
1723static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1724 struct rx_agg_cmp *rx_agg)
1725{
1726 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1727 struct bnxt_tpa_info *tpa_info;
1728
1729 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1730 tpa_info = &rxr->rx_tpa[agg_id];
1731 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1732 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1733}
1734
1735static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1736 struct sk_buff *skb)
1737{
1738 if (skb->dev != bp->dev) {
1739
1740 bnxt_vf_rep_rx(bp, skb);
1741 return;
1742 }
1743 skb_record_rx_queue(skb, bnapi->index);
1744 napi_gro_receive(&bnapi->napi, skb);
1745}
1746
1747
1748
1749
1750
1751
1752
1753
1754static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1755 u32 *raw_cons, u8 *event)
1756{
1757 struct bnxt_napi *bnapi = cpr->bnapi;
1758 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1759 struct net_device *dev = bp->dev;
1760 struct rx_cmp *rxcmp;
1761 struct rx_cmp_ext *rxcmp1;
1762 u32 tmp_raw_cons = *raw_cons;
1763 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1764 struct bnxt_sw_rx_bd *rx_buf;
1765 unsigned int len;
1766 u8 *data_ptr, agg_bufs, cmp_type;
1767 dma_addr_t dma_addr;
1768 struct sk_buff *skb;
1769 u32 flags, misc;
1770 void *data;
1771 int rc = 0;
1772
1773 rxcmp = (struct rx_cmp *)
1774 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1775
1776 cmp_type = RX_CMP_TYPE(rxcmp);
1777
1778 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1779 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1780 goto next_rx_no_prod_no_len;
1781 }
1782
1783 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1784 cp_cons = RING_CMP(tmp_raw_cons);
1785 rxcmp1 = (struct rx_cmp_ext *)
1786 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1787
1788 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1789 return -EBUSY;
1790
1791
1792
1793
1794 dma_rmb();
1795 prod = rxr->rx_prod;
1796
1797 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1798 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1799 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1800
1801 *event |= BNXT_RX_EVENT;
1802 goto next_rx_no_prod_no_len;
1803
1804 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1805 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1806 (struct rx_tpa_end_cmp *)rxcmp,
1807 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1808
1809 if (IS_ERR(skb))
1810 return -EBUSY;
1811
1812 rc = -ENOMEM;
1813 if (likely(skb)) {
1814 bnxt_deliver_skb(bp, bnapi, skb);
1815 rc = 1;
1816 }
1817 *event |= BNXT_RX_EVENT;
1818 goto next_rx_no_prod_no_len;
1819 }
1820
1821 cons = rxcmp->rx_cmp_opaque;
1822 if (unlikely(cons != rxr->rx_next_cons)) {
1823 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1824
1825
1826 if (rxr->rx_next_cons != 0xffff)
1827 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1828 cons, rxr->rx_next_cons);
1829 bnxt_sched_reset(bp, rxr);
1830 if (rc1)
1831 return rc1;
1832 goto next_rx_no_prod_no_len;
1833 }
1834 rx_buf = &rxr->rx_buf_ring[cons];
1835 data = rx_buf->data;
1836 data_ptr = rx_buf->data_ptr;
1837 prefetch(data_ptr);
1838
1839 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1840 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1841
1842 if (agg_bufs) {
1843 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1844 return -EBUSY;
1845
1846 cp_cons = NEXT_CMP(cp_cons);
1847 *event |= BNXT_AGG_EVENT;
1848 }
1849 *event |= BNXT_RX_EVENT;
1850
1851 rx_buf->data = NULL;
1852 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1853 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1854
1855 bnxt_reuse_rx_data(rxr, cons, data);
1856 if (agg_bufs)
1857 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1858 false);
1859
1860 rc = -EIO;
1861 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1862 bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1863 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1864 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1865 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1866 rx_err);
1867 bnxt_sched_reset(bp, rxr);
1868 }
1869 }
1870 goto next_rx_no_len;
1871 }
1872
1873 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
1874 len = flags >> RX_CMP_LEN_SHIFT;
1875 dma_addr = rx_buf->mapping;
1876
1877 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1878 rc = 1;
1879 goto next_rx;
1880 }
1881
1882 if (len <= bp->rx_copy_thresh) {
1883 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1884 bnxt_reuse_rx_data(rxr, cons, data);
1885 if (!skb) {
1886 if (agg_bufs)
1887 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1888 agg_bufs, false);
1889 rc = -ENOMEM;
1890 goto next_rx;
1891 }
1892 } else {
1893 u32 payload;
1894
1895 if (rx_buf->data_ptr == data_ptr)
1896 payload = misc & RX_CMP_PAYLOAD_OFFSET;
1897 else
1898 payload = 0;
1899 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1900 payload | len);
1901 if (!skb) {
1902 rc = -ENOMEM;
1903 goto next_rx;
1904 }
1905 }
1906
1907 if (agg_bufs) {
1908 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1909 if (!skb) {
1910 rc = -ENOMEM;
1911 goto next_rx;
1912 }
1913 }
1914
1915 if (RX_CMP_HASH_VALID(rxcmp)) {
1916 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1917 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1918
1919
1920 if (hash_type != 1 && hash_type != 3)
1921 type = PKT_HASH_TYPE_L3;
1922 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1923 }
1924
1925 cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1926 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1927
1928 if ((rxcmp1->rx_cmp_flags2 &
1929 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1930 (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1931 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1932 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1933 __be16 vlan_proto = htons(meta_data >>
1934 RX_CMP_FLAGS2_METADATA_TPID_SFT);
1935
1936 if (eth_type_vlan(vlan_proto)) {
1937 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1938 } else {
1939 dev_kfree_skb(skb);
1940 goto next_rx;
1941 }
1942 }
1943
1944 skb_checksum_none_assert(skb);
1945 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1946 if (dev->features & NETIF_F_RXCSUM) {
1947 skb->ip_summed = CHECKSUM_UNNECESSARY;
1948 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1949 }
1950 } else {
1951 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1952 if (dev->features & NETIF_F_RXCSUM)
1953 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1954 }
1955 }
1956
1957 if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) ==
1958 RX_CMP_FLAGS_ITYPE_PTP_W_TS)) {
1959 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1960 u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1961 u64 ns, ts;
1962
1963 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
1964 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
1965
1966 spin_lock_bh(&ptp->ptp_lock);
1967 ns = timecounter_cyc2time(&ptp->tc, ts);
1968 spin_unlock_bh(&ptp->ptp_lock);
1969 memset(skb_hwtstamps(skb), 0,
1970 sizeof(*skb_hwtstamps(skb)));
1971 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
1972 }
1973 }
1974 }
1975 bnxt_deliver_skb(bp, bnapi, skb);
1976 rc = 1;
1977
1978next_rx:
1979 cpr->rx_packets += 1;
1980 cpr->rx_bytes += len;
1981
1982next_rx_no_len:
1983 rxr->rx_prod = NEXT_RX(prod);
1984 rxr->rx_next_cons = NEXT_RX(cons);
1985
1986next_rx_no_prod_no_len:
1987 *raw_cons = tmp_raw_cons;
1988
1989 return rc;
1990}
1991
1992
1993
1994
1995static int bnxt_force_rx_discard(struct bnxt *bp,
1996 struct bnxt_cp_ring_info *cpr,
1997 u32 *raw_cons, u8 *event)
1998{
1999 u32 tmp_raw_cons = *raw_cons;
2000 struct rx_cmp_ext *rxcmp1;
2001 struct rx_cmp *rxcmp;
2002 u16 cp_cons;
2003 u8 cmp_type;
2004
2005 cp_cons = RING_CMP(tmp_raw_cons);
2006 rxcmp = (struct rx_cmp *)
2007 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2008
2009 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2010 cp_cons = RING_CMP(tmp_raw_cons);
2011 rxcmp1 = (struct rx_cmp_ext *)
2012 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2013
2014 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2015 return -EBUSY;
2016
2017
2018
2019
2020 dma_rmb();
2021 cmp_type = RX_CMP_TYPE(rxcmp);
2022 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
2023 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2024 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2025 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2026 struct rx_tpa_end_cmp_ext *tpa_end1;
2027
2028 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2029 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2030 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2031 }
2032 return bnxt_rx_pkt(bp, cpr, raw_cons, event);
2033}
2034
2035u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2036{
2037 struct bnxt_fw_health *fw_health = bp->fw_health;
2038 u32 reg = fw_health->regs[reg_idx];
2039 u32 reg_type, reg_off, val = 0;
2040
2041 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2042 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2043 switch (reg_type) {
2044 case BNXT_FW_HEALTH_REG_TYPE_CFG:
2045 pci_read_config_dword(bp->pdev, reg_off, &val);
2046 break;
2047 case BNXT_FW_HEALTH_REG_TYPE_GRC:
2048 reg_off = fw_health->mapped_regs[reg_idx];
2049 fallthrough;
2050 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2051 val = readl(bp->bar0 + reg_off);
2052 break;
2053 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2054 val = readl(bp->bar1 + reg_off);
2055 break;
2056 }
2057 if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2058 val &= fw_health->fw_reset_inprog_reg_mask;
2059 return val;
2060}
2061
2062static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2063{
2064 int i;
2065
2066 for (i = 0; i < bp->rx_nr_rings; i++) {
2067 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2068 struct bnxt_ring_grp_info *grp_info;
2069
2070 grp_info = &bp->grp_info[grp_idx];
2071 if (grp_info->agg_fw_ring_id == ring_id)
2072 return grp_idx;
2073 }
2074 return INVALID_HW_RING_ID;
2075}
2076
2077#define BNXT_GET_EVENT_PORT(data) \
2078 ((data) & \
2079 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2080
2081#define BNXT_EVENT_RING_TYPE(data2) \
2082 ((data2) & \
2083 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2084
2085#define BNXT_EVENT_RING_TYPE_RX(data2) \
2086 (BNXT_EVENT_RING_TYPE(data2) == \
2087 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2088
2089static int bnxt_async_event_process(struct bnxt *bp,
2090 struct hwrm_async_event_cmpl *cmpl)
2091{
2092 u16 event_id = le16_to_cpu(cmpl->event_id);
2093 u32 data1 = le32_to_cpu(cmpl->event_data1);
2094 u32 data2 = le32_to_cpu(cmpl->event_data2);
2095
2096
2097 switch (event_id) {
2098 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2099 struct bnxt_link_info *link_info = &bp->link_info;
2100
2101 if (BNXT_VF(bp))
2102 goto async_event_process_exit;
2103
2104
2105 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2106 (data1 & 0x20000)) {
2107 u16 fw_speed = link_info->force_link_speed;
2108 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2109
2110 if (speed != SPEED_UNKNOWN)
2111 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2112 speed);
2113 }
2114 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2115 }
2116 fallthrough;
2117 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2118 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2119 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2120 fallthrough;
2121 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2122 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2123 break;
2124 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2125 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2126 break;
2127 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2128 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2129
2130 if (BNXT_VF(bp))
2131 break;
2132
2133 if (bp->pf.port_id != port_id)
2134 break;
2135
2136 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2137 break;
2138 }
2139 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2140 if (BNXT_PF(bp))
2141 goto async_event_process_exit;
2142 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2143 break;
2144 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2145 char *fatal_str = "non-fatal";
2146
2147 if (!bp->fw_health)
2148 goto async_event_process_exit;
2149
2150 bp->fw_reset_timestamp = jiffies;
2151 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2152 if (!bp->fw_reset_min_dsecs)
2153 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2154 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2155 if (!bp->fw_reset_max_dsecs)
2156 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2157 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2158 fatal_str = "fatal";
2159 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2160 }
2161 netif_warn(bp, hw, bp->dev,
2162 "Firmware %s reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2163 fatal_str, data1, data2,
2164 bp->fw_reset_min_dsecs * 100,
2165 bp->fw_reset_max_dsecs * 100);
2166 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2167 break;
2168 }
2169 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2170 struct bnxt_fw_health *fw_health = bp->fw_health;
2171
2172 if (!fw_health)
2173 goto async_event_process_exit;
2174
2175 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2176 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2177 if (!fw_health->enabled) {
2178 netif_info(bp, drv, bp->dev,
2179 "Error recovery info: error recovery[0]\n");
2180 break;
2181 }
2182 fw_health->tmr_multiplier =
2183 DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2184 bp->current_interval * 10);
2185 fw_health->tmr_counter = fw_health->tmr_multiplier;
2186 fw_health->last_fw_heartbeat =
2187 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2188 fw_health->last_fw_reset_cnt =
2189 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2190 netif_info(bp, drv, bp->dev,
2191 "Error recovery info: error recovery[1], master[%d], reset count[%u], health status: 0x%x\n",
2192 fw_health->master, fw_health->last_fw_reset_cnt,
2193 bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG));
2194 goto async_event_process_exit;
2195 }
2196 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2197 netif_notice(bp, hw, bp->dev,
2198 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2199 data1, data2);
2200 goto async_event_process_exit;
2201 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2202 struct bnxt_rx_ring_info *rxr;
2203 u16 grp_idx;
2204
2205 if (bp->flags & BNXT_FLAG_CHIP_P5)
2206 goto async_event_process_exit;
2207
2208 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2209 BNXT_EVENT_RING_TYPE(data2), data1);
2210 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2211 goto async_event_process_exit;
2212
2213 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2214 if (grp_idx == INVALID_HW_RING_ID) {
2215 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2216 data1);
2217 goto async_event_process_exit;
2218 }
2219 rxr = bp->bnapi[grp_idx]->rx_ring;
2220 bnxt_sched_reset(bp, rxr);
2221 goto async_event_process_exit;
2222 }
2223 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2224 struct bnxt_fw_health *fw_health = bp->fw_health;
2225
2226 netif_notice(bp, hw, bp->dev,
2227 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2228 data1, data2);
2229 if (fw_health) {
2230 fw_health->echo_req_data1 = data1;
2231 fw_health->echo_req_data2 = data2;
2232 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2233 break;
2234 }
2235 goto async_event_process_exit;
2236 }
2237 default:
2238 goto async_event_process_exit;
2239 }
2240 bnxt_queue_sp_work(bp);
2241async_event_process_exit:
2242 bnxt_ulp_async_events(bp, cmpl);
2243 return 0;
2244}
2245
2246static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2247{
2248 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2249 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2250 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2251 (struct hwrm_fwd_req_cmpl *)txcmp;
2252
2253 switch (cmpl_type) {
2254 case CMPL_BASE_TYPE_HWRM_DONE:
2255 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2256 if (seq_id == bp->hwrm_intr_seq_id)
2257 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2258 else
2259 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2260 break;
2261
2262 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2263 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2264
2265 if ((vf_id < bp->pf.first_vf_id) ||
2266 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2267 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2268 vf_id);
2269 return -EINVAL;
2270 }
2271
2272 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2273 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2274 bnxt_queue_sp_work(bp);
2275 break;
2276
2277 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2278 bnxt_async_event_process(bp,
2279 (struct hwrm_async_event_cmpl *)txcmp);
2280 break;
2281
2282 default:
2283 break;
2284 }
2285
2286 return 0;
2287}
2288
2289static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2290{
2291 struct bnxt_napi *bnapi = dev_instance;
2292 struct bnxt *bp = bnapi->bp;
2293 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2294 u32 cons = RING_CMP(cpr->cp_raw_cons);
2295
2296 cpr->event_ctr++;
2297 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2298 napi_schedule(&bnapi->napi);
2299 return IRQ_HANDLED;
2300}
2301
2302static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2303{
2304 u32 raw_cons = cpr->cp_raw_cons;
2305 u16 cons = RING_CMP(raw_cons);
2306 struct tx_cmp *txcmp;
2307
2308 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2309
2310 return TX_CMP_VALID(txcmp, raw_cons);
2311}
2312
2313static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2314{
2315 struct bnxt_napi *bnapi = dev_instance;
2316 struct bnxt *bp = bnapi->bp;
2317 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2318 u32 cons = RING_CMP(cpr->cp_raw_cons);
2319 u32 int_status;
2320
2321 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2322
2323 if (!bnxt_has_work(bp, cpr)) {
2324 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2325
2326 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2327 return IRQ_NONE;
2328 }
2329
2330
2331 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2332
2333
2334 if (unlikely(atomic_read(&bp->intr_sem) != 0))
2335 return IRQ_HANDLED;
2336
2337 napi_schedule(&bnapi->napi);
2338 return IRQ_HANDLED;
2339}
2340
2341static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2342 int budget)
2343{
2344 struct bnxt_napi *bnapi = cpr->bnapi;
2345 u32 raw_cons = cpr->cp_raw_cons;
2346 u32 cons;
2347 int tx_pkts = 0;
2348 int rx_pkts = 0;
2349 u8 event = 0;
2350 struct tx_cmp *txcmp;
2351
2352 cpr->has_more_work = 0;
2353 cpr->had_work_done = 1;
2354 while (1) {
2355 int rc;
2356
2357 cons = RING_CMP(raw_cons);
2358 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2359
2360 if (!TX_CMP_VALID(txcmp, raw_cons))
2361 break;
2362
2363
2364
2365
2366 dma_rmb();
2367 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2368 tx_pkts++;
2369
2370 if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2371 rx_pkts = budget;
2372 raw_cons = NEXT_RAW_CMP(raw_cons);
2373 if (budget)
2374 cpr->has_more_work = 1;
2375 break;
2376 }
2377 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2378 if (likely(budget))
2379 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2380 else
2381 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2382 &event);
2383 if (likely(rc >= 0))
2384 rx_pkts += rc;
2385
2386
2387
2388
2389
2390 else if (rc == -ENOMEM && budget)
2391 rx_pkts++;
2392 else if (rc == -EBUSY)
2393 break;
2394 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2395 CMPL_BASE_TYPE_HWRM_DONE) ||
2396 (TX_CMP_TYPE(txcmp) ==
2397 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2398 (TX_CMP_TYPE(txcmp) ==
2399 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2400 bnxt_hwrm_handler(bp, txcmp);
2401 }
2402 raw_cons = NEXT_RAW_CMP(raw_cons);
2403
2404 if (rx_pkts && rx_pkts == budget) {
2405 cpr->has_more_work = 1;
2406 break;
2407 }
2408 }
2409
2410 if (event & BNXT_REDIRECT_EVENT)
2411 xdp_do_flush_map();
2412
2413 if (event & BNXT_TX_EVENT) {
2414 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2415 u16 prod = txr->tx_prod;
2416
2417
2418 wmb();
2419
2420 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2421 }
2422
2423 cpr->cp_raw_cons = raw_cons;
2424 bnapi->tx_pkts += tx_pkts;
2425 bnapi->events |= event;
2426 return rx_pkts;
2427}
2428
2429static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2430{
2431 if (bnapi->tx_pkts) {
2432 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2433 bnapi->tx_pkts = 0;
2434 }
2435
2436 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2437 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2438
2439 if (bnapi->events & BNXT_AGG_EVENT)
2440 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2441 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2442 }
2443 bnapi->events = 0;
2444}
2445
2446static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2447 int budget)
2448{
2449 struct bnxt_napi *bnapi = cpr->bnapi;
2450 int rx_pkts;
2451
2452 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2453
2454
2455
2456
2457
2458 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2459
2460 __bnxt_poll_work_done(bp, bnapi);
2461 return rx_pkts;
2462}
2463
2464static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2465{
2466 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2467 struct bnxt *bp = bnapi->bp;
2468 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2469 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2470 struct tx_cmp *txcmp;
2471 struct rx_cmp_ext *rxcmp1;
2472 u32 cp_cons, tmp_raw_cons;
2473 u32 raw_cons = cpr->cp_raw_cons;
2474 u32 rx_pkts = 0;
2475 u8 event = 0;
2476
2477 while (1) {
2478 int rc;
2479
2480 cp_cons = RING_CMP(raw_cons);
2481 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2482
2483 if (!TX_CMP_VALID(txcmp, raw_cons))
2484 break;
2485
2486
2487
2488
2489 dma_rmb();
2490 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2491 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2492 cp_cons = RING_CMP(tmp_raw_cons);
2493 rxcmp1 = (struct rx_cmp_ext *)
2494 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2495
2496 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2497 break;
2498
2499
2500 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2501 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2502
2503 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2504 if (likely(rc == -EIO) && budget)
2505 rx_pkts++;
2506 else if (rc == -EBUSY)
2507 break;
2508 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2509 CMPL_BASE_TYPE_HWRM_DONE)) {
2510 bnxt_hwrm_handler(bp, txcmp);
2511 } else {
2512 netdev_err(bp->dev,
2513 "Invalid completion received on special ring\n");
2514 }
2515 raw_cons = NEXT_RAW_CMP(raw_cons);
2516
2517 if (rx_pkts == budget)
2518 break;
2519 }
2520
2521 cpr->cp_raw_cons = raw_cons;
2522 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2523 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2524
2525 if (event & BNXT_AGG_EVENT)
2526 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2527
2528 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2529 napi_complete_done(napi, rx_pkts);
2530 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2531 }
2532 return rx_pkts;
2533}
2534
2535static int bnxt_poll(struct napi_struct *napi, int budget)
2536{
2537 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2538 struct bnxt *bp = bnapi->bp;
2539 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2540 int work_done = 0;
2541
2542 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2543 napi_complete(napi);
2544 return 0;
2545 }
2546 while (1) {
2547 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2548
2549 if (work_done >= budget) {
2550 if (!budget)
2551 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2552 break;
2553 }
2554
2555 if (!bnxt_has_work(bp, cpr)) {
2556 if (napi_complete_done(napi, work_done))
2557 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2558 break;
2559 }
2560 }
2561 if (bp->flags & BNXT_FLAG_DIM) {
2562 struct dim_sample dim_sample = {};
2563
2564 dim_update_sample(cpr->event_ctr,
2565 cpr->rx_packets,
2566 cpr->rx_bytes,
2567 &dim_sample);
2568 net_dim(&cpr->dim, dim_sample);
2569 }
2570 return work_done;
2571}
2572
2573static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2574{
2575 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2576 int i, work_done = 0;
2577
2578 for (i = 0; i < 2; i++) {
2579 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2580
2581 if (cpr2) {
2582 work_done += __bnxt_poll_work(bp, cpr2,
2583 budget - work_done);
2584 cpr->has_more_work |= cpr2->has_more_work;
2585 }
2586 }
2587 return work_done;
2588}
2589
2590static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2591 u64 dbr_type)
2592{
2593 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2594 int i;
2595
2596 for (i = 0; i < 2; i++) {
2597 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2598 struct bnxt_db_info *db;
2599
2600 if (cpr2 && cpr2->had_work_done) {
2601 db = &cpr2->cp_db;
2602 writeq(db->db_key64 | dbr_type |
2603 RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2604 cpr2->had_work_done = 0;
2605 }
2606 }
2607 __bnxt_poll_work_done(bp, bnapi);
2608}
2609
2610static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2611{
2612 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2613 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2614 u32 raw_cons = cpr->cp_raw_cons;
2615 struct bnxt *bp = bnapi->bp;
2616 struct nqe_cn *nqcmp;
2617 int work_done = 0;
2618 u32 cons;
2619
2620 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2621 napi_complete(napi);
2622 return 0;
2623 }
2624 if (cpr->has_more_work) {
2625 cpr->has_more_work = 0;
2626 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2627 }
2628 while (1) {
2629 cons = RING_CMP(raw_cons);
2630 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2631
2632 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2633 if (cpr->has_more_work)
2634 break;
2635
2636 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2637 cpr->cp_raw_cons = raw_cons;
2638 if (napi_complete_done(napi, work_done))
2639 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2640 cpr->cp_raw_cons);
2641 return work_done;
2642 }
2643
2644
2645
2646
2647 dma_rmb();
2648
2649 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2650 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2651 struct bnxt_cp_ring_info *cpr2;
2652
2653 cpr2 = cpr->cp_ring_arr[idx];
2654 work_done += __bnxt_poll_work(bp, cpr2,
2655 budget - work_done);
2656 cpr->has_more_work |= cpr2->has_more_work;
2657 } else {
2658 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2659 }
2660 raw_cons = NEXT_RAW_CMP(raw_cons);
2661 }
2662 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2663 if (raw_cons != cpr->cp_raw_cons) {
2664 cpr->cp_raw_cons = raw_cons;
2665 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2666 }
2667 return work_done;
2668}
2669
2670static void bnxt_free_tx_skbs(struct bnxt *bp)
2671{
2672 int i, max_idx;
2673 struct pci_dev *pdev = bp->pdev;
2674
2675 if (!bp->tx_ring)
2676 return;
2677
2678 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2679 for (i = 0; i < bp->tx_nr_rings; i++) {
2680 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2681 int j;
2682
2683 for (j = 0; j < max_idx;) {
2684 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2685 struct sk_buff *skb;
2686 int k, last;
2687
2688 if (i < bp->tx_nr_rings_xdp &&
2689 tx_buf->action == XDP_REDIRECT) {
2690 dma_unmap_single(&pdev->dev,
2691 dma_unmap_addr(tx_buf, mapping),
2692 dma_unmap_len(tx_buf, len),
2693 PCI_DMA_TODEVICE);
2694 xdp_return_frame(tx_buf->xdpf);
2695 tx_buf->action = 0;
2696 tx_buf->xdpf = NULL;
2697 j++;
2698 continue;
2699 }
2700
2701 skb = tx_buf->skb;
2702 if (!skb) {
2703 j++;
2704 continue;
2705 }
2706
2707 tx_buf->skb = NULL;
2708
2709 if (tx_buf->is_push) {
2710 dev_kfree_skb(skb);
2711 j += 2;
2712 continue;
2713 }
2714
2715 dma_unmap_single(&pdev->dev,
2716 dma_unmap_addr(tx_buf, mapping),
2717 skb_headlen(skb),
2718 PCI_DMA_TODEVICE);
2719
2720 last = tx_buf->nr_frags;
2721 j += 2;
2722 for (k = 0; k < last; k++, j++) {
2723 int ring_idx = j & bp->tx_ring_mask;
2724 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2725
2726 tx_buf = &txr->tx_buf_ring[ring_idx];
2727 dma_unmap_page(
2728 &pdev->dev,
2729 dma_unmap_addr(tx_buf, mapping),
2730 skb_frag_size(frag), PCI_DMA_TODEVICE);
2731 }
2732 dev_kfree_skb(skb);
2733 }
2734 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2735 }
2736}
2737
2738static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2739{
2740 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2741 struct pci_dev *pdev = bp->pdev;
2742 struct bnxt_tpa_idx_map *map;
2743 int i, max_idx, max_agg_idx;
2744
2745 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2746 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2747 if (!rxr->rx_tpa)
2748 goto skip_rx_tpa_free;
2749
2750 for (i = 0; i < bp->max_tpa; i++) {
2751 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2752 u8 *data = tpa_info->data;
2753
2754 if (!data)
2755 continue;
2756
2757 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2758 bp->rx_buf_use_size, bp->rx_dir,
2759 DMA_ATTR_WEAK_ORDERING);
2760
2761 tpa_info->data = NULL;
2762
2763 kfree(data);
2764 }
2765
2766skip_rx_tpa_free:
2767 for (i = 0; i < max_idx; i++) {
2768 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2769 dma_addr_t mapping = rx_buf->mapping;
2770 void *data = rx_buf->data;
2771
2772 if (!data)
2773 continue;
2774
2775 rx_buf->data = NULL;
2776 if (BNXT_RX_PAGE_MODE(bp)) {
2777 mapping -= bp->rx_dma_offset;
2778 dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2779 bp->rx_dir,
2780 DMA_ATTR_WEAK_ORDERING);
2781 page_pool_recycle_direct(rxr->page_pool, data);
2782 } else {
2783 dma_unmap_single_attrs(&pdev->dev, mapping,
2784 bp->rx_buf_use_size, bp->rx_dir,
2785 DMA_ATTR_WEAK_ORDERING);
2786 kfree(data);
2787 }
2788 }
2789 for (i = 0; i < max_agg_idx; i++) {
2790 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2791 struct page *page = rx_agg_buf->page;
2792
2793 if (!page)
2794 continue;
2795
2796 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2797 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2798 DMA_ATTR_WEAK_ORDERING);
2799
2800 rx_agg_buf->page = NULL;
2801 __clear_bit(i, rxr->rx_agg_bmap);
2802
2803 __free_page(page);
2804 }
2805 if (rxr->rx_page) {
2806 __free_page(rxr->rx_page);
2807 rxr->rx_page = NULL;
2808 }
2809 map = rxr->rx_tpa_idx_map;
2810 if (map)
2811 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2812}
2813
2814static void bnxt_free_rx_skbs(struct bnxt *bp)
2815{
2816 int i;
2817
2818 if (!bp->rx_ring)
2819 return;
2820
2821 for (i = 0; i < bp->rx_nr_rings; i++)
2822 bnxt_free_one_rx_ring_skbs(bp, i);
2823}
2824
2825static void bnxt_free_skbs(struct bnxt *bp)
2826{
2827 bnxt_free_tx_skbs(bp);
2828 bnxt_free_rx_skbs(bp);
2829}
2830
2831static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
2832{
2833 u8 init_val = mem_init->init_val;
2834 u16 offset = mem_init->offset;
2835 u8 *p2 = p;
2836 int i;
2837
2838 if (!init_val)
2839 return;
2840 if (offset == BNXT_MEM_INVALID_OFFSET) {
2841 memset(p, init_val, len);
2842 return;
2843 }
2844 for (i = 0; i < len; i += mem_init->size)
2845 *(p2 + i + offset) = init_val;
2846}
2847
2848static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2849{
2850 struct pci_dev *pdev = bp->pdev;
2851 int i;
2852
2853 for (i = 0; i < rmem->nr_pages; i++) {
2854 if (!rmem->pg_arr[i])
2855 continue;
2856
2857 dma_free_coherent(&pdev->dev, rmem->page_size,
2858 rmem->pg_arr[i], rmem->dma_arr[i]);
2859
2860 rmem->pg_arr[i] = NULL;
2861 }
2862 if (rmem->pg_tbl) {
2863 size_t pg_tbl_size = rmem->nr_pages * 8;
2864
2865 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2866 pg_tbl_size = rmem->page_size;
2867 dma_free_coherent(&pdev->dev, pg_tbl_size,
2868 rmem->pg_tbl, rmem->pg_tbl_map);
2869 rmem->pg_tbl = NULL;
2870 }
2871 if (rmem->vmem_size && *rmem->vmem) {
2872 vfree(*rmem->vmem);
2873 *rmem->vmem = NULL;
2874 }
2875}
2876
2877static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2878{
2879 struct pci_dev *pdev = bp->pdev;
2880 u64 valid_bit = 0;
2881 int i;
2882
2883 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2884 valid_bit = PTU_PTE_VALID;
2885 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2886 size_t pg_tbl_size = rmem->nr_pages * 8;
2887
2888 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2889 pg_tbl_size = rmem->page_size;
2890 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2891 &rmem->pg_tbl_map,
2892 GFP_KERNEL);
2893 if (!rmem->pg_tbl)
2894 return -ENOMEM;
2895 }
2896
2897 for (i = 0; i < rmem->nr_pages; i++) {
2898 u64 extra_bits = valid_bit;
2899
2900 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2901 rmem->page_size,
2902 &rmem->dma_arr[i],
2903 GFP_KERNEL);
2904 if (!rmem->pg_arr[i])
2905 return -ENOMEM;
2906
2907 if (rmem->mem_init)
2908 bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
2909 rmem->page_size);
2910 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2911 if (i == rmem->nr_pages - 2 &&
2912 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2913 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2914 else if (i == rmem->nr_pages - 1 &&
2915 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2916 extra_bits |= PTU_PTE_LAST;
2917 rmem->pg_tbl[i] =
2918 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2919 }
2920 }
2921
2922 if (rmem->vmem_size) {
2923 *rmem->vmem = vzalloc(rmem->vmem_size);
2924 if (!(*rmem->vmem))
2925 return -ENOMEM;
2926 }
2927 return 0;
2928}
2929
2930static void bnxt_free_tpa_info(struct bnxt *bp)
2931{
2932 int i;
2933
2934 for (i = 0; i < bp->rx_nr_rings; i++) {
2935 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2936
2937 kfree(rxr->rx_tpa_idx_map);
2938 rxr->rx_tpa_idx_map = NULL;
2939 if (rxr->rx_tpa) {
2940 kfree(rxr->rx_tpa[0].agg_arr);
2941 rxr->rx_tpa[0].agg_arr = NULL;
2942 }
2943 kfree(rxr->rx_tpa);
2944 rxr->rx_tpa = NULL;
2945 }
2946}
2947
2948static int bnxt_alloc_tpa_info(struct bnxt *bp)
2949{
2950 int i, j, total_aggs = 0;
2951
2952 bp->max_tpa = MAX_TPA;
2953 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2954 if (!bp->max_tpa_v2)
2955 return 0;
2956 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2957 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2958 }
2959
2960 for (i = 0; i < bp->rx_nr_rings; i++) {
2961 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2962 struct rx_agg_cmp *agg;
2963
2964 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2965 GFP_KERNEL);
2966 if (!rxr->rx_tpa)
2967 return -ENOMEM;
2968
2969 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2970 continue;
2971 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2972 rxr->rx_tpa[0].agg_arr = agg;
2973 if (!agg)
2974 return -ENOMEM;
2975 for (j = 1; j < bp->max_tpa; j++)
2976 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2977 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2978 GFP_KERNEL);
2979 if (!rxr->rx_tpa_idx_map)
2980 return -ENOMEM;
2981 }
2982 return 0;
2983}
2984
2985static void bnxt_free_rx_rings(struct bnxt *bp)
2986{
2987 int i;
2988
2989 if (!bp->rx_ring)
2990 return;
2991
2992 bnxt_free_tpa_info(bp);
2993 for (i = 0; i < bp->rx_nr_rings; i++) {
2994 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2995 struct bnxt_ring_struct *ring;
2996
2997 if (rxr->xdp_prog)
2998 bpf_prog_put(rxr->xdp_prog);
2999
3000 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3001 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3002
3003 page_pool_destroy(rxr->page_pool);
3004 rxr->page_pool = NULL;
3005
3006 kfree(rxr->rx_agg_bmap);
3007 rxr->rx_agg_bmap = NULL;
3008
3009 ring = &rxr->rx_ring_struct;
3010 bnxt_free_ring(bp, &ring->ring_mem);
3011
3012 ring = &rxr->rx_agg_ring_struct;
3013 bnxt_free_ring(bp, &ring->ring_mem);
3014 }
3015}
3016
3017static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3018 struct bnxt_rx_ring_info *rxr)
3019{
3020 struct page_pool_params pp = { 0 };
3021
3022 pp.pool_size = bp->rx_ring_size;
3023 pp.nid = dev_to_node(&bp->pdev->dev);
3024 pp.dev = &bp->pdev->dev;
3025 pp.dma_dir = DMA_BIDIRECTIONAL;
3026
3027 rxr->page_pool = page_pool_create(&pp);
3028 if (IS_ERR(rxr->page_pool)) {
3029 int err = PTR_ERR(rxr->page_pool);
3030
3031 rxr->page_pool = NULL;
3032 return err;
3033 }
3034 return 0;
3035}
3036
3037static int bnxt_alloc_rx_rings(struct bnxt *bp)
3038{
3039 int i, rc = 0, agg_rings = 0;
3040
3041 if (!bp->rx_ring)
3042 return -ENOMEM;
3043
3044 if (bp->flags & BNXT_FLAG_AGG_RINGS)
3045 agg_rings = 1;
3046
3047 for (i = 0; i < bp->rx_nr_rings; i++) {
3048 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3049 struct bnxt_ring_struct *ring;
3050
3051 ring = &rxr->rx_ring_struct;
3052
3053 rc = bnxt_alloc_rx_page_pool(bp, rxr);
3054 if (rc)
3055 return rc;
3056
3057 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3058 if (rc < 0)
3059 return rc;
3060
3061 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3062 MEM_TYPE_PAGE_POOL,
3063 rxr->page_pool);
3064 if (rc) {
3065 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3066 return rc;
3067 }
3068
3069 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3070 if (rc)
3071 return rc;
3072
3073 ring->grp_idx = i;
3074 if (agg_rings) {
3075 u16 mem_size;
3076
3077 ring = &rxr->rx_agg_ring_struct;
3078 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3079 if (rc)
3080 return rc;
3081
3082 ring->grp_idx = i;
3083 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3084 mem_size = rxr->rx_agg_bmap_size / 8;
3085 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3086 if (!rxr->rx_agg_bmap)
3087 return -ENOMEM;
3088 }
3089 }
3090 if (bp->flags & BNXT_FLAG_TPA)
3091 rc = bnxt_alloc_tpa_info(bp);
3092 return rc;
3093}
3094
3095static void bnxt_free_tx_rings(struct bnxt *bp)
3096{
3097 int i;
3098 struct pci_dev *pdev = bp->pdev;
3099
3100 if (!bp->tx_ring)
3101 return;
3102
3103 for (i = 0; i < bp->tx_nr_rings; i++) {
3104 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3105 struct bnxt_ring_struct *ring;
3106
3107 if (txr->tx_push) {
3108 dma_free_coherent(&pdev->dev, bp->tx_push_size,
3109 txr->tx_push, txr->tx_push_mapping);
3110 txr->tx_push = NULL;
3111 }
3112
3113 ring = &txr->tx_ring_struct;
3114
3115 bnxt_free_ring(bp, &ring->ring_mem);
3116 }
3117}
3118
3119static int bnxt_alloc_tx_rings(struct bnxt *bp)
3120{
3121 int i, j, rc;
3122 struct pci_dev *pdev = bp->pdev;
3123
3124 bp->tx_push_size = 0;
3125 if (bp->tx_push_thresh) {
3126 int push_size;
3127
3128 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3129 bp->tx_push_thresh);
3130
3131 if (push_size > 256) {
3132 push_size = 0;
3133 bp->tx_push_thresh = 0;
3134 }
3135
3136 bp->tx_push_size = push_size;
3137 }
3138
3139 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3140 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3141 struct bnxt_ring_struct *ring;
3142 u8 qidx;
3143
3144 ring = &txr->tx_ring_struct;
3145
3146 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3147 if (rc)
3148 return rc;
3149
3150 ring->grp_idx = txr->bnapi->index;
3151 if (bp->tx_push_size) {
3152 dma_addr_t mapping;
3153
3154
3155
3156
3157 txr->tx_push = dma_alloc_coherent(&pdev->dev,
3158 bp->tx_push_size,
3159 &txr->tx_push_mapping,
3160 GFP_KERNEL);
3161
3162 if (!txr->tx_push)
3163 return -ENOMEM;
3164
3165 mapping = txr->tx_push_mapping +
3166 sizeof(struct tx_push_bd);
3167 txr->data_mapping = cpu_to_le64(mapping);
3168 }
3169 qidx = bp->tc_to_qidx[j];
3170 ring->queue_id = bp->q_info[qidx].queue_id;
3171 if (i < bp->tx_nr_rings_xdp)
3172 continue;
3173 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3174 j++;
3175 }
3176 return 0;
3177}
3178
3179static void bnxt_free_cp_rings(struct bnxt *bp)
3180{
3181 int i;
3182
3183 if (!bp->bnapi)
3184 return;
3185
3186 for (i = 0; i < bp->cp_nr_rings; i++) {
3187 struct bnxt_napi *bnapi = bp->bnapi[i];
3188 struct bnxt_cp_ring_info *cpr;
3189 struct bnxt_ring_struct *ring;
3190 int j;
3191
3192 if (!bnapi)
3193 continue;
3194
3195 cpr = &bnapi->cp_ring;
3196 ring = &cpr->cp_ring_struct;
3197
3198 bnxt_free_ring(bp, &ring->ring_mem);
3199
3200 for (j = 0; j < 2; j++) {
3201 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3202
3203 if (cpr2) {
3204 ring = &cpr2->cp_ring_struct;
3205 bnxt_free_ring(bp, &ring->ring_mem);
3206 kfree(cpr2);
3207 cpr->cp_ring_arr[j] = NULL;
3208 }
3209 }
3210 }
3211}
3212
3213static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3214{
3215 struct bnxt_ring_mem_info *rmem;
3216 struct bnxt_ring_struct *ring;
3217 struct bnxt_cp_ring_info *cpr;
3218 int rc;
3219
3220 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3221 if (!cpr)
3222 return NULL;
3223
3224 ring = &cpr->cp_ring_struct;
3225 rmem = &ring->ring_mem;
3226 rmem->nr_pages = bp->cp_nr_pages;
3227 rmem->page_size = HW_CMPD_RING_SIZE;
3228 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3229 rmem->dma_arr = cpr->cp_desc_mapping;
3230 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3231 rc = bnxt_alloc_ring(bp, rmem);
3232 if (rc) {
3233 bnxt_free_ring(bp, rmem);
3234 kfree(cpr);
3235 cpr = NULL;
3236 }
3237 return cpr;
3238}
3239
3240static int bnxt_alloc_cp_rings(struct bnxt *bp)
3241{
3242 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3243 int i, rc, ulp_base_vec, ulp_msix;
3244
3245 ulp_msix = bnxt_get_ulp_msix_num(bp);
3246 ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3247 for (i = 0; i < bp->cp_nr_rings; i++) {
3248 struct bnxt_napi *bnapi = bp->bnapi[i];
3249 struct bnxt_cp_ring_info *cpr;
3250 struct bnxt_ring_struct *ring;
3251
3252 if (!bnapi)
3253 continue;
3254
3255 cpr = &bnapi->cp_ring;
3256 cpr->bnapi = bnapi;
3257 ring = &cpr->cp_ring_struct;
3258
3259 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3260 if (rc)
3261 return rc;
3262
3263 if (ulp_msix && i >= ulp_base_vec)
3264 ring->map_idx = i + ulp_msix;
3265 else
3266 ring->map_idx = i;
3267
3268 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3269 continue;
3270
3271 if (i < bp->rx_nr_rings) {
3272 struct bnxt_cp_ring_info *cpr2 =
3273 bnxt_alloc_cp_sub_ring(bp);
3274
3275 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3276 if (!cpr2)
3277 return -ENOMEM;
3278 cpr2->bnapi = bnapi;
3279 }
3280 if ((sh && i < bp->tx_nr_rings) ||
3281 (!sh && i >= bp->rx_nr_rings)) {
3282 struct bnxt_cp_ring_info *cpr2 =
3283 bnxt_alloc_cp_sub_ring(bp);
3284
3285 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3286 if (!cpr2)
3287 return -ENOMEM;
3288 cpr2->bnapi = bnapi;
3289 }
3290 }
3291 return 0;
3292}
3293
3294static void bnxt_init_ring_struct(struct bnxt *bp)
3295{
3296 int i;
3297
3298 for (i = 0; i < bp->cp_nr_rings; i++) {
3299 struct bnxt_napi *bnapi = bp->bnapi[i];
3300 struct bnxt_ring_mem_info *rmem;
3301 struct bnxt_cp_ring_info *cpr;
3302 struct bnxt_rx_ring_info *rxr;
3303 struct bnxt_tx_ring_info *txr;
3304 struct bnxt_ring_struct *ring;
3305
3306 if (!bnapi)
3307 continue;
3308
3309 cpr = &bnapi->cp_ring;
3310 ring = &cpr->cp_ring_struct;
3311 rmem = &ring->ring_mem;
3312 rmem->nr_pages = bp->cp_nr_pages;
3313 rmem->page_size = HW_CMPD_RING_SIZE;
3314 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3315 rmem->dma_arr = cpr->cp_desc_mapping;
3316 rmem->vmem_size = 0;
3317
3318 rxr = bnapi->rx_ring;
3319 if (!rxr)
3320 goto skip_rx;
3321
3322 ring = &rxr->rx_ring_struct;
3323 rmem = &ring->ring_mem;
3324 rmem->nr_pages = bp->rx_nr_pages;
3325 rmem->page_size = HW_RXBD_RING_SIZE;
3326 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3327 rmem->dma_arr = rxr->rx_desc_mapping;
3328 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3329 rmem->vmem = (void **)&rxr->rx_buf_ring;
3330
3331 ring = &rxr->rx_agg_ring_struct;
3332 rmem = &ring->ring_mem;
3333 rmem->nr_pages = bp->rx_agg_nr_pages;
3334 rmem->page_size = HW_RXBD_RING_SIZE;
3335 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3336 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3337 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3338 rmem->vmem = (void **)&rxr->rx_agg_ring;
3339
3340skip_rx:
3341 txr = bnapi->tx_ring;
3342 if (!txr)
3343 continue;
3344
3345 ring = &txr->tx_ring_struct;
3346 rmem = &ring->ring_mem;
3347 rmem->nr_pages = bp->tx_nr_pages;
3348 rmem->page_size = HW_RXBD_RING_SIZE;
3349 rmem->pg_arr = (void **)txr->tx_desc_ring;
3350 rmem->dma_arr = txr->tx_desc_mapping;
3351 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3352 rmem->vmem = (void **)&txr->tx_buf_ring;
3353 }
3354}
3355
3356static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3357{
3358 int i;
3359 u32 prod;
3360 struct rx_bd **rx_buf_ring;
3361
3362 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3363 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3364 int j;
3365 struct rx_bd *rxbd;
3366
3367 rxbd = rx_buf_ring[i];
3368 if (!rxbd)
3369 continue;
3370
3371 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3372 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3373 rxbd->rx_bd_opaque = prod;
3374 }
3375 }
3376}
3377
3378static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3379{
3380 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3381 struct net_device *dev = bp->dev;
3382 u32 prod;
3383 int i;
3384
3385 prod = rxr->rx_prod;
3386 for (i = 0; i < bp->rx_ring_size; i++) {
3387 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3388 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3389 ring_nr, i, bp->rx_ring_size);
3390 break;
3391 }
3392 prod = NEXT_RX(prod);
3393 }
3394 rxr->rx_prod = prod;
3395
3396 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3397 return 0;
3398
3399 prod = rxr->rx_agg_prod;
3400 for (i = 0; i < bp->rx_agg_ring_size; i++) {
3401 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3402 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3403 ring_nr, i, bp->rx_ring_size);
3404 break;
3405 }
3406 prod = NEXT_RX_AGG(prod);
3407 }
3408 rxr->rx_agg_prod = prod;
3409
3410 if (rxr->rx_tpa) {
3411 dma_addr_t mapping;
3412 u8 *data;
3413
3414 for (i = 0; i < bp->max_tpa; i++) {
3415 data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3416 if (!data)
3417 return -ENOMEM;
3418
3419 rxr->rx_tpa[i].data = data;
3420 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3421 rxr->rx_tpa[i].mapping = mapping;
3422 }
3423 }
3424 return 0;
3425}
3426
3427static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3428{
3429 struct bnxt_rx_ring_info *rxr;
3430 struct bnxt_ring_struct *ring;
3431 u32 type;
3432
3433 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3434 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3435
3436 if (NET_IP_ALIGN == 2)
3437 type |= RX_BD_FLAGS_SOP;
3438
3439 rxr = &bp->rx_ring[ring_nr];
3440 ring = &rxr->rx_ring_struct;
3441 bnxt_init_rxbd_pages(ring, type);
3442
3443 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3444 bpf_prog_add(bp->xdp_prog, 1);
3445 rxr->xdp_prog = bp->xdp_prog;
3446 }
3447 ring->fw_ring_id = INVALID_HW_RING_ID;
3448
3449 ring = &rxr->rx_agg_ring_struct;
3450 ring->fw_ring_id = INVALID_HW_RING_ID;
3451
3452 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3453 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3454 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3455
3456 bnxt_init_rxbd_pages(ring, type);
3457 }
3458
3459 return bnxt_alloc_one_rx_ring(bp, ring_nr);
3460}
3461
3462static void bnxt_init_cp_rings(struct bnxt *bp)
3463{
3464 int i, j;
3465
3466 for (i = 0; i < bp->cp_nr_rings; i++) {
3467 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3468 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3469
3470 ring->fw_ring_id = INVALID_HW_RING_ID;
3471 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3472 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3473 for (j = 0; j < 2; j++) {
3474 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3475
3476 if (!cpr2)
3477 continue;
3478
3479 ring = &cpr2->cp_ring_struct;
3480 ring->fw_ring_id = INVALID_HW_RING_ID;
3481 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3482 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3483 }
3484 }
3485}
3486
3487static int bnxt_init_rx_rings(struct bnxt *bp)
3488{
3489 int i, rc = 0;
3490
3491 if (BNXT_RX_PAGE_MODE(bp)) {
3492 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3493 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3494 } else {
3495 bp->rx_offset = BNXT_RX_OFFSET;
3496 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3497 }
3498
3499 for (i = 0; i < bp->rx_nr_rings; i++) {
3500 rc = bnxt_init_one_rx_ring(bp, i);
3501 if (rc)
3502 break;
3503 }
3504
3505 return rc;
3506}
3507
3508static int bnxt_init_tx_rings(struct bnxt *bp)
3509{
3510 u16 i;
3511
3512 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3513 MAX_SKB_FRAGS + 1);
3514
3515 for (i = 0; i < bp->tx_nr_rings; i++) {
3516 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3517 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3518
3519 ring->fw_ring_id = INVALID_HW_RING_ID;
3520 }
3521
3522 return 0;
3523}
3524
3525static void bnxt_free_ring_grps(struct bnxt *bp)
3526{
3527 kfree(bp->grp_info);
3528 bp->grp_info = NULL;
3529}
3530
3531static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3532{
3533 int i;
3534
3535 if (irq_re_init) {
3536 bp->grp_info = kcalloc(bp->cp_nr_rings,
3537 sizeof(struct bnxt_ring_grp_info),
3538 GFP_KERNEL);
3539 if (!bp->grp_info)
3540 return -ENOMEM;
3541 }
3542 for (i = 0; i < bp->cp_nr_rings; i++) {
3543 if (irq_re_init)
3544 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3545 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3546 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3547 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3548 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3549 }
3550 return 0;
3551}
3552
3553static void bnxt_free_vnics(struct bnxt *bp)
3554{
3555 kfree(bp->vnic_info);
3556 bp->vnic_info = NULL;
3557 bp->nr_vnics = 0;
3558}
3559
3560static int bnxt_alloc_vnics(struct bnxt *bp)
3561{
3562 int num_vnics = 1;
3563
3564#ifdef CONFIG_RFS_ACCEL
3565 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3566 num_vnics += bp->rx_nr_rings;
3567#endif
3568
3569 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3570 num_vnics++;
3571
3572 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3573 GFP_KERNEL);
3574 if (!bp->vnic_info)
3575 return -ENOMEM;
3576
3577 bp->nr_vnics = num_vnics;
3578 return 0;
3579}
3580
3581static void bnxt_init_vnics(struct bnxt *bp)
3582{
3583 int i;
3584
3585 for (i = 0; i < bp->nr_vnics; i++) {
3586 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3587 int j;
3588
3589 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3590 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3591 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3592
3593 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3594
3595 if (bp->vnic_info[i].rss_hash_key) {
3596 if (i == 0)
3597 prandom_bytes(vnic->rss_hash_key,
3598 HW_HASH_KEY_SIZE);
3599 else
3600 memcpy(vnic->rss_hash_key,
3601 bp->vnic_info[0].rss_hash_key,
3602 HW_HASH_KEY_SIZE);
3603 }
3604 }
3605}
3606
3607static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3608{
3609 int pages;
3610
3611 pages = ring_size / desc_per_pg;
3612
3613 if (!pages)
3614 return 1;
3615
3616 pages++;
3617
3618 while (pages & (pages - 1))
3619 pages++;
3620
3621 return pages;
3622}
3623
3624void bnxt_set_tpa_flags(struct bnxt *bp)
3625{
3626 bp->flags &= ~BNXT_FLAG_TPA;
3627 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3628 return;
3629 if (bp->dev->features & NETIF_F_LRO)
3630 bp->flags |= BNXT_FLAG_LRO;
3631 else if (bp->dev->features & NETIF_F_GRO_HW)
3632 bp->flags |= BNXT_FLAG_GRO;
3633}
3634
3635
3636
3637
3638void bnxt_set_ring_params(struct bnxt *bp)
3639{
3640 u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3641 u32 agg_factor = 0, agg_ring_size = 0;
3642
3643
3644 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3645
3646 rx_space = rx_size + NET_SKB_PAD +
3647 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3648
3649 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3650 ring_size = bp->rx_ring_size;
3651 bp->rx_agg_ring_size = 0;
3652 bp->rx_agg_nr_pages = 0;
3653
3654 if (bp->flags & BNXT_FLAG_TPA)
3655 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3656
3657 bp->flags &= ~BNXT_FLAG_JUMBO;
3658 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3659 u32 jumbo_factor;
3660
3661 bp->flags |= BNXT_FLAG_JUMBO;
3662 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3663 if (jumbo_factor > agg_factor)
3664 agg_factor = jumbo_factor;
3665 }
3666 agg_ring_size = ring_size * agg_factor;
3667
3668 if (agg_ring_size) {
3669 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3670 RX_DESC_CNT);
3671 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3672 u32 tmp = agg_ring_size;
3673
3674 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3675 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3676 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3677 tmp, agg_ring_size);
3678 }
3679 bp->rx_agg_ring_size = agg_ring_size;
3680 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3681 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3682 rx_space = rx_size + NET_SKB_PAD +
3683 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3684 }
3685
3686 bp->rx_buf_use_size = rx_size;
3687 bp->rx_buf_size = rx_space;
3688
3689 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3690 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3691
3692 ring_size = bp->tx_ring_size;
3693 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3694 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3695
3696 max_rx_cmpl = bp->rx_ring_size;
3697
3698
3699
3700
3701 if (bp->flags & BNXT_FLAG_TPA)
3702 max_rx_cmpl += bp->max_tpa;
3703
3704 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3705 bp->cp_ring_size = ring_size;
3706
3707 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3708 if (bp->cp_nr_pages > MAX_CP_PAGES) {
3709 bp->cp_nr_pages = MAX_CP_PAGES;
3710 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3711 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3712 ring_size, bp->cp_ring_size);
3713 }
3714 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3715 bp->cp_ring_mask = bp->cp_bit - 1;
3716}
3717
3718
3719
3720
3721int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3722{
3723 if (page_mode) {
3724 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3725 return -EOPNOTSUPP;
3726 bp->dev->max_mtu =
3727 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3728 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3729 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3730 bp->rx_dir = DMA_BIDIRECTIONAL;
3731 bp->rx_skb_func = bnxt_rx_page_skb;
3732
3733 netdev_update_features(bp->dev);
3734 } else {
3735 bp->dev->max_mtu = bp->max_mtu;
3736 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3737 bp->rx_dir = DMA_FROM_DEVICE;
3738 bp->rx_skb_func = bnxt_rx_skb;
3739 }
3740 return 0;
3741}
3742
3743static void bnxt_free_vnic_attributes(struct bnxt *bp)
3744{
3745 int i;
3746 struct bnxt_vnic_info *vnic;
3747 struct pci_dev *pdev = bp->pdev;
3748
3749 if (!bp->vnic_info)
3750 return;
3751
3752 for (i = 0; i < bp->nr_vnics; i++) {
3753 vnic = &bp->vnic_info[i];
3754
3755 kfree(vnic->fw_grp_ids);
3756 vnic->fw_grp_ids = NULL;
3757
3758 kfree(vnic->uc_list);
3759 vnic->uc_list = NULL;
3760
3761 if (vnic->mc_list) {
3762 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3763 vnic->mc_list, vnic->mc_list_mapping);
3764 vnic->mc_list = NULL;
3765 }
3766
3767 if (vnic->rss_table) {
3768 dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3769 vnic->rss_table,
3770 vnic->rss_table_dma_addr);
3771 vnic->rss_table = NULL;
3772 }
3773
3774 vnic->rss_hash_key = NULL;
3775 vnic->flags = 0;
3776 }
3777}
3778
3779static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3780{
3781 int i, rc = 0, size;
3782 struct bnxt_vnic_info *vnic;
3783 struct pci_dev *pdev = bp->pdev;
3784 int max_rings;
3785
3786 for (i = 0; i < bp->nr_vnics; 
