linux/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2// Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4#include <linux/dma-mapping.h>
   5#include <linux/etherdevice.h>
   6#include <linux/interrupt.h>
   7#ifdef CONFIG_RFS_ACCEL
   8#include <linux/cpu_rmap.h>
   9#endif
  10#include <linux/if_vlan.h>
  11#include <linux/irq.h>
  12#include <linux/ip.h>
  13#include <linux/ipv6.h>
  14#include <linux/module.h>
  15#include <linux/pci.h>
  16#include <linux/aer.h>
  17#include <linux/skbuff.h>
  18#include <linux/sctp.h>
  19#include <net/gre.h>
  20#include <net/ip6_checksum.h>
  21#include <net/pkt_cls.h>
  22#include <net/tcp.h>
  23#include <net/vxlan.h>
  24#include <net/geneve.h>
  25
  26#include "hnae3.h"
  27#include "hns3_enet.h"
  28/* All hns3 tracepoints are defined by the include below, which
  29 * must be included exactly once across the whole kernel with
  30 * CREATE_TRACE_POINTS defined
  31 */
  32#define CREATE_TRACE_POINTS
  33#include "hns3_trace.h"
  34
  35#define hns3_set_field(origin, shift, val)      ((origin) |= (val) << (shift))
  36#define hns3_tx_bd_count(S)     DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)
  37
  38#define hns3_rl_err(fmt, ...)                                           \
  39        do {                                                            \
  40                if (net_ratelimit())                                    \
  41                        netdev_err(fmt, ##__VA_ARGS__);                 \
  42        } while (0)
  43
  44static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);
  45
  46static const char hns3_driver_name[] = "hns3";
  47static const char hns3_driver_string[] =
  48                        "Hisilicon Ethernet Network Driver for Hip08 Family";
  49static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  50static struct hnae3_client client;
  51
  52static int debug = -1;
  53module_param(debug, int, 0);
  54MODULE_PARM_DESC(debug, " Network interface message level setting");
  55
  56static unsigned int tx_spare_buf_size;
  57module_param(tx_spare_buf_size, uint, 0400);
  58MODULE_PARM_DESC(tx_spare_buf_size, "Size used to allocate tx spare buffer");
  59
  60static unsigned int tx_sgl = 1;
  61module_param(tx_sgl, uint, 0600);
  62MODULE_PARM_DESC(tx_sgl, "Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping");
  63
  64#define HNS3_SGL_SIZE(nfrag)    (sizeof(struct scatterlist) * (nfrag) + \
  65                                 sizeof(struct sg_table))
  66#define HNS3_MAX_SGL_SIZE       ALIGN(HNS3_SGL_SIZE(HNS3_MAX_TSO_BD_NUM),\
  67                                      dma_get_cache_alignment())
  68
  69#define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  70                           NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
  71
  72#define HNS3_INNER_VLAN_TAG     1
  73#define HNS3_OUTER_VLAN_TAG     2
  74
  75#define HNS3_MIN_TX_LEN         33U
  76
  77/* hns3_pci_tbl - PCI Device ID Table
  78 *
  79 * Last entry must be all 0s
  80 *
  81 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  82 *   Class, Class Mask, private data (not used) }
  83 */
  84static const struct pci_device_id hns3_pci_tbl[] = {
  85        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  86        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  87        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  88         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  89        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  90         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  91        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  92         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  93        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  94         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  95        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  96         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  97        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
  98         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  99        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},
 100        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
 101         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
 102        /* required last entry */
 103        {0, }
 104};
 105MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
 106
 107#define HNS3_RX_PTYPE_ENTRY(ptype, l, s, t) \
 108        {       ptype, \
 109                l, \
 110                CHECKSUM_##s, \
 111                HNS3_L3_TYPE_##t, \
 112                1 }
 113
 114#define HNS3_RX_PTYPE_UNUSED_ENTRY(ptype) \
 115                { ptype, 0, CHECKSUM_NONE, HNS3_L3_TYPE_PARSE_FAIL, 0 }
 116
 117static const struct hns3_rx_ptype hns3_rx_ptype_tbl[] = {
 118        HNS3_RX_PTYPE_UNUSED_ENTRY(0),
 119        HNS3_RX_PTYPE_ENTRY(1, 0, COMPLETE, ARP),
 120        HNS3_RX_PTYPE_ENTRY(2, 0, COMPLETE, RARP),
 121        HNS3_RX_PTYPE_ENTRY(3, 0, COMPLETE, LLDP),
 122        HNS3_RX_PTYPE_ENTRY(4, 0, COMPLETE, PARSE_FAIL),
 123        HNS3_RX_PTYPE_ENTRY(5, 0, COMPLETE, PARSE_FAIL),
 124        HNS3_RX_PTYPE_ENTRY(6, 0, COMPLETE, PARSE_FAIL),
 125        HNS3_RX_PTYPE_ENTRY(7, 0, COMPLETE, CNM),
 126        HNS3_RX_PTYPE_ENTRY(8, 0, NONE, PARSE_FAIL),
 127        HNS3_RX_PTYPE_UNUSED_ENTRY(9),
 128        HNS3_RX_PTYPE_UNUSED_ENTRY(10),
 129        HNS3_RX_PTYPE_UNUSED_ENTRY(11),
 130        HNS3_RX_PTYPE_UNUSED_ENTRY(12),
 131        HNS3_RX_PTYPE_UNUSED_ENTRY(13),
 132        HNS3_RX_PTYPE_UNUSED_ENTRY(14),
 133        HNS3_RX_PTYPE_UNUSED_ENTRY(15),
 134        HNS3_RX_PTYPE_ENTRY(16, 0, COMPLETE, PARSE_FAIL),
 135        HNS3_RX_PTYPE_ENTRY(17, 0, COMPLETE, IPV4),
 136        HNS3_RX_PTYPE_ENTRY(18, 0, COMPLETE, IPV4),
 137        HNS3_RX_PTYPE_ENTRY(19, 0, UNNECESSARY, IPV4),
 138        HNS3_RX_PTYPE_ENTRY(20, 0, UNNECESSARY, IPV4),
 139        HNS3_RX_PTYPE_ENTRY(21, 0, NONE, IPV4),
 140        HNS3_RX_PTYPE_ENTRY(22, 0, UNNECESSARY, IPV4),
 141        HNS3_RX_PTYPE_ENTRY(23, 0, NONE, IPV4),
 142        HNS3_RX_PTYPE_ENTRY(24, 0, NONE, IPV4),
 143        HNS3_RX_PTYPE_ENTRY(25, 0, UNNECESSARY, IPV4),
 144        HNS3_RX_PTYPE_UNUSED_ENTRY(26),
 145        HNS3_RX_PTYPE_UNUSED_ENTRY(27),
 146        HNS3_RX_PTYPE_UNUSED_ENTRY(28),
 147        HNS3_RX_PTYPE_ENTRY(29, 0, COMPLETE, PARSE_FAIL),
 148        HNS3_RX_PTYPE_ENTRY(30, 0, COMPLETE, PARSE_FAIL),
 149        HNS3_RX_PTYPE_ENTRY(31, 0, COMPLETE, IPV4),
 150        HNS3_RX_PTYPE_ENTRY(32, 0, COMPLETE, IPV4),
 151        HNS3_RX_PTYPE_ENTRY(33, 1, UNNECESSARY, IPV4),
 152        HNS3_RX_PTYPE_ENTRY(34, 1, UNNECESSARY, IPV4),
 153        HNS3_RX_PTYPE_ENTRY(35, 1, UNNECESSARY, IPV4),
 154        HNS3_RX_PTYPE_ENTRY(36, 0, COMPLETE, IPV4),
 155        HNS3_RX_PTYPE_ENTRY(37, 0, COMPLETE, IPV4),
 156        HNS3_RX_PTYPE_UNUSED_ENTRY(38),
 157        HNS3_RX_PTYPE_ENTRY(39, 0, COMPLETE, IPV6),
 158        HNS3_RX_PTYPE_ENTRY(40, 0, COMPLETE, IPV6),
 159        HNS3_RX_PTYPE_ENTRY(41, 1, UNNECESSARY, IPV6),
 160        HNS3_RX_PTYPE_ENTRY(42, 1, UNNECESSARY, IPV6),
 161        HNS3_RX_PTYPE_ENTRY(43, 1, UNNECESSARY, IPV6),
 162        HNS3_RX_PTYPE_ENTRY(44, 0, COMPLETE, IPV6),
 163        HNS3_RX_PTYPE_ENTRY(45, 0, COMPLETE, IPV6),
 164        HNS3_RX_PTYPE_UNUSED_ENTRY(46),
 165        HNS3_RX_PTYPE_UNUSED_ENTRY(47),
 166        HNS3_RX_PTYPE_UNUSED_ENTRY(48),
 167        HNS3_RX_PTYPE_UNUSED_ENTRY(49),
 168        HNS3_RX_PTYPE_UNUSED_ENTRY(50),
 169        HNS3_RX_PTYPE_UNUSED_ENTRY(51),
 170        HNS3_RX_PTYPE_UNUSED_ENTRY(52),
 171        HNS3_RX_PTYPE_UNUSED_ENTRY(53),
 172        HNS3_RX_PTYPE_UNUSED_ENTRY(54),
 173        HNS3_RX_PTYPE_UNUSED_ENTRY(55),
 174        HNS3_RX_PTYPE_UNUSED_ENTRY(56),
 175        HNS3_RX_PTYPE_UNUSED_ENTRY(57),
 176        HNS3_RX_PTYPE_UNUSED_ENTRY(58),
 177        HNS3_RX_PTYPE_UNUSED_ENTRY(59),
 178        HNS3_RX_PTYPE_UNUSED_ENTRY(60),
 179        HNS3_RX_PTYPE_UNUSED_ENTRY(61),
 180        HNS3_RX_PTYPE_UNUSED_ENTRY(62),
 181        HNS3_RX_PTYPE_UNUSED_ENTRY(63),
 182        HNS3_RX_PTYPE_UNUSED_ENTRY(64),
 183        HNS3_RX_PTYPE_UNUSED_ENTRY(65),
 184        HNS3_RX_PTYPE_UNUSED_ENTRY(66),
 185        HNS3_RX_PTYPE_UNUSED_ENTRY(67),
 186        HNS3_RX_PTYPE_UNUSED_ENTRY(68),
 187        HNS3_RX_PTYPE_UNUSED_ENTRY(69),
 188        HNS3_RX_PTYPE_UNUSED_ENTRY(70),
 189        HNS3_RX_PTYPE_UNUSED_ENTRY(71),
 190        HNS3_RX_PTYPE_UNUSED_ENTRY(72),
 191        HNS3_RX_PTYPE_UNUSED_ENTRY(73),
 192        HNS3_RX_PTYPE_UNUSED_ENTRY(74),
 193        HNS3_RX_PTYPE_UNUSED_ENTRY(75),
 194        HNS3_RX_PTYPE_UNUSED_ENTRY(76),
 195        HNS3_RX_PTYPE_UNUSED_ENTRY(77),
 196        HNS3_RX_PTYPE_UNUSED_ENTRY(78),
 197        HNS3_RX_PTYPE_UNUSED_ENTRY(79),
 198        HNS3_RX_PTYPE_UNUSED_ENTRY(80),
 199        HNS3_RX_PTYPE_UNUSED_ENTRY(81),
 200        HNS3_RX_PTYPE_UNUSED_ENTRY(82),
 201        HNS3_RX_PTYPE_UNUSED_ENTRY(83),
 202        HNS3_RX_PTYPE_UNUSED_ENTRY(84),
 203        HNS3_RX_PTYPE_UNUSED_ENTRY(85),
 204        HNS3_RX_PTYPE_UNUSED_ENTRY(86),
 205        HNS3_RX_PTYPE_UNUSED_ENTRY(87),
 206        HNS3_RX_PTYPE_UNUSED_ENTRY(88),
 207        HNS3_RX_PTYPE_UNUSED_ENTRY(89),
 208        HNS3_RX_PTYPE_UNUSED_ENTRY(90),
 209        HNS3_RX_PTYPE_UNUSED_ENTRY(91),
 210        HNS3_RX_PTYPE_UNUSED_ENTRY(92),
 211        HNS3_RX_PTYPE_UNUSED_ENTRY(93),
 212        HNS3_RX_PTYPE_UNUSED_ENTRY(94),
 213        HNS3_RX_PTYPE_UNUSED_ENTRY(95),
 214        HNS3_RX_PTYPE_UNUSED_ENTRY(96),
 215        HNS3_RX_PTYPE_UNUSED_ENTRY(97),
 216        HNS3_RX_PTYPE_UNUSED_ENTRY(98),
 217        HNS3_RX_PTYPE_UNUSED_ENTRY(99),
 218        HNS3_RX_PTYPE_UNUSED_ENTRY(100),
 219        HNS3_RX_PTYPE_UNUSED_ENTRY(101),
 220        HNS3_RX_PTYPE_UNUSED_ENTRY(102),
 221        HNS3_RX_PTYPE_UNUSED_ENTRY(103),
 222        HNS3_RX_PTYPE_UNUSED_ENTRY(104),
 223        HNS3_RX_PTYPE_UNUSED_ENTRY(105),
 224        HNS3_RX_PTYPE_UNUSED_ENTRY(106),
 225        HNS3_RX_PTYPE_UNUSED_ENTRY(107),
 226        HNS3_RX_PTYPE_UNUSED_ENTRY(108),
 227        HNS3_RX_PTYPE_UNUSED_ENTRY(109),
 228        HNS3_RX_PTYPE_UNUSED_ENTRY(110),
 229        HNS3_RX_PTYPE_ENTRY(111, 0, COMPLETE, IPV6),
 230        HNS3_RX_PTYPE_ENTRY(112, 0, COMPLETE, IPV6),
 231        HNS3_RX_PTYPE_ENTRY(113, 0, UNNECESSARY, IPV6),
 232        HNS3_RX_PTYPE_ENTRY(114, 0, UNNECESSARY, IPV6),
 233        HNS3_RX_PTYPE_ENTRY(115, 0, NONE, IPV6),
 234        HNS3_RX_PTYPE_ENTRY(116, 0, UNNECESSARY, IPV6),
 235        HNS3_RX_PTYPE_ENTRY(117, 0, NONE, IPV6),
 236        HNS3_RX_PTYPE_ENTRY(118, 0, NONE, IPV6),
 237        HNS3_RX_PTYPE_ENTRY(119, 0, UNNECESSARY, IPV6),
 238        HNS3_RX_PTYPE_UNUSED_ENTRY(120),
 239        HNS3_RX_PTYPE_UNUSED_ENTRY(121),
 240        HNS3_RX_PTYPE_UNUSED_ENTRY(122),
 241        HNS3_RX_PTYPE_ENTRY(123, 0, COMPLETE, PARSE_FAIL),
 242        HNS3_RX_PTYPE_ENTRY(124, 0, COMPLETE, PARSE_FAIL),
 243        HNS3_RX_PTYPE_ENTRY(125, 0, COMPLETE, IPV4),
 244        HNS3_RX_PTYPE_ENTRY(126, 0, COMPLETE, IPV4),
 245        HNS3_RX_PTYPE_ENTRY(127, 1, UNNECESSARY, IPV4),
 246        HNS3_RX_PTYPE_ENTRY(128, 1, UNNECESSARY, IPV4),
 247        HNS3_RX_PTYPE_ENTRY(129, 1, UNNECESSARY, IPV4),
 248        HNS3_RX_PTYPE_ENTRY(130, 0, COMPLETE, IPV4),
 249        HNS3_RX_PTYPE_ENTRY(131, 0, COMPLETE, IPV4),
 250        HNS3_RX_PTYPE_UNUSED_ENTRY(132),
 251        HNS3_RX_PTYPE_ENTRY(133, 0, COMPLETE, IPV6),
 252        HNS3_RX_PTYPE_ENTRY(134, 0, COMPLETE, IPV6),
 253        HNS3_RX_PTYPE_ENTRY(135, 1, UNNECESSARY, IPV6),
 254        HNS3_RX_PTYPE_ENTRY(136, 1, UNNECESSARY, IPV6),
 255        HNS3_RX_PTYPE_ENTRY(137, 1, UNNECESSARY, IPV6),
 256        HNS3_RX_PTYPE_ENTRY(138, 0, COMPLETE, IPV6),
 257        HNS3_RX_PTYPE_ENTRY(139, 0, COMPLETE, IPV6),
 258        HNS3_RX_PTYPE_UNUSED_ENTRY(140),
 259        HNS3_RX_PTYPE_UNUSED_ENTRY(141),
 260        HNS3_RX_PTYPE_UNUSED_ENTRY(142),
 261        HNS3_RX_PTYPE_UNUSED_ENTRY(143),
 262        HNS3_RX_PTYPE_UNUSED_ENTRY(144),
 263        HNS3_RX_PTYPE_UNUSED_ENTRY(145),
 264        HNS3_RX_PTYPE_UNUSED_ENTRY(146),
 265        HNS3_RX_PTYPE_UNUSED_ENTRY(147),
 266        HNS3_RX_PTYPE_UNUSED_ENTRY(148),
 267        HNS3_RX_PTYPE_UNUSED_ENTRY(149),
 268        HNS3_RX_PTYPE_UNUSED_ENTRY(150),
 269        HNS3_RX_PTYPE_UNUSED_ENTRY(151),
 270        HNS3_RX_PTYPE_UNUSED_ENTRY(152),
 271        HNS3_RX_PTYPE_UNUSED_ENTRY(153),
 272        HNS3_RX_PTYPE_UNUSED_ENTRY(154),
 273        HNS3_RX_PTYPE_UNUSED_ENTRY(155),
 274        HNS3_RX_PTYPE_UNUSED_ENTRY(156),
 275        HNS3_RX_PTYPE_UNUSED_ENTRY(157),
 276        HNS3_RX_PTYPE_UNUSED_ENTRY(158),
 277        HNS3_RX_PTYPE_UNUSED_ENTRY(159),
 278        HNS3_RX_PTYPE_UNUSED_ENTRY(160),
 279        HNS3_RX_PTYPE_UNUSED_ENTRY(161),
 280        HNS3_RX_PTYPE_UNUSED_ENTRY(162),
 281        HNS3_RX_PTYPE_UNUSED_ENTRY(163),
 282        HNS3_RX_PTYPE_UNUSED_ENTRY(164),
 283        HNS3_RX_PTYPE_UNUSED_ENTRY(165),
 284        HNS3_RX_PTYPE_UNUSED_ENTRY(166),
 285        HNS3_RX_PTYPE_UNUSED_ENTRY(167),
 286        HNS3_RX_PTYPE_UNUSED_ENTRY(168),
 287        HNS3_RX_PTYPE_UNUSED_ENTRY(169),
 288        HNS3_RX_PTYPE_UNUSED_ENTRY(170),
 289        HNS3_RX_PTYPE_UNUSED_ENTRY(171),
 290        HNS3_RX_PTYPE_UNUSED_ENTRY(172),
 291        HNS3_RX_PTYPE_UNUSED_ENTRY(173),
 292        HNS3_RX_PTYPE_UNUSED_ENTRY(174),
 293        HNS3_RX_PTYPE_UNUSED_ENTRY(175),
 294        HNS3_RX_PTYPE_UNUSED_ENTRY(176),
 295        HNS3_RX_PTYPE_UNUSED_ENTRY(177),
 296        HNS3_RX_PTYPE_UNUSED_ENTRY(178),
 297        HNS3_RX_PTYPE_UNUSED_ENTRY(179),
 298        HNS3_RX_PTYPE_UNUSED_ENTRY(180),
 299        HNS3_RX_PTYPE_UNUSED_ENTRY(181),
 300        HNS3_RX_PTYPE_UNUSED_ENTRY(182),
 301        HNS3_RX_PTYPE_UNUSED_ENTRY(183),
 302        HNS3_RX_PTYPE_UNUSED_ENTRY(184),
 303        HNS3_RX_PTYPE_UNUSED_ENTRY(185),
 304        HNS3_RX_PTYPE_UNUSED_ENTRY(186),
 305        HNS3_RX_PTYPE_UNUSED_ENTRY(187),
 306        HNS3_RX_PTYPE_UNUSED_ENTRY(188),
 307        HNS3_RX_PTYPE_UNUSED_ENTRY(189),
 308        HNS3_RX_PTYPE_UNUSED_ENTRY(190),
 309        HNS3_RX_PTYPE_UNUSED_ENTRY(191),
 310        HNS3_RX_PTYPE_UNUSED_ENTRY(192),
 311        HNS3_RX_PTYPE_UNUSED_ENTRY(193),
 312        HNS3_RX_PTYPE_UNUSED_ENTRY(194),
 313        HNS3_RX_PTYPE_UNUSED_ENTRY(195),
 314        HNS3_RX_PTYPE_UNUSED_ENTRY(196),
 315        HNS3_RX_PTYPE_UNUSED_ENTRY(197),
 316        HNS3_RX_PTYPE_UNUSED_ENTRY(198),
 317        HNS3_RX_PTYPE_UNUSED_ENTRY(199),
 318        HNS3_RX_PTYPE_UNUSED_ENTRY(200),
 319        HNS3_RX_PTYPE_UNUSED_ENTRY(201),
 320        HNS3_RX_PTYPE_UNUSED_ENTRY(202),
 321        HNS3_RX_PTYPE_UNUSED_ENTRY(203),
 322        HNS3_RX_PTYPE_UNUSED_ENTRY(204),
 323        HNS3_RX_PTYPE_UNUSED_ENTRY(205),
 324        HNS3_RX_PTYPE_UNUSED_ENTRY(206),
 325        HNS3_RX_PTYPE_UNUSED_ENTRY(207),
 326        HNS3_RX_PTYPE_UNUSED_ENTRY(208),
 327        HNS3_RX_PTYPE_UNUSED_ENTRY(209),
 328        HNS3_RX_PTYPE_UNUSED_ENTRY(210),
 329        HNS3_RX_PTYPE_UNUSED_ENTRY(211),
 330        HNS3_RX_PTYPE_UNUSED_ENTRY(212),
 331        HNS3_RX_PTYPE_UNUSED_ENTRY(213),
 332        HNS3_RX_PTYPE_UNUSED_ENTRY(214),
 333        HNS3_RX_PTYPE_UNUSED_ENTRY(215),
 334        HNS3_RX_PTYPE_UNUSED_ENTRY(216),
 335        HNS3_RX_PTYPE_UNUSED_ENTRY(217),
 336        HNS3_RX_PTYPE_UNUSED_ENTRY(218),
 337        HNS3_RX_PTYPE_UNUSED_ENTRY(219),
 338        HNS3_RX_PTYPE_UNUSED_ENTRY(220),
 339        HNS3_RX_PTYPE_UNUSED_ENTRY(221),
 340        HNS3_RX_PTYPE_UNUSED_ENTRY(222),
 341        HNS3_RX_PTYPE_UNUSED_ENTRY(223),
 342        HNS3_RX_PTYPE_UNUSED_ENTRY(224),
 343        HNS3_RX_PTYPE_UNUSED_ENTRY(225),
 344        HNS3_RX_PTYPE_UNUSED_ENTRY(226),
 345        HNS3_RX_PTYPE_UNUSED_ENTRY(227),
 346        HNS3_RX_PTYPE_UNUSED_ENTRY(228),
 347        HNS3_RX_PTYPE_UNUSED_ENTRY(229),
 348        HNS3_RX_PTYPE_UNUSED_ENTRY(230),
 349        HNS3_RX_PTYPE_UNUSED_ENTRY(231),
 350        HNS3_RX_PTYPE_UNUSED_ENTRY(232),
 351        HNS3_RX_PTYPE_UNUSED_ENTRY(233),
 352        HNS3_RX_PTYPE_UNUSED_ENTRY(234),
 353        HNS3_RX_PTYPE_UNUSED_ENTRY(235),
 354        HNS3_RX_PTYPE_UNUSED_ENTRY(236),
 355        HNS3_RX_PTYPE_UNUSED_ENTRY(237),
 356        HNS3_RX_PTYPE_UNUSED_ENTRY(238),
 357        HNS3_RX_PTYPE_UNUSED_ENTRY(239),
 358        HNS3_RX_PTYPE_UNUSED_ENTRY(240),
 359        HNS3_RX_PTYPE_UNUSED_ENTRY(241),
 360        HNS3_RX_PTYPE_UNUSED_ENTRY(242),
 361        HNS3_RX_PTYPE_UNUSED_ENTRY(243),
 362        HNS3_RX_PTYPE_UNUSED_ENTRY(244),
 363        HNS3_RX_PTYPE_UNUSED_ENTRY(245),
 364        HNS3_RX_PTYPE_UNUSED_ENTRY(246),
 365        HNS3_RX_PTYPE_UNUSED_ENTRY(247),
 366        HNS3_RX_PTYPE_UNUSED_ENTRY(248),
 367        HNS3_RX_PTYPE_UNUSED_ENTRY(249),
 368        HNS3_RX_PTYPE_UNUSED_ENTRY(250),
 369        HNS3_RX_PTYPE_UNUSED_ENTRY(251),
 370        HNS3_RX_PTYPE_UNUSED_ENTRY(252),
 371        HNS3_RX_PTYPE_UNUSED_ENTRY(253),
 372        HNS3_RX_PTYPE_UNUSED_ENTRY(254),
 373        HNS3_RX_PTYPE_UNUSED_ENTRY(255),
 374};
 375
 376#define HNS3_INVALID_PTYPE \
 377                ARRAY_SIZE(hns3_rx_ptype_tbl)
 378
 379static irqreturn_t hns3_irq_handle(int irq, void *vector)
 380{
 381        struct hns3_enet_tqp_vector *tqp_vector = vector;
 382
 383        napi_schedule_irqoff(&tqp_vector->napi);
 384        tqp_vector->event_cnt++;
 385
 386        return IRQ_HANDLED;
 387}
 388
 389static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
 390{
 391        struct hns3_enet_tqp_vector *tqp_vectors;
 392        unsigned int i;
 393
 394        for (i = 0; i < priv->vector_num; i++) {
 395                tqp_vectors = &priv->tqp_vector[i];
 396
 397                if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
 398                        continue;
 399
 400                /* clear the affinity mask */
 401                irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
 402
 403                /* release the irq resource */
 404                free_irq(tqp_vectors->vector_irq, tqp_vectors);
 405                tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
 406        }
 407}
 408
 409static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
 410{
 411        struct hns3_enet_tqp_vector *tqp_vectors;
 412        int txrx_int_idx = 0;
 413        int rx_int_idx = 0;
 414        int tx_int_idx = 0;
 415        unsigned int i;
 416        int ret;
 417
 418        for (i = 0; i < priv->vector_num; i++) {
 419                tqp_vectors = &priv->tqp_vector[i];
 420
 421                if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 422                        continue;
 423
 424                if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 425                        snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 426                                 "%s-%s-%s-%d", hns3_driver_name,
 427                                 pci_name(priv->ae_handle->pdev),
 428                                 "TxRx", txrx_int_idx++);
 429                        txrx_int_idx++;
 430                } else if (tqp_vectors->rx_group.ring) {
 431                        snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 432                                 "%s-%s-%s-%d", hns3_driver_name,
 433                                 pci_name(priv->ae_handle->pdev),
 434                                 "Rx", rx_int_idx++);
 435                } else if (tqp_vectors->tx_group.ring) {
 436                        snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 437                                 "%s-%s-%s-%d", hns3_driver_name,
 438                                 pci_name(priv->ae_handle->pdev),
 439                                 "Tx", tx_int_idx++);
 440                } else {
 441                        /* Skip this unused q_vector */
 442                        continue;
 443                }
 444
 445                tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 446
 447                irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN);
 448                ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 449                                  tqp_vectors->name, tqp_vectors);
 450                if (ret) {
 451                        netdev_err(priv->netdev, "request irq(%d) fail\n",
 452                                   tqp_vectors->vector_irq);
 453                        hns3_nic_uninit_irq(priv);
 454                        return ret;
 455                }
 456
 457                irq_set_affinity_hint(tqp_vectors->vector_irq,
 458                                      &tqp_vectors->affinity_mask);
 459
 460                tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 461        }
 462
 463        return 0;
 464}
 465
 466static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 467                                 u32 mask_en)
 468{
 469        writel(mask_en, tqp_vector->mask_addr);
 470}
 471
 472static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 473{
 474        napi_enable(&tqp_vector->napi);
 475        enable_irq(tqp_vector->vector_irq);
 476
 477        /* enable vector */
 478        hns3_mask_vector_irq(tqp_vector, 1);
 479}
 480
 481static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 482{
 483        /* disable vector */
 484        hns3_mask_vector_irq(tqp_vector, 0);
 485
 486        disable_irq(tqp_vector->vector_irq);
 487        napi_disable(&tqp_vector->napi);
 488        cancel_work_sync(&tqp_vector->rx_group.dim.work);
 489        cancel_work_sync(&tqp_vector->tx_group.dim.work);
 490}
 491
 492void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 493                                 u32 rl_value)
 494{
 495        u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 496
 497        /* this defines the configuration for RL (Interrupt Rate Limiter).
 498         * Rl defines rate of interrupts i.e. number of interrupts-per-second
 499         * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 500         */
 501        if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&
 502            !tqp_vector->rx_group.coal.adapt_enable)
 503                /* According to the hardware, the range of rl_reg is
 504                 * 0-59 and the unit is 4.
 505                 */
 506                rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 507
 508        writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 509}
 510
 511void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 512                                    u32 gl_value)
 513{
 514        u32 new_val;
 515
 516        if (tqp_vector->rx_group.coal.unit_1us)
 517                new_val = gl_value | HNS3_INT_GL_1US;
 518        else
 519                new_val = hns3_gl_usec_to_reg(gl_value);
 520
 521        writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 522}
 523
 524void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 525                                    u32 gl_value)
 526{
 527        u32 new_val;
 528
 529        if (tqp_vector->tx_group.coal.unit_1us)
 530                new_val = gl_value | HNS3_INT_GL_1US;
 531        else
 532                new_val = hns3_gl_usec_to_reg(gl_value);
 533
 534        writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 535}
 536
 537void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,
 538                                    u32 ql_value)
 539{
 540        writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);
 541}
 542
 543void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,
 544                                    u32 ql_value)
 545{
 546        writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);
 547}
 548
 549static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,
 550                                      struct hns3_nic_priv *priv)
 551{
 552        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
 553        struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
 554        struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
 555        struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal;
 556        struct hns3_enet_coalesce *prx_coal = &priv->rx_coal;
 557
 558        tx_coal->adapt_enable = ptx_coal->adapt_enable;
 559        rx_coal->adapt_enable = prx_coal->adapt_enable;
 560
 561        tx_coal->int_gl = ptx_coal->int_gl;
 562        rx_coal->int_gl = prx_coal->int_gl;
 563
 564        rx_coal->flow_level = prx_coal->flow_level;
 565        tx_coal->flow_level = ptx_coal->flow_level;
 566
 567        /* device version above V3(include V3), GL can configure 1us
 568         * unit, so uses 1us unit.
 569         */
 570        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {
 571                tx_coal->unit_1us = 1;
 572                rx_coal->unit_1us = 1;
 573        }
 574
 575        if (ae_dev->dev_specs.int_ql_max) {
 576                tx_coal->ql_enable = 1;
 577                rx_coal->ql_enable = 1;
 578                tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
 579                rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
 580                tx_coal->int_ql = ptx_coal->int_ql;
 581                rx_coal->int_ql = prx_coal->int_ql;
 582        }
 583}
 584
 585static void
 586hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 587                             struct hns3_nic_priv *priv)
 588{
 589        struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
 590        struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
 591        struct hnae3_handle *h = priv->ae_handle;
 592
 593        hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);
 594        hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);
 595        hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 596
 597        if (tx_coal->ql_enable)
 598                hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);
 599
 600        if (rx_coal->ql_enable)
 601                hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);
 602}
 603
 604static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 605{
 606        struct hnae3_handle *h = hns3_get_handle(netdev);
 607        struct hnae3_knic_private_info *kinfo = &h->kinfo;
 608        struct hnae3_tc_info *tc_info = &kinfo->tc_info;
 609        unsigned int queue_size = kinfo->num_tqps;
 610        int i, ret;
 611
 612        if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) {
 613                netdev_reset_tc(netdev);
 614        } else {
 615                ret = netdev_set_num_tc(netdev, tc_info->num_tc);
 616                if (ret) {
 617                        netdev_err(netdev,
 618                                   "netdev_set_num_tc fail, ret=%d!\n", ret);
 619                        return ret;
 620                }
 621
 622                for (i = 0; i < HNAE3_MAX_TC; i++) {
 623                        if (!test_bit(i, &tc_info->tc_en))
 624                                continue;
 625
 626                        netdev_set_tc_queue(netdev, i, tc_info->tqp_count[i],
 627                                            tc_info->tqp_offset[i]);
 628                }
 629        }
 630
 631        ret = netif_set_real_num_tx_queues(netdev, queue_size);
 632        if (ret) {
 633                netdev_err(netdev,
 634                           "netif_set_real_num_tx_queues fail, ret=%d!\n", ret);
 635                return ret;
 636        }
 637
 638        ret = netif_set_real_num_rx_queues(netdev, queue_size);
 639        if (ret) {
 640                netdev_err(netdev,
 641                           "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 642                return ret;
 643        }
 644
 645        return 0;
 646}
 647
 648u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 649{
 650        u16 alloc_tqps, max_rss_size, rss_size;
 651
 652        h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
 653        rss_size = alloc_tqps / h->kinfo.tc_info.num_tc;
 654
 655        return min_t(u16, rss_size, max_rss_size);
 656}
 657
 658static void hns3_tqp_enable(struct hnae3_queue *tqp)
 659{
 660        u32 rcb_reg;
 661
 662        rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 663        rcb_reg |= BIT(HNS3_RING_EN_B);
 664        hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 665}
 666
 667static void hns3_tqp_disable(struct hnae3_queue *tqp)
 668{
 669        u32 rcb_reg;
 670
 671        rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 672        rcb_reg &= ~BIT(HNS3_RING_EN_B);
 673        hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 674}
 675
 676static void hns3_free_rx_cpu_rmap(struct net_device *netdev)
 677{
 678#ifdef CONFIG_RFS_ACCEL
 679        free_irq_cpu_rmap(netdev->rx_cpu_rmap);
 680        netdev->rx_cpu_rmap = NULL;
 681#endif
 682}
 683
 684static int hns3_set_rx_cpu_rmap(struct net_device *netdev)
 685{
 686#ifdef CONFIG_RFS_ACCEL
 687        struct hns3_nic_priv *priv = netdev_priv(netdev);
 688        struct hns3_enet_tqp_vector *tqp_vector;
 689        int i, ret;
 690
 691        if (!netdev->rx_cpu_rmap) {
 692                netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num);
 693                if (!netdev->rx_cpu_rmap)
 694                        return -ENOMEM;
 695        }
 696
 697        for (i = 0; i < priv->vector_num; i++) {
 698                tqp_vector = &priv->tqp_vector[i];
 699                ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap,
 700                                       tqp_vector->vector_irq);
 701                if (ret) {
 702                        hns3_free_rx_cpu_rmap(netdev);
 703                        return ret;
 704                }
 705        }
 706#endif
 707        return 0;
 708}
 709
 710static int hns3_nic_net_up(struct net_device *netdev)
 711{
 712        struct hns3_nic_priv *priv = netdev_priv(netdev);
 713        struct hnae3_handle *h = priv->ae_handle;
 714        int i, j;
 715        int ret;
 716
 717        ret = hns3_nic_reset_all_ring(h);
 718        if (ret)
 719                return ret;
 720
 721        clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 722
 723        /* enable the vectors */
 724        for (i = 0; i < priv->vector_num; i++)
 725                hns3_vector_enable(&priv->tqp_vector[i]);
 726
 727        /* enable rcb */
 728        for (j = 0; j < h->kinfo.num_tqps; j++)
 729                hns3_tqp_enable(h->kinfo.tqp[j]);
 730
 731        /* start the ae_dev */
 732        ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 733        if (ret) {
 734                set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 735                while (j--)
 736                        hns3_tqp_disable(h->kinfo.tqp[j]);
 737
 738                for (j = i - 1; j >= 0; j--)
 739                        hns3_vector_disable(&priv->tqp_vector[j]);
 740        }
 741
 742        return ret;
 743}
 744
 745static void hns3_config_xps(struct hns3_nic_priv *priv)
 746{
 747        int i;
 748
 749        for (i = 0; i < priv->vector_num; i++) {
 750                struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];
 751                struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;
 752
 753                while (ring) {
 754                        int ret;
 755
 756                        ret = netif_set_xps_queue(priv->netdev,
 757                                                  &tqp_vector->affinity_mask,
 758                                                  ring->tqp->tqp_index);
 759                        if (ret)
 760                                netdev_warn(priv->netdev,
 761                                            "set xps queue failed: %d", ret);
 762
 763                        ring = ring->next;
 764                }
 765        }
 766}
 767
 768static int hns3_nic_net_open(struct net_device *netdev)
 769{
 770        struct hns3_nic_priv *priv = netdev_priv(netdev);
 771        struct hnae3_handle *h = hns3_get_handle(netdev);
 772        struct hnae3_knic_private_info *kinfo;
 773        int i, ret;
 774
 775        if (hns3_nic_resetting(netdev))
 776                return -EBUSY;
 777
 778        netif_carrier_off(netdev);
 779
 780        ret = hns3_nic_set_real_num_queue(netdev);
 781        if (ret)
 782                return ret;
 783
 784        ret = hns3_nic_net_up(netdev);
 785        if (ret) {
 786                netdev_err(netdev, "net up fail, ret=%d!\n", ret);
 787                return ret;
 788        }
 789
 790        kinfo = &h->kinfo;
 791        for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)
 792                netdev_set_prio_tc_map(netdev, i, kinfo->tc_info.prio_tc[i]);
 793
 794        if (h->ae_algo->ops->set_timer_task)
 795                h->ae_algo->ops->set_timer_task(priv->ae_handle, true);
 796
 797        hns3_config_xps(priv);
 798
 799        netif_dbg(h, drv, netdev, "net open\n");
 800
 801        return 0;
 802}
 803
 804static void hns3_reset_tx_queue(struct hnae3_handle *h)
 805{
 806        struct net_device *ndev = h->kinfo.netdev;
 807        struct hns3_nic_priv *priv = netdev_priv(ndev);
 808        struct netdev_queue *dev_queue;
 809        u32 i;
 810
 811        for (i = 0; i < h->kinfo.num_tqps; i++) {
 812                dev_queue = netdev_get_tx_queue(ndev,
 813                                                priv->ring[i].queue_index);
 814                netdev_tx_reset_queue(dev_queue);
 815        }
 816}
 817
 818static void hns3_nic_net_down(struct net_device *netdev)
 819{
 820        struct hns3_nic_priv *priv = netdev_priv(netdev);
 821        struct hnae3_handle *h = hns3_get_handle(netdev);
 822        const struct hnae3_ae_ops *ops;
 823        int i;
 824
 825        /* disable vectors */
 826        for (i = 0; i < priv->vector_num; i++)
 827                hns3_vector_disable(&priv->tqp_vector[i]);
 828
 829        /* disable rcb */
 830        for (i = 0; i < h->kinfo.num_tqps; i++)
 831                hns3_tqp_disable(h->kinfo.tqp[i]);
 832
 833        /* stop ae_dev */
 834        ops = priv->ae_handle->ae_algo->ops;
 835        if (ops->stop)
 836                ops->stop(priv->ae_handle);
 837
 838        /* delay ring buffer clearing to hns3_reset_notify_uninit_enet
 839         * during reset process, because driver may not be able
 840         * to disable the ring through firmware when downing the netdev.
 841         */
 842        if (!hns3_nic_resetting(netdev))
 843                hns3_clear_all_ring(priv->ae_handle, false);
 844
 845        hns3_reset_tx_queue(priv->ae_handle);
 846}
 847
 848static int hns3_nic_net_stop(struct net_device *netdev)
 849{
 850        struct hns3_nic_priv *priv = netdev_priv(netdev);
 851        struct hnae3_handle *h = hns3_get_handle(netdev);
 852
 853        if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 854                return 0;
 855
 856        netif_dbg(h, drv, netdev, "net stop\n");
 857
 858        if (h->ae_algo->ops->set_timer_task)
 859                h->ae_algo->ops->set_timer_task(priv->ae_handle, false);
 860
 861        netif_carrier_off(netdev);
 862        netif_tx_disable(netdev);
 863
 864        hns3_nic_net_down(netdev);
 865
 866        return 0;
 867}
 868
 869static int hns3_nic_uc_sync(struct net_device *netdev,
 870                            const unsigned char *addr)
 871{
 872        struct hnae3_handle *h = hns3_get_handle(netdev);
 873
 874        if (h->ae_algo->ops->add_uc_addr)
 875                return h->ae_algo->ops->add_uc_addr(h, addr);
 876
 877        return 0;
 878}
 879
 880static int hns3_nic_uc_unsync(struct net_device *netdev,
 881                              const unsigned char *addr)
 882{
 883        struct hnae3_handle *h = hns3_get_handle(netdev);
 884
 885        /* need ignore the request of removing device address, because
 886         * we store the device address and other addresses of uc list
 887         * in the function's mac filter list.
 888         */
 889        if (ether_addr_equal(addr, netdev->dev_addr))
 890                return 0;
 891
 892        if (h->ae_algo->ops->rm_uc_addr)
 893                return h->ae_algo->ops->rm_uc_addr(h, addr);
 894
 895        return 0;
 896}
 897
 898static int hns3_nic_mc_sync(struct net_device *netdev,
 899                            const unsigned char *addr)
 900{
 901        struct hnae3_handle *h = hns3_get_handle(netdev);
 902
 903        if (h->ae_algo->ops->add_mc_addr)
 904                return h->ae_algo->ops->add_mc_addr(h, addr);
 905
 906        return 0;
 907}
 908
 909static int hns3_nic_mc_unsync(struct net_device *netdev,
 910                              const unsigned char *addr)
 911{
 912        struct hnae3_handle *h = hns3_get_handle(netdev);
 913
 914        if (h->ae_algo->ops->rm_mc_addr)
 915                return h->ae_algo->ops->rm_mc_addr(h, addr);
 916
 917        return 0;
 918}
 919
 920static u8 hns3_get_netdev_flags(struct net_device *netdev)
 921{
 922        u8 flags = 0;
 923
 924        if (netdev->flags & IFF_PROMISC)
 925                flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE;
 926        else if (netdev->flags & IFF_ALLMULTI)
 927                flags = HNAE3_USER_MPE;
 928
 929        return flags;
 930}
 931
 932static void hns3_nic_set_rx_mode(struct net_device *netdev)
 933{
 934        struct hnae3_handle *h = hns3_get_handle(netdev);
 935        u8 new_flags;
 936
 937        new_flags = hns3_get_netdev_flags(netdev);
 938
 939        __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
 940        __dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync);
 941
 942        /* User mode Promisc mode enable and vlan filtering is disabled to
 943         * let all packets in.
 944         */
 945        h->netdev_flags = new_flags;
 946        hns3_request_update_promisc_mode(h);
 947}
 948
 949void hns3_request_update_promisc_mode(struct hnae3_handle *handle)
 950{
 951        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 952
 953        if (ops->request_update_promisc_mode)
 954                ops->request_update_promisc_mode(handle);
 955}
 956
 957static u32 hns3_tx_spare_space(struct hns3_enet_ring *ring)
 958{
 959        struct hns3_tx_spare *tx_spare = ring->tx_spare;
 960        u32 ntc, ntu;
 961
 962        /* This smp_load_acquire() pairs with smp_store_release() in
 963         * hns3_tx_spare_update() called in tx desc cleaning process.
 964         */
 965        ntc = smp_load_acquire(&tx_spare->last_to_clean);
 966        ntu = tx_spare->next_to_use;
 967
 968        if (ntc > ntu)
 969                return ntc - ntu - 1;
 970
 971        /* The free tx buffer is divided into two part, so pick the
 972         * larger one.
 973         */
 974        return (ntc > (tx_spare->len - ntu) ? ntc :
 975                        (tx_spare->len - ntu)) - 1;
 976}
 977
 978static void hns3_tx_spare_update(struct hns3_enet_ring *ring)
 979{
 980        struct hns3_tx_spare *tx_spare = ring->tx_spare;
 981
 982        if (!tx_spare ||
 983            tx_spare->last_to_clean == tx_spare->next_to_clean)
 984                return;
 985
 986        /* This smp_store_release() pairs with smp_load_acquire() in
 987         * hns3_tx_spare_space() called in xmit process.
 988         */
 989        smp_store_release(&tx_spare->last_to_clean,
 990                          tx_spare->next_to_clean);
 991}
 992
 993static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring,
 994                                   struct sk_buff *skb,
 995                                   u32 space)
 996{
 997        u32 len = skb->len <= ring->tx_copybreak ? skb->len :
 998                                skb_headlen(skb);
 999
1000        if (len > ring->tx_copybreak)
1001                return false;
1002
1003        if (ALIGN(len, dma_get_cache_alignment()) > space) {
1004                u64_stats_update_begin(&ring->syncp);
1005                ring->stats.tx_spare_full++;
1006                u64_stats_update_end(&ring->syncp);
1007                return false;
1008        }
1009
1010        return true;
1011}
1012
1013static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring,
1014                                struct sk_buff *skb,
1015                                u32 space)
1016{
1017        if (skb->len <= ring->tx_copybreak || !tx_sgl ||
1018            (!skb_has_frag_list(skb) &&
1019             skb_shinfo(skb)->nr_frags < tx_sgl))
1020                return false;
1021
1022        if (space < HNS3_MAX_SGL_SIZE) {
1023                u64_stats_update_begin(&ring->syncp);
1024                ring->stats.tx_spare_full++;
1025                u64_stats_update_end(&ring->syncp);
1026                return false;
1027        }
1028
1029        return true;
1030}
1031
1032static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring)
1033{
1034        struct hns3_tx_spare *tx_spare;
1035        struct page *page;
1036        u32 alloc_size;
1037        dma_addr_t dma;
1038        int order;
1039
1040        alloc_size = tx_spare_buf_size ? tx_spare_buf_size :
1041                     ring->tqp->handle->kinfo.tx_spare_buf_size;
1042        if (!alloc_size)
1043                return;
1044
1045        order = get_order(alloc_size);
1046        tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare),
1047                                GFP_KERNEL);
1048        if (!tx_spare) {
1049                /* The driver still work without the tx spare buffer */
1050                dev_warn(ring_to_dev(ring), "failed to allocate hns3_tx_spare\n");
1051                return;
1052        }
1053
1054        page = alloc_pages_node(dev_to_node(ring_to_dev(ring)),
1055                                GFP_KERNEL, order);
1056        if (!page) {
1057                dev_warn(ring_to_dev(ring), "failed to allocate tx spare pages\n");
1058                devm_kfree(ring_to_dev(ring), tx_spare);
1059                return;
1060        }
1061
1062        dma = dma_map_page(ring_to_dev(ring), page, 0,
1063                           PAGE_SIZE << order, DMA_TO_DEVICE);
1064        if (dma_mapping_error(ring_to_dev(ring), dma)) {
1065                dev_warn(ring_to_dev(ring), "failed to map pages for tx spare\n");
1066                put_page(page);
1067                devm_kfree(ring_to_dev(ring), tx_spare);
1068                return;
1069        }
1070
1071        tx_spare->dma = dma;
1072        tx_spare->buf = page_address(page);
1073        tx_spare->len = PAGE_SIZE << order;
1074        ring->tx_spare = tx_spare;
1075}
1076
1077/* Use hns3_tx_spare_space() to make sure there is enough buffer
1078 * before calling below function to allocate tx buffer.
1079 */
1080static void *hns3_tx_spare_alloc(struct hns3_enet_ring *ring,
1081                                 unsigned int size, dma_addr_t *dma,
1082                                 u32 *cb_len)
1083{
1084        struct hns3_tx_spare *tx_spare = ring->tx_spare;
1085        u32 ntu = tx_spare->next_to_use;
1086
1087        size = ALIGN(size, dma_get_cache_alignment());
1088        *cb_len = size;
1089
1090        /* Tx spare buffer wraps back here because the end of
1091         * freed tx buffer is not enough.
1092         */
1093        if (ntu + size > tx_spare->len) {
1094                *cb_len += (tx_spare->len - ntu);
1095                ntu = 0;
1096        }
1097
1098        tx_spare->next_to_use = ntu + size;
1099        if (tx_spare->next_to_use == tx_spare->len)
1100                tx_spare->next_to_use = 0;
1101
1102        *dma = tx_spare->dma + ntu;
1103
1104        return tx_spare->buf + ntu;
1105}
1106
1107static void hns3_tx_spare_rollback(struct hns3_enet_ring *ring, u32 len)
1108{
1109        struct hns3_tx_spare *tx_spare = ring->tx_spare;
1110
1111        if (len > tx_spare->next_to_use) {
1112                len -= tx_spare->next_to_use;
1113                tx_spare->next_to_use = tx_spare->len - len;
1114        } else {
1115                tx_spare->next_to_use -= len;
1116        }
1117}
1118
1119static void hns3_tx_spare_reclaim_cb(struct hns3_enet_ring *ring,
1120                                     struct hns3_desc_cb *cb)
1121{
1122        struct hns3_tx_spare *tx_spare = ring->tx_spare;
1123        u32 ntc = tx_spare->next_to_clean;
1124        u32 len = cb->length;
1125
1126        tx_spare->next_to_clean += len;
1127
1128        if (tx_spare->next_to_clean >= tx_spare->len) {
1129                tx_spare->next_to_clean -= tx_spare->len;
1130
1131                if (tx_spare->next_to_clean) {
1132                        ntc = 0;
1133                        len = tx_spare->next_to_clean;
1134                }
1135        }
1136
1137        /* This tx spare buffer is only really reclaimed after calling
1138         * hns3_tx_spare_update(), so it is still safe to use the info in
1139         * the tx buffer to do the dma sync or sg unmapping after
1140         * tx_spare->next_to_clean is moved forword.
1141         */
1142        if (cb->type & (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) {
1143                dma_addr_t dma = tx_spare->dma + ntc;
1144
1145                dma_sync_single_for_cpu(ring_to_dev(ring), dma, len,
1146                                        DMA_TO_DEVICE);
1147        } else {
1148                struct sg_table *sgt = tx_spare->buf + ntc;
1149
1150                dma_unmap_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents,
1151                             DMA_TO_DEVICE);
1152        }
1153}
1154
1155static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs,
1156                        u16 *mss, u32 *type_cs_vlan_tso, u32 *send_bytes)
1157{
1158        u32 l4_offset, hdr_len;
1159        union l3_hdr_info l3;
1160        union l4_hdr_info l4;
1161        u32 l4_paylen;
1162        int ret;
1163
1164        if (!skb_is_gso(skb))
1165                return 0;
1166
1167        ret = skb_cow_head(skb, 0);
1168        if (unlikely(ret < 0))
1169                return ret;
1170
1171        l3.hdr = skb_network_header(skb);
1172        l4.hdr = skb_transport_header(skb);
1173
1174        /* Software should clear the IPv4's checksum field when tso is
1175         * needed.
1176         */
1177        if (l3.v4->version == 4)
1178                l3.v4->check = 0;
1179
1180        /* tunnel packet */
1181        if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1182                                         SKB_GSO_GRE_CSUM |
1183                                         SKB_GSO_UDP_TUNNEL |
1184                                         SKB_GSO_UDP_TUNNEL_CSUM)) {
1185                /* reset l3&l4 pointers from outer to inner headers */
1186                l3.hdr = skb_inner_network_header(skb);
1187                l4.hdr = skb_inner_transport_header(skb);
1188
1189                /* Software should clear the IPv4's checksum field when
1190                 * tso is needed.
1191                 */
1192                if (l3.v4->version == 4)
1193                        l3.v4->check = 0;
1194        }
1195
1196        /* normal or tunnel packet */
1197        l4_offset = l4.hdr - skb->data;
1198
1199        /* remove payload length from inner pseudo checksum when tso */
1200        l4_paylen = skb->len - l4_offset;
1201
1202        if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
1203                hdr_len = sizeof(*l4.udp) + l4_offset;
1204                csum_replace_by_diff(&l4.udp->check,
1205                                     (__force __wsum)htonl(l4_paylen));
1206        } else {
1207                hdr_len = (l4.tcp->doff << 2) + l4_offset;
1208                csum_replace_by_diff(&l4.tcp->check,
1209                                     (__force __wsum)htonl(l4_paylen));
1210        }
1211
1212        *send_bytes = (skb_shinfo(skb)->gso_segs - 1) * hdr_len + skb->len;
1213
1214        /* find the txbd field values */
1215        *paylen_fdop_ol4cs = skb->len - hdr_len;
1216        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1);
1217
1218        /* offload outer UDP header checksum */
1219        if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
1220                hns3_set_field(*paylen_fdop_ol4cs, HNS3_TXD_OL4CS_B, 1);
1221
1222        /* get MSS for TSO */
1223        *mss = skb_shinfo(skb)->gso_size;
1224
1225        trace_hns3_tso(skb);
1226
1227        return 0;
1228}
1229
1230static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
1231                                u8 *il4_proto)
1232{
1233        union l3_hdr_info l3;
1234        unsigned char *l4_hdr;
1235        unsigned char *exthdr;
1236        u8 l4_proto_tmp;
1237        __be16 frag_off;
1238
1239        /* find outer header point */
1240        l3.hdr = skb_network_header(skb);
1241        l4_hdr = skb_transport_header(skb);
1242
1243        if (skb->protocol == htons(ETH_P_IPV6)) {
1244                exthdr = l3.hdr + sizeof(*l3.v6);
1245                l4_proto_tmp = l3.v6->nexthdr;
1246                if (l4_hdr != exthdr)
1247                        ipv6_skip_exthdr(skb, exthdr - skb->data,
1248                                         &l4_proto_tmp, &frag_off);
1249        } else if (skb->protocol == htons(ETH_P_IP)) {
1250                l4_proto_tmp = l3.v4->protocol;
1251        } else {
1252                return -EINVAL;
1253        }
1254
1255        *ol4_proto = l4_proto_tmp;
1256
1257        /* tunnel packet */
1258        if (!skb->encapsulation) {
1259                *il4_proto = 0;
1260                return 0;
1261        }
1262
1263        /* find inner header point */
1264        l3.hdr = skb_inner_network_header(skb);
1265        l4_hdr = skb_inner_transport_header(skb);
1266
1267        if (l3.v6->version == 6) {
1268                exthdr = l3.hdr + sizeof(*l3.v6);
1269                l4_proto_tmp = l3.v6->nexthdr;
1270                if (l4_hdr != exthdr)
1271                        ipv6_skip_exthdr(skb, exthdr - skb->data,
1272                                         &l4_proto_tmp, &frag_off);
1273        } else if (l3.v4->version == 4) {
1274                l4_proto_tmp = l3.v4->protocol;
1275        }
1276
1277        *il4_proto = l4_proto_tmp;
1278
1279        return 0;
1280}
1281
1282/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
1283 * and it is udp packet, which has a dest port as the IANA assigned.
1284 * the hardware is expected to do the checksum offload, but the
1285 * hardware will not do the checksum offload when udp dest port is
1286 * 4789, 4790 or 6081.
1287 */
1288static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
1289{
1290        struct hns3_nic_priv *priv = netdev_priv(skb->dev);
1291        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
1292        union l4_hdr_info l4;
1293
1294        /* device version above V3(include V3), the hardware can
1295         * do this checksum offload.
1296         */
1297        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
1298                return false;
1299
1300        l4.hdr = skb_transport_header(skb);
1301
1302        if (!(!skb->encapsulation &&
1303              (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) ||
1304              l4.udp->dest == htons(GENEVE_UDP_PORT) ||
1305              l4.udp->dest == htons(4790))))
1306                return false;
1307
1308        return true;
1309}
1310
1311static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
1312                                  u32 *ol_type_vlan_len_msec)
1313{
1314        u32 l2_len, l3_len, l4_len;
1315        unsigned char *il2_hdr;
1316        union l3_hdr_info l3;
1317        union l4_hdr_info l4;
1318
1319        l3.hdr = skb_network_header(skb);
1320        l4.hdr = skb_transport_header(skb);
1321
1322        /* compute OL2 header size, defined in 2 Bytes */
1323        l2_len = l3.hdr - skb->data;
1324        hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1);
1325
1326        /* compute OL3 header size, defined in 4 Bytes */
1327        l3_len = l4.hdr - l3.hdr;
1328        hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2);
1329
1330        il2_hdr = skb_inner_mac_header(skb);
1331        /* compute OL4 header size, defined in 4 Bytes */
1332        l4_len = il2_hdr - l4.hdr;
1333        hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2);
1334
1335        /* define outer network header type */
1336        if (skb->protocol == htons(ETH_P_IP)) {
1337                if (skb_is_gso(skb))
1338                        hns3_set_field(*ol_type_vlan_len_msec,
1339                                       HNS3_TXD_OL3T_S,
1340                                       HNS3_OL3T_IPV4_CSUM);
1341                else
1342                        hns3_set_field(*ol_type_vlan_len_msec,
1343                                       HNS3_TXD_OL3T_S,
1344                                       HNS3_OL3T_IPV4_NO_CSUM);
1345        } else if (skb->protocol == htons(ETH_P_IPV6)) {
1346                hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S,
1347                               HNS3_OL3T_IPV6);
1348        }
1349
1350        if (ol4_proto == IPPROTO_UDP)
1351                hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
1352                               HNS3_TUN_MAC_IN_UDP);
1353        else if (ol4_proto == IPPROTO_GRE)
1354                hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
1355                               HNS3_TUN_NVGRE);
1356}
1357
1358static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
1359                           u8 il4_proto, u32 *type_cs_vlan_tso,
1360                           u32 *ol_type_vlan_len_msec)
1361{
1362        unsigned char *l2_hdr = skb->data;
1363        u32 l4_proto = ol4_proto;
1364        union l4_hdr_info l4;
1365        union l3_hdr_info l3;
1366        u32 l2_len, l3_len;
1367
1368        l4.hdr = skb_transport_header(skb);
1369        l3.hdr = skb_network_header(skb);
1370
1371        /* handle encapsulation skb */
1372        if (skb->encapsulation) {
1373                /* If this is a not UDP/GRE encapsulation skb */
1374                if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {
1375                        /* drop the skb tunnel packet if hardware don't support,
1376                         * because hardware can't calculate csum when TSO.
1377                         */
1378                        if (skb_is_gso(skb))
1379                                return -EDOM;
1380
1381                        /* the stack computes the IP header already,
1382                         * driver calculate l4 checksum when not TSO.
1383                         */
1384                        return skb_checksum_help(skb);
1385                }
1386
1387                hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec);
1388
1389                /* switch to inner header */
1390                l2_hdr = skb_inner_mac_header(skb);
1391                l3.hdr = skb_inner_network_header(skb);
1392                l4.hdr = skb_inner_transport_header(skb);
1393                l4_proto = il4_proto;
1394        }
1395
1396        if (l3.v4->version == 4) {
1397                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
1398                               HNS3_L3T_IPV4);
1399
1400                /* the stack computes the IP header already, the only time we
1401                 * need the hardware to recompute it is in the case of TSO.
1402                 */
1403                if (skb_is_gso(skb))
1404                        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
1405        } else if (l3.v6->version == 6) {
1406                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
1407                               HNS3_L3T_IPV6);
1408        }
1409
1410        /* compute inner(/normal) L2 header size, defined in 2 Bytes */
1411        l2_len = l3.hdr - l2_hdr;
1412        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1);
1413
1414        /* compute inner(/normal) L3 header size, defined in 4 Bytes */
1415        l3_len = l4.hdr - l3.hdr;
1416        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2);
1417
1418        /* compute inner(/normal) L4 header size, defined in 4 Bytes */
1419        switch (l4_proto) {
1420        case IPPROTO_TCP:
1421                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
1422                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
1423                               HNS3_L4T_TCP);
1424                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
1425                               l4.tcp->doff);
1426                break;
1427        case IPPROTO_UDP:
1428                if (hns3_tunnel_csum_bug(skb))
1429                        return skb_checksum_help(skb);
1430
1431                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
1432                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
1433                               HNS3_L4T_UDP);
1434                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
1435                               (sizeof(struct udphdr) >> 2));
1436                break;
1437        case IPPROTO_SCTP:
1438                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
1439                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
1440                               HNS3_L4T_SCTP);
1441                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
1442                               (sizeof(struct sctphdr) >> 2));
1443                break;
1444        default:
1445                /* drop the skb tunnel packet if hardware don't support,
1446                 * because hardware can't calculate csum when TSO.
1447                 */
1448                if (skb_is_gso(skb))
1449                        return -EDOM;
1450
1451                /* the stack computes the IP header already,
1452                 * driver calculate l4 checksum when not TSO.
1453                 */
1454                return skb_checksum_help(skb);
1455        }
1456
1457        return 0;
1458}
1459
1460static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,
1461                             struct sk_buff *skb)
1462{
1463        struct hnae3_handle *handle = tx_ring->tqp->handle;
1464        struct hnae3_ae_dev *ae_dev;
1465        struct vlan_ethhdr *vhdr;
1466        int rc;
1467
1468        if (!(skb->protocol == htons(ETH_P_8021Q) ||
1469              skb_vlan_tag_present(skb)))
1470                return 0;
1471
1472        /* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert
1473         * VLAN enabled, only one VLAN header is allowed in skb, otherwise it
1474         * will cause RAS error.
1475         */
1476        ae_dev = pci_get_drvdata(handle->pdev);
1477        if (unlikely(skb_vlan_tagged_multi(skb) &&
1478                     ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2 &&
1479                     handle->port_base_vlan_state ==
1480                     HNAE3_PORT_BASE_VLAN_ENABLE))
1481                return -EINVAL;
1482
1483        if (skb->protocol == htons(ETH_P_8021Q) &&
1484            !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1485                /* When HW VLAN acceleration is turned off, and the stack
1486                 * sets the protocol to 802.1q, the driver just need to
1487                 * set the protocol to the encapsulated ethertype.
1488                 */
1489                skb->protocol = vlan_get_protocol(skb);
1490                return 0;
1491        }
1492
1493        if (skb_vlan_tag_present(skb)) {
1494                /* Based on hw strategy, use out_vtag in two layer tag case,
1495                 * and use inner_vtag in one tag case.
1496                 */
1497                if (skb->protocol == htons(ETH_P_8021Q) &&
1498                    handle->port_base_vlan_state ==
1499                    HNAE3_PORT_BASE_VLAN_DISABLE)
1500                        rc = HNS3_OUTER_VLAN_TAG;
1501                else
1502                        rc = HNS3_INNER_VLAN_TAG;
1503
1504                skb->protocol = vlan_get_protocol(skb);
1505                return rc;
1506        }
1507
1508        rc = skb_cow_head(skb, 0);
1509        if (unlikely(rc < 0))
1510                return rc;
1511
1512        vhdr = (struct vlan_ethhdr *)skb->data;
1513        vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)
1514                                         & VLAN_PRIO_MASK);
1515
1516        skb->protocol = vlan_get_protocol(skb);
1517        return 0;
1518}
1519
1520/* check if the hardware is capable of checksum offloading */
1521static bool hns3_check_hw_tx_csum(struct sk_buff *skb)
1522{
1523        struct hns3_nic_priv *priv = netdev_priv(skb->dev);
1524
1525        /* Kindly note, due to backward compatibility of the TX descriptor,
1526         * HW checksum of the non-IP packets and GSO packets is handled at
1527         * different place in the following code
1528         */
1529        if (skb_csum_is_sctp(skb) || skb_is_gso(skb) ||
1530            !test_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state))
1531                return false;
1532
1533        return true;
1534}
1535
1536static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,
1537                              struct sk_buff *skb, struct hns3_desc *desc,
1538                              struct hns3_desc_cb *desc_cb)
1539{
1540        u32 ol_type_vlan_len_msec = 0;
1541        u32 paylen_ol4cs = skb->len;
1542        u32 type_cs_vlan_tso = 0;
1543        u16 mss_hw_csum = 0;
1544        u16 inner_vtag = 0;
1545        u16 out_vtag = 0;
1546        int ret;
1547
1548        ret = hns3_handle_vtags(ring, skb);
1549        if (unlikely(ret < 0)) {
1550                u64_stats_update_begin(&ring->syncp);
1551                ring->stats.tx_vlan_err++;
1552                u64_stats_update_end(&ring->syncp);
1553                return ret;
1554        } else if (ret == HNS3_INNER_VLAN_TAG) {
1555                inner_vtag = skb_vlan_tag_get(skb);
1556                inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1557                                VLAN_PRIO_MASK;
1558                hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1);
1559        } else if (ret == HNS3_OUTER_VLAN_TAG) {
1560                out_vtag = skb_vlan_tag_get(skb);
1561                out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1562                                VLAN_PRIO_MASK;
1563                hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B,
1564                               1);
1565        }
1566
1567        desc_cb->send_bytes = skb->len;
1568
1569        if (skb->ip_summed == CHECKSUM_PARTIAL) {
1570                u8 ol4_proto, il4_proto;
1571
1572                if (hns3_check_hw_tx_csum(skb)) {
1573                        /* set checksum start and offset, defined in 2 Bytes */
1574                        hns3_set_field(type_cs_vlan_tso, HNS3_TXD_CSUM_START_S,
1575                                       skb_checksum_start_offset(skb) >> 1);
1576                        hns3_set_field(ol_type_vlan_len_msec,
1577                                       HNS3_TXD_CSUM_OFFSET_S,
1578                                       skb->csum_offset >> 1);
1579                        mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B);
1580                        goto out_hw_tx_csum;
1581                }
1582
1583                skb_reset_mac_len(skb);
1584
1585                ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1586                if (unlikely(ret < 0)) {
1587                        u64_stats_update_begin(&ring->syncp);
1588                        ring->stats.tx_l4_proto_err++;
1589                        u64_stats_update_end(&ring->syncp);
1590                        return ret;
1591                }
1592
1593                ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto,
1594                                      &type_cs_vlan_tso,
1595                                      &ol_type_vlan_len_msec);
1596                if (unlikely(ret < 0)) {
1597                        u64_stats_update_begin(&ring->syncp);
1598                        ring->stats.tx_l2l3l4_err++;
1599                        u64_stats_update_end(&ring->syncp);
1600                        return ret;
1601                }
1602
1603                ret = hns3_set_tso(skb, &paylen_ol4cs, &mss_hw_csum,
1604                                   &type_cs_vlan_tso, &desc_cb->send_bytes);
1605                if (unlikely(ret < 0)) {
1606                        u64_stats_update_begin(&ring->syncp);
1607                        ring->stats.tx_tso_err++;
1608                        u64_stats_update_end(&ring->syncp);
1609                        return ret;
1610                }
1611        }
1612
1613out_hw_tx_csum:
1614        /* Set txbd */
1615        desc->tx.ol_type_vlan_len_msec =
1616                cpu_to_le32(ol_type_vlan_len_msec);
1617        desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso);
1618        desc->tx.paylen_ol4cs = cpu_to_le32(paylen_ol4cs);
1619        desc->tx.mss_hw_csum = cpu_to_le16(mss_hw_csum);
1620        desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1621        desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1622
1623        return 0;
1624}
1625
1626static int hns3_fill_desc(struct hns3_enet_ring *ring, dma_addr_t dma,
1627                          unsigned int size)
1628{
1629#define HNS3_LIKELY_BD_NUM      1
1630
1631        struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1632        unsigned int frag_buf_num;
1633        int k, sizeoflast;
1634
1635        if (likely(size <= HNS3_MAX_BD_SIZE)) {
1636                desc->addr = cpu_to_le64(dma);
1637                desc->tx.send_size = cpu_to_le16(size);
1638                desc->tx.bdtp_fe_sc_vld_ra_ri =
1639                        cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1640
1641                trace_hns3_tx_desc(ring, ring->next_to_use);
1642                ring_ptr_move_fw(ring, next_to_use);
1643                return HNS3_LIKELY_BD_NUM;
1644        }
1645
1646        frag_buf_num = hns3_tx_bd_count(size);
1647        sizeoflast = size % HNS3_MAX_BD_SIZE;
1648        sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1649
1650        /* When frag size is bigger than hardware limit, split this frag */
1651        for (k = 0; k < frag_buf_num; k++) {
1652                /* now, fill the descriptor */
1653                desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1654                desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1655                                     (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1656                desc->tx.bdtp_fe_sc_vld_ra_ri =
1657                                cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1658
1659                trace_hns3_tx_desc(ring, ring->next_to_use);
1660                /* move ring pointer to next */
1661                ring_ptr_move_fw(ring, next_to_use);
1662
1663                desc = &ring->desc[ring->next_to_use];
1664        }
1665
1666        return frag_buf_num;
1667}
1668
1669static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv,
1670                                  unsigned int type)
1671{
1672        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1673        struct device *dev = ring_to_dev(ring);
1674        unsigned int size;
1675        dma_addr_t dma;
1676
1677        if (type & (DESC_TYPE_FRAGLIST_SKB | DESC_TYPE_SKB)) {
1678                struct sk_buff *skb = (struct sk_buff *)priv;
1679
1680                size = skb_headlen(skb);
1681                if (!size)
1682                        return 0;
1683
1684                dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1685        } else if (type & DESC_TYPE_BOUNCE_HEAD) {
1686                /* Head data has been filled in hns3_handle_tx_bounce(),
1687                 * just return 0 here.
1688                 */
1689                return 0;
1690        } else {
1691                skb_frag_t *frag = (skb_frag_t *)priv;
1692
1693                size = skb_frag_size(frag);
1694                if (!size)
1695                        return 0;
1696
1697                dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1698        }
1699
1700        if (unlikely(dma_mapping_error(dev, dma))) {
1701                u64_stats_update_begin(&ring->syncp);
1702                ring->stats.sw_err_cnt++;
1703                u64_stats_update_end(&ring->syncp);
1704                return -ENOMEM;
1705        }
1706
1707        desc_cb->priv = priv;
1708        desc_cb->length = size;
1709        desc_cb->dma = dma;
1710        desc_cb->type = type;
1711
1712        return hns3_fill_desc(ring, dma, size);
1713}
1714
1715static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1716                                    unsigned int bd_num)
1717{
1718        unsigned int size;
1719        int i;
1720
1721        size = skb_headlen(skb);
1722        while (size > HNS3_MAX_BD_SIZE) {
1723                bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1724                size -= HNS3_MAX_BD_SIZE;
1725
1726                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1727                        return bd_num;
1728        }
1729
1730        if (size) {
1731                bd_size[bd_num++] = size;
1732                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1733                        return bd_num;
1734        }
1735
1736        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1737                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1738                size = skb_frag_size(frag);
1739                if (!size)
1740                        continue;
1741
1742                while (size > HNS3_MAX_BD_SIZE) {
1743                        bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1744                        size -= HNS3_MAX_BD_SIZE;
1745
1746                        if (bd_num > HNS3_MAX_TSO_BD_NUM)
1747                                return bd_num;
1748                }
1749
1750                bd_size[bd_num++] = size;
1751                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1752                        return bd_num;
1753        }
1754
1755        return bd_num;
1756}
1757
1758static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1759                                   u8 max_non_tso_bd_num, unsigned int bd_num,
1760                                   unsigned int recursion_level)
1761{
1762#define HNS3_MAX_RECURSION_LEVEL        24
1763
1764        struct sk_buff *frag_skb;
1765
1766        /* If the total len is within the max bd limit */
1767        if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level &&
1768                   !skb_has_frag_list(skb) &&
1769                   skb_shinfo(skb)->nr_frags < max_non_tso_bd_num))
1770                return skb_shinfo(skb)->nr_frags + 1U;
1771
1772        if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL))
1773                return UINT_MAX;
1774
1775        bd_num = hns3_skb_bd_num(skb, bd_size, bd_num);
1776        if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)
1777                return bd_num;
1778
1779        skb_walk_frags(skb, frag_skb) {
1780                bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num,
1781                                        bd_num, recursion_level + 1);
1782                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1783                        return bd_num;
1784        }
1785
1786        return bd_num;
1787}
1788
1789static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)
1790{
1791        if (!skb->encapsulation)
1792                return skb_transport_offset(skb) + tcp_hdrlen(skb);
1793
1794        return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
1795}
1796
1797/* HW need every continuous max_non_tso_bd_num buffer data to be larger
1798 * than MSS, we simplify it by ensuring skb_headlen + the first continuous
1799 * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss,
1800 * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger
1801 * than MSS except the last max_non_tso_bd_num - 1 frags.
1802 */
1803static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,
1804                                     unsigned int bd_num, u8 max_non_tso_bd_num)
1805{
1806        unsigned int tot_len = 0;
1807        int i;
1808
1809        for (i = 0; i < max_non_tso_bd_num - 1U; i++)
1810                tot_len += bd_size[i];
1811
1812        /* ensure the first max_non_tso_bd_num frags is greater than
1813         * mss + header
1814         */
1815        if (tot_len + bd_size[max_non_tso_bd_num - 1U] <
1816            skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb))
1817                return true;
1818
1819        /* ensure every continuous max_non_tso_bd_num - 1 buffer is greater
1820         * than mss except the last one.
1821         */
1822        for (i = 0; i < bd_num - max_non_tso_bd_num; i++) {
1823                tot_len -= bd_size[i];
1824                tot_len += bd_size[i + max_non_tso_bd_num - 1U];
1825
1826                if (tot_len < skb_shinfo(skb)->gso_size)
1827                        return true;
1828        }
1829
1830        return false;
1831}
1832
1833void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)
1834{
1835        int i;
1836
1837        for (i = 0; i < MAX_SKB_FRAGS; i++)
1838                size[i] = skb_frag_size(&shinfo->frags[i]);
1839}
1840
1841static int hns3_skb_linearize(struct hns3_enet_ring *ring,
1842                              struct sk_buff *skb,
1843                              u8 max_non_tso_bd_num,
1844                              unsigned int bd_num)
1845{
1846        /* 'bd_num == UINT_MAX' means the skb' fraglist has a
1847         * recursion level of over HNS3_MAX_RECURSION_LEVEL.
1848         */
1849        if (bd_num == UINT_MAX) {
1850                u64_stats_update_begin(&ring->syncp);
1851                ring->stats.over_max_recursion++;
1852                u64_stats_update_end(&ring->syncp);
1853                return -ENOMEM;
1854        }
1855
1856        /* The skb->len has exceeded the hw limitation, linearization
1857         * will not help.
1858         */
1859        if (skb->len > HNS3_MAX_TSO_SIZE ||
1860            (!skb_is_gso(skb) && skb->len >
1861             HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num))) {
1862                u64_stats_update_begin(&ring->syncp);
1863                ring->stats.hw_limitation++;
1864                u64_stats_update_end(&ring->syncp);
1865                return -ENOMEM;
1866        }
1867
1868        if (__skb_linearize(skb)) {
1869                u64_stats_update_begin(&ring->syncp);
1870                ring->stats.sw_err_cnt++;
1871                u64_stats_update_end(&ring->syncp);
1872                return -ENOMEM;
1873        }
1874
1875        return 0;
1876}
1877
1878static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,
1879                                  struct net_device *netdev,
1880                                  struct sk_buff *skb)
1881{
1882        struct hns3_nic_priv *priv = netdev_priv(netdev);
1883        u8 max_non_tso_bd_num = priv->max_non_tso_bd_num;
1884        unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];
1885        unsigned int bd_num;
1886
1887        bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0);
1888        if (unlikely(bd_num > max_non_tso_bd_num)) {
1889                if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&
1890                    !hns3_skb_need_linearized(skb, bd_size, bd_num,
1891                                              max_non_tso_bd_num)) {
1892                        trace_hns3_over_max_bd(skb);
1893                        goto out;
1894                }
1895
1896                if (hns3_skb_linearize(ring, skb, max_non_tso_bd_num,
1897                                       bd_num))
1898                        return -ENOMEM;
1899
1900                bd_num = hns3_tx_bd_count(skb->len);
1901
1902                u64_stats_update_begin(&ring->syncp);
1903                ring->stats.tx_copy++;
1904                u64_stats_update_end(&ring->syncp);
1905        }
1906
1907out:
1908        if (likely(ring_space(ring) >= bd_num))
1909                return bd_num;
1910
1911        netif_stop_subqueue(netdev, ring->queue_index);
1912        smp_mb(); /* Memory barrier before checking ring_space */
1913
1914        /* Start queue in case hns3_clean_tx_ring has just made room
1915         * available and has not seen the queue stopped state performed
1916         * by netif_stop_subqueue above.
1917         */
1918        if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&
1919            !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
1920                netif_start_subqueue(netdev, ring->queue_index);
1921                return bd_num;
1922        }
1923
1924        u64_stats_update_begin(&ring->syncp);
1925        ring->stats.tx_busy++;
1926        u64_stats_update_end(&ring->syncp);
1927
1928        return -EBUSY;
1929}
1930
1931static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1932{
1933        struct device *dev = ring_to_dev(ring);
1934        unsigned int i;
1935
1936        for (i = 0; i < ring->desc_num; i++) {
1937                struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1938                struct hns3_desc_cb *desc_cb;
1939
1940                memset(desc, 0, sizeof(*desc));
1941
1942                /* check if this is where we started */
1943                if (ring->next_to_use == next_to_use_orig)
1944                        break;
1945
1946                /* rollback one */
1947                ring_ptr_move_bw(ring, next_to_use);
1948
1949                desc_cb = &ring->desc_cb[ring->next_to_use];
1950
1951                if (!desc_cb->dma)
1952                        continue;
1953
1954                /* unmap the descriptor dma address */
1955                if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))
1956                        dma_unmap_single(dev, desc_cb->dma, desc_cb->length,
1957                                         DMA_TO_DEVICE);
1958                else if (desc_cb->type &
1959                         (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL))
1960                        hns3_tx_spare_rollback(ring, desc_cb->length);
1961                else if (desc_cb->length)
1962                        dma_unmap_page(dev, desc_cb->dma, desc_cb->length,
1963                                       DMA_TO_DEVICE);
1964
1965                desc_cb->length = 0;
1966                desc_cb->dma = 0;
1967                desc_cb->type = DESC_TYPE_UNKNOWN;
1968        }
1969}
1970
1971static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,
1972                                 struct sk_buff *skb, unsigned int type)
1973{
1974        struct sk_buff *frag_skb;
1975        int i, ret, bd_num = 0;
1976
1977        ret = hns3_map_and_fill_desc(ring, skb, type);
1978        if (unlikely(ret < 0))
1979                return ret;
1980
1981        bd_num += ret;
1982
1983        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1984                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1985
1986                ret = hns3_map_and_fill_desc(ring, frag, DESC_TYPE_PAGE);
1987                if (unlikely(ret < 0))
1988                        return ret;
1989
1990                bd_num += ret;
1991        }
1992
1993        skb_walk_frags(skb, frag_skb) {
1994                ret = hns3_fill_skb_to_desc(ring, frag_skb,
1995                                            DESC_TYPE_FRAGLIST_SKB);
1996                if (unlikely(ret < 0))
1997                        return ret;
1998
1999                bd_num += ret;
2000        }
2001
2002        return bd_num;
2003}
2004
2005static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,
2006                             bool doorbell)
2007{
2008        ring->pending_buf += num;
2009
2010        if (!doorbell) {
2011                u64_stats_update_begin(&ring->syncp);
2012                ring->stats.tx_more++;
2013                u64_stats_update_end(&ring->syncp);
2014                return;
2015        }
2016
2017        if (!ring->pending_buf)
2018                return;
2019
2020        writel(ring->pending_buf,
2021               ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG);
2022        ring->pending_buf = 0;
2023        WRITE_ONCE(ring->last_to_use, ring->next_to_use);
2024}
2025
2026static void hns3_tsyn(struct net_device *netdev, struct sk_buff *skb,
2027                      struct hns3_desc *desc)
2028{
2029        struct hnae3_handle *h = hns3_get_handle(netdev);
2030
2031        if (!(h->ae_algo->ops->set_tx_hwts_info &&
2032              h->ae_algo->ops->set_tx_hwts_info(h, skb)))
2033                return;
2034
2035        desc->tx.bdtp_fe_sc_vld_ra_ri |= cpu_to_le16(BIT(HNS3_TXD_TSYN_B));
2036}
2037
2038static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring,
2039                                 struct sk_buff *skb)
2040{
2041        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
2042        unsigned int type = DESC_TYPE_BOUNCE_HEAD;
2043        unsigned int size = skb_headlen(skb);
2044        dma_addr_t dma;
2045        int bd_num = 0;
2046        u32 cb_len;
2047        void *buf;
2048        int ret;
2049
2050        if (skb->len <= ring->tx_copybreak) {
2051                size = skb->len;
2052                type = DESC_TYPE_BOUNCE_ALL;
2053        }
2054
2055        /* hns3_can_use_tx_bounce() is called to ensure the below
2056         * function can always return the tx buffer.
2057         */
2058        buf = hns3_tx_spare_alloc(ring, size, &dma, &cb_len);
2059
2060        ret = skb_copy_bits(skb, 0, buf, size);
2061        if (unlikely(ret < 0)) {
2062                hns3_tx_spare_rollback(ring, cb_len);
2063                u64_stats_update_begin(&ring->syncp);
2064                ring->stats.copy_bits_err++;
2065                u64_stats_update_end(&ring->syncp);
2066                return ret;
2067        }
2068
2069        desc_cb->priv = skb;
2070        desc_cb->length = cb_len;
2071        desc_cb->dma = dma;
2072        desc_cb->type = type;
2073
2074        bd_num += hns3_fill_desc(ring, dma, size);
2075
2076        if (type == DESC_TYPE_BOUNCE_HEAD) {
2077                ret = hns3_fill_skb_to_desc(ring, skb,
2078                                            DESC_TYPE_BOUNCE_HEAD);
2079                if (unlikely(ret < 0))
2080                        return ret;
2081
2082                bd_num += ret;
2083        }
2084
2085        dma_sync_single_for_device(ring_to_dev(ring), dma, size,
2086                                   DMA_TO_DEVICE);
2087
2088        u64_stats_update_begin(&ring->syncp);
2089        ring->stats.tx_bounce++;
2090        u64_stats_update_end(&ring->syncp);
2091        return bd_num;
2092}
2093
2094static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring,
2095                              struct sk_buff *skb)
2096{
2097        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
2098        u32 nfrag = skb_shinfo(skb)->nr_frags + 1;
2099        struct sg_table *sgt;
2100        int i, bd_num = 0;
2101        dma_addr_t dma;
2102        u32 cb_len;
2103        int nents;
2104
2105        if (skb_has_frag_list(skb))
2106                nfrag = HNS3_MAX_TSO_BD_NUM;
2107
2108        /* hns3_can_use_tx_sgl() is called to ensure the below
2109         * function can always return the tx buffer.
2110         */
2111        sgt = hns3_tx_spare_alloc(ring, HNS3_SGL_SIZE(nfrag),
2112                                  &dma, &cb_len);
2113
2114        /* scatterlist follows by the sg table */
2115        sgt->sgl = (struct scatterlist *)(sgt + 1);
2116        sg_init_table(sgt->sgl, nfrag);
2117        nents = skb_to_sgvec(skb, sgt->sgl, 0, skb->len);
2118        if (unlikely(nents < 0)) {
2119                hns3_tx_spare_rollback(ring, cb_len);
2120                u64_stats_update_begin(&ring->syncp);
2121                ring->stats.skb2sgl_err++;
2122                u64_stats_update_end(&ring->syncp);
2123                return -ENOMEM;
2124        }
2125
2126        sgt->orig_nents = nents;
2127        sgt->nents = dma_map_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents,
2128                                DMA_TO_DEVICE);
2129        if (unlikely(!sgt->nents)) {
2130                hns3_tx_spare_rollback(ring, cb_len);
2131                u64_stats_update_begin(&ring->syncp);
2132                ring->stats.map_sg_err++;
2133                u64_stats_update_end(&ring->syncp);
2134                return -ENOMEM;
2135        }
2136
2137        desc_cb->priv = skb;
2138        desc_cb->length = cb_len;
2139        desc_cb->dma = dma;
2140        desc_cb->type = DESC_TYPE_SGL_SKB;
2141
2142        for (i = 0; i < sgt->nents; i++)
2143                bd_num += hns3_fill_desc(ring, sg_dma_address(sgt->sgl + i),
2144                                         sg_dma_len(sgt->sgl + i));
2145
2146        u64_stats_update_begin(&ring->syncp);
2147        ring->stats.tx_sgl++;
2148        u64_stats_update_end(&ring->syncp);
2149
2150        return bd_num;
2151}
2152
2153static int hns3_handle_desc_filling(struct hns3_enet_ring *ring,
2154                                    struct sk_buff *skb)
2155{
2156        u32 space;
2157
2158        if (!ring->tx_spare)
2159                goto out;
2160
2161        space = hns3_tx_spare_space(ring);
2162
2163        if (hns3_can_use_tx_sgl(ring, skb, space))
2164                return hns3_handle_tx_sgl(ring, skb);
2165
2166        if (hns3_can_use_tx_bounce(ring, skb, space))
2167                return hns3_handle_tx_bounce(ring, skb);
2168
2169out:
2170        return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);
2171}
2172
2173netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
2174{
2175        struct hns3_nic_priv *priv = netdev_priv(netdev);
2176        struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];
2177        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
2178        struct netdev_queue *dev_queue;
2179        int pre_ntu, next_to_use_head;
2180        bool doorbell;
2181        int ret;
2182
2183        /* Hardware can only handle short frames above 32 bytes */
2184        if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {
2185                hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
2186
2187                u64_stats_update_begin(&ring->syncp);
2188                ring->stats.sw_err_cnt++;
2189                u64_stats_update_end(&ring->syncp);
2190
2191                return NETDEV_TX_OK;
2192        }
2193
2194        /* Prefetch the data used later */
2195        prefetch(skb->data);
2196
2197        ret = hns3_nic_maybe_stop_tx(ring, netdev, skb);
2198        if (unlikely(ret <= 0)) {
2199                if (ret == -EBUSY) {
2200                        hns3_tx_doorbell(ring, 0, true);
2201                        return NETDEV_TX_BUSY;
2202                }
2203
2204                hns3_rl_err(netdev, "xmit error: %d!\n", ret);
2205                goto out_err_tx_ok;
2206        }
2207
2208        next_to_use_head = ring->next_to_use;
2209
2210        ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use],
2211                                 desc_cb);
2212        if (unlikely(ret < 0))
2213                goto fill_err;
2214
2215        /* 'ret < 0' means filling error, 'ret == 0' means skb->len is
2216         * zero, which is unlikely, and 'ret > 0' means how many tx desc
2217         * need to be notified to the hw.
2218         */
2219        ret = hns3_handle_desc_filling(ring, skb);
2220        if (unlikely(ret <= 0))
2221                goto fill_err;
2222
2223        pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) :
2224                                        (ring->desc_num - 1);
2225
2226        if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
2227                hns3_tsyn(netdev, skb, &ring->desc[pre_ntu]);
2228
2229        ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |=
2230                                cpu_to_le16(BIT(HNS3_TXD_FE_B));
2231        trace_hns3_tx_desc(ring, pre_ntu);
2232
2233        skb_tx_timestamp(skb);
2234
2235        /* Complete translate all packets */
2236        dev_queue = netdev_get_tx_queue(netdev, ring->queue_index);
2237        doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes,
2238                                          netdev_xmit_more());
2239        hns3_tx_doorbell(ring, ret, doorbell);
2240
2241        return NETDEV_TX_OK;
2242
2243fill_err:
2244        hns3_clear_desc(ring, next_to_use_head);
2245
2246out_err_tx_ok:
2247        dev_kfree_skb_any(skb);
2248        hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
2249        return NETDEV_TX_OK;
2250}
2251
2252static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
2253{
2254        struct hnae3_handle *h = hns3_get_handle(netdev);
2255        struct sockaddr *mac_addr = p;
2256        int ret;
2257
2258        if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
2259                return -EADDRNOTAVAIL;
2260
2261        if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
2262                netdev_info(netdev, "already using mac address %pM\n",
2263                            mac_addr->sa_data);
2264                return 0;
2265        }
2266
2267        /* For VF device, if there is a perm_addr, then the user will not
2268         * be allowed to change the address.
2269         */
2270        if (!hns3_is_phys_func(h->pdev) &&
2271            !is_zero_ether_addr(netdev->perm_addr)) {
2272                netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n",
2273                           netdev->perm_addr, mac_addr->sa_data);
2274                return -EPERM;
2275        }
2276
2277        ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
2278        if (ret) {
2279                netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
2280                return ret;
2281        }
2282
2283        ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
2284
2285        return 0;
2286}
2287
2288static int hns3_nic_do_ioctl(struct net_device *netdev,
2289                             struct ifreq *ifr, int cmd)
2290{
2291        struct hnae3_handle *h = hns3_get_handle(netdev);
2292
2293        if (!netif_running(netdev))
2294                return -EINVAL;
2295
2296        if (!h->ae_algo->ops->do_ioctl)
2297                return -EOPNOTSUPP;
2298
2299        return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
2300}
2301
2302static int hns3_nic_set_features(struct net_device *netdev,
2303                                 netdev_features_t features)
2304{
2305        netdev_features_t changed = netdev->features ^ features;
2306        struct hns3_nic_priv *priv = netdev_priv(netdev);
2307        struct hnae3_handle *h = priv->ae_handle;
2308        bool enable;
2309        int ret;
2310
2311        if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
2312                enable = !!(features & NETIF_F_GRO_HW);
2313                ret = h->ae_algo->ops->set_gro_en(h, enable);
2314                if (ret)
2315                        return ret;
2316        }
2317
2318        if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
2319            h->ae_algo->ops->enable_hw_strip_rxvtag) {
2320                enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
2321                ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable);
2322                if (ret)
2323                        return ret;
2324        }
2325
2326        if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
2327                enable = !!(features & NETIF_F_NTUPLE);
2328                h->ae_algo->ops->enable_fd(h, enable);
2329        }
2330
2331        if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&
2332            h->ae_algo->ops->cls_flower_active(h)) {
2333                netdev_err(netdev,
2334                           "there are offloaded TC filters active, cannot disable HW TC offload");
2335                return -EINVAL;
2336        }
2337
2338        if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2339            h->ae_algo->ops->enable_vlan_filter) {
2340                enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER);
2341                ret = h->ae_algo->ops->enable_vlan_filter(h, enable);
2342                if (ret)
2343                        return ret;
2344        }
2345
2346        netdev->features = features;
2347        return 0;
2348}
2349
2350static netdev_features_t hns3_features_check(struct sk_buff *skb,
2351                                             struct net_device *dev,
2352                                             netdev_features_t features)
2353{
2354#define HNS3_MAX_HDR_LEN        480U
2355#define HNS3_MAX_L4_HDR_LEN     60U
2356
2357        size_t len;
2358
2359        if (skb->ip_summed != CHECKSUM_PARTIAL)
2360                return features;
2361
2362        if (skb->encapsulation)
2363                len = skb_inner_transport_header(skb) - skb->data;
2364        else
2365                len = skb_transport_header(skb) - skb->data;
2366
2367        /* Assume L4 is 60 byte as TCP is the only protocol with a
2368         * a flexible value, and it's max len is 60 bytes.
2369         */
2370        len += HNS3_MAX_L4_HDR_LEN;
2371
2372        /* Hardware only supports checksum on the skb with a max header
2373         * len of 480 bytes.
2374         */
2375        if (len > HNS3_MAX_HDR_LEN)
2376                features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2377
2378        return features;
2379}
2380
2381static void hns3_nic_get_stats64(struct net_device *netdev,
2382                                 struct rtnl_link_stats64 *stats)
2383{
2384        struct hns3_nic_priv *priv = netdev_priv(netdev);
2385        int queue_num = priv->ae_handle->kinfo.num_tqps;
2386        struct hnae3_handle *handle = priv->ae_handle;
2387        struct hns3_enet_ring *ring;
2388        u64 rx_length_errors = 0;
2389        u64 rx_crc_errors = 0;
2390        u64 rx_multicast = 0;
2391        unsigned int start;
2392        u64 tx_errors = 0;
2393        u64 rx_errors = 0;
2394        unsigned int idx;
2395        u64 tx_bytes = 0;
2396        u64 rx_bytes = 0;
2397        u64 tx_pkts = 0;
2398        u64 rx_pkts = 0;
2399        u64 tx_drop = 0;
2400        u64 rx_drop = 0;
2401
2402        if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
2403                return;
2404
2405        handle->ae_algo->ops->update_stats(handle, &netdev->stats);
2406
2407        for (idx = 0; idx < queue_num; idx++) {
2408                /* fetch the tx stats */
2409                ring = &priv->ring[idx];
2410                do {
2411                        start = u64_stats_fetch_begin_irq(&ring->syncp);
2412                        tx_bytes += ring->stats.tx_bytes;
2413                        tx_pkts += ring->stats.tx_pkts;
2414                        tx_drop += ring->stats.sw_err_cnt;
2415                        tx_drop += ring->stats.tx_vlan_err;
2416                        tx_drop += ring->stats.tx_l4_proto_err;
2417                        tx_drop += ring->stats.tx_l2l3l4_err;
2418                        tx_drop += ring->stats.tx_tso_err;
2419                        tx_drop += ring->stats.over_max_recursion;
2420                        tx_drop += ring->stats.hw_limitation;
2421                        tx_drop += ring->stats.copy_bits_err;
2422                        tx_drop += ring->stats.skb2sgl_err;
2423                        tx_drop += ring->stats.map_sg_err;
2424                        tx_errors += ring->stats.sw_err_cnt;
2425                        tx_errors += ring->stats.tx_vlan_err;
2426                        tx_errors += ring->stats.tx_l4_proto_err;
2427                        tx_errors += ring->stats.tx_l2l3l4_err;
2428                        tx_errors += ring->stats.tx_tso_err;
2429                        tx_errors += ring->stats.over_max_recursion;
2430                        tx_errors += ring->stats.hw_limitation;
2431                        tx_errors += ring->stats.copy_bits_err;
2432                        tx_errors += ring->stats.skb2sgl_err;
2433                        tx_errors += ring->stats.map_sg_err;
2434                } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
2435
2436                /* fetch the rx stats */
2437                ring = &priv->ring[idx + queue_num];
2438                do {
2439                        start = u64_stats_fetch_begin_irq(&ring->syncp);
2440                        rx_bytes += ring->stats.rx_bytes;
2441                        rx_pkts += ring->stats.rx_pkts;
2442                        rx_drop += ring->stats.l2_err;
2443                        rx_errors += ring->stats.l2_err;
2444                        rx_errors += ring->stats.l3l4_csum_err;
2445                        rx_crc_errors += ring->stats.l2_err;
2446                        rx_multicast += ring->stats.rx_multicast;
2447                        rx_length_errors += ring->stats.err_pkt_len;
2448                } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
2449        }
2450
2451        stats->tx_bytes = tx_bytes;
2452        stats->tx_packets = tx_pkts;
2453        stats->rx_bytes = rx_bytes;
2454        stats->rx_packets = rx_pkts;
2455
2456        stats->rx_errors = rx_errors;
2457        stats->multicast = rx_multicast;
2458        stats->rx_length_errors = rx_length_errors;
2459        stats->rx_crc_errors = rx_crc_errors;
2460        stats->rx_missed_errors = netdev->stats.rx_missed_errors;
2461
2462        stats->tx_errors = tx_errors;
2463        stats->rx_dropped = rx_drop;
2464        stats->tx_dropped = tx_drop;
2465        stats->collisions = netdev->stats.collisions;
2466        stats->rx_over_errors = netdev->stats.rx_over_errors;
2467        stats->rx_frame_errors = netdev->stats.rx_frame_errors;
2468        stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
2469        stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
2470        stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
2471        stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
2472        stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
2473        stats->tx_window_errors = netdev->stats.tx_window_errors;
2474        stats->rx_compressed = netdev->stats.rx_compressed;
2475        stats->tx_compressed = netdev->stats.tx_compressed;
2476}
2477
2478static int hns3_setup_tc(struct net_device *netdev, void *type_data)
2479{
2480        struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
2481        struct hnae3_knic_private_info *kinfo;
2482        u8 tc = mqprio_qopt->qopt.num_tc;
2483        u16 mode = mqprio_qopt->mode;
2484        u8 hw = mqprio_qopt->qopt.hw;
2485        struct hnae3_handle *h;
2486
2487        if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
2488               mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
2489                return -EOPNOTSUPP;
2490
2491        if (tc > HNAE3_MAX_TC)
2492                return -EINVAL;
2493
2494        if (!netdev)
2495                return -EINVAL;
2496
2497        h = hns3_get_handle(netdev);
2498        kinfo = &h->kinfo;
2499
2500        netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc);
2501
2502        return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
2503                kinfo->dcb_ops->setup_tc(h, mqprio_qopt) : -EOPNOTSUPP;
2504}
2505
2506static int hns3_setup_tc_cls_flower(struct hns3_nic_priv *priv,
2507                                    struct flow_cls_offload *flow)
2508{
2509        int tc = tc_classid_to_hwtc(priv->netdev, flow->classid);
2510        struct hnae3_handle *h = hns3_get_handle(priv->netdev);
2511
2512        switch (flow->command) {
2513        case FLOW_CLS_REPLACE:
2514                if (h->ae_algo->ops->add_cls_flower)
2515                        return h->ae_algo->ops->add_cls_flower(h, flow, tc);
2516                break;
2517        case FLOW_CLS_DESTROY:
2518                if (h->ae_algo->ops->del_cls_flower)
2519                        return h->ae_algo->ops->del_cls_flower(h, flow);
2520                break;
2521        default:
2522                break;
2523        }
2524
2525        return -EOPNOTSUPP;
2526}
2527
2528static int hns3_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2529                                  void *cb_priv)
2530{
2531        struct hns3_nic_priv *priv = cb_priv;
2532
2533        if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
2534                return -EOPNOTSUPP;
2535
2536        switch (type) {
2537        case TC_SETUP_CLSFLOWER:
2538                return hns3_setup_tc_cls_flower(priv, type_data);
2539        default:
2540                return -EOPNOTSUPP;
2541        }
2542}
2543
2544static LIST_HEAD(hns3_block_cb_list);
2545
2546static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
2547                             void *type_data)
2548{
2549        struct hns3_nic_priv *priv = netdev_priv(dev);
2550        int ret;
2551
2552        switch (type) {
2553        case TC_SETUP_QDISC_MQPRIO:
2554                ret = hns3_setup_tc(dev, type_data);
2555                break;
2556        case TC_SETUP_BLOCK:
2557                ret = flow_block_cb_setup_simple(type_data,
2558                                                 &hns3_block_cb_list,
2559                                                 hns3_setup_tc_block_cb,
2560                                                 priv, priv, true);
2561                break;
2562        default:
2563                return -EOPNOTSUPP;
2564        }
2565
2566        return ret;
2567}
2568
2569static int hns3_vlan_rx_add_vid(struct net_device *netdev,
2570                                __be16 proto, u16 vid)
2571{
2572        struct hnae3_handle *h = hns3_get_handle(netdev);
2573        int ret = -EIO;
2574
2575        if (h->ae_algo->ops->set_vlan_filter)
2576                ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
2577
2578        return ret;
2579}
2580
2581static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
2582                                 __be16 proto, u16 vid)
2583{
2584        struct hnae3_handle *h = hns3_get_handle(netdev);
2585        int ret = -EIO;
2586
2587        if (h->ae_algo->ops->set_vlan_filter)
2588                ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
2589
2590        return ret;
2591}
2592
2593static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2594                                u8 qos, __be16 vlan_proto)
2595{
2596        struct hnae3_handle *h = hns3_get_handle(netdev);
2597        int ret = -EIO;
2598
2599        netif_dbg(h, drv, netdev,
2600                  "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",
2601                  vf, vlan, qos, ntohs(vlan_proto));
2602
2603        if (h->ae_algo->ops->set_vf_vlan_filter)
2604                ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
2605                                                          qos, vlan_proto);
2606
2607        return ret;
2608}
2609
2610static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
2611{
2612        struct hnae3_handle *handle = hns3_get_handle(netdev);
2613
2614        if (hns3_nic_resetting(netdev))
2615                return -EBUSY;
2616
2617        if (!handle->ae_algo->ops->set_vf_spoofchk)
2618                return -EOPNOTSUPP;
2619
2620        return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable);
2621}
2622
2623static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable)
2624{
2625        struct hnae3_handle *handle = hns3_get_handle(netdev);
2626
2627        if (!handle->ae_algo->ops->set_vf_trust)
2628                return -EOPNOTSUPP;
2629
2630        return handle->ae_algo->ops->set_vf_trust(handle, vf, enable);
2631}
2632
2633static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
2634{
2635        struct hnae3_handle *h = hns3_get_handle(netdev);
2636        int ret;
2637
2638        if (hns3_nic_resetting(netdev))
2639                return -EBUSY;
2640
2641        if (!h->ae_algo->ops->set_mtu)
2642                return -EOPNOTSUPP;
2643
2644        netif_dbg(h, drv, netdev,
2645                  "change mtu from %u to %d\n", netdev->mtu, new_mtu);
2646
2647        ret = h->ae_algo->ops->set_mtu(h, new_mtu);
2648        if (ret)
2649                netdev_err(netdev, "failed to change MTU in hardware %d\n",
2650                           ret);
2651        else
2652                netdev->mtu = new_mtu;
2653
2654        return ret;
2655}
2656
2657static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
2658{
2659        struct hns3_nic_priv *priv = netdev_priv(ndev);
2660        struct hnae3_handle *h = hns3_get_handle(ndev);
2661        struct hns3_enet_ring *tx_ring;
2662        struct napi_struct *napi;
2663        int timeout_queue = 0;
2664        int hw_head, hw_tail;
2665        int fbd_num, fbd_oft;
2666        int ebd_num, ebd_oft;
2667        int bd_num, bd_err;
2668        int ring_en, tc;
2669        int i;
2670
2671        /* Find the stopped queue the same way the stack does */
2672        for (i = 0; i < ndev->num_tx_queues; i++) {
2673                struct netdev_queue *q;
2674                unsigned long trans_start;
2675
2676                q = netdev_get_tx_queue(ndev, i);
2677                trans_start = q->trans_start;
2678                if (netif_xmit_stopped(q) &&
2679                    time_after(jiffies,
2680                               (trans_start + ndev->watchdog_timeo))) {
2681                        timeout_queue = i;
2682                        netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",
2683                                    q->state,
2684                                    jiffies_to_msecs(jiffies - trans_start));
2685                        break;
2686                }
2687        }
2688
2689        if (i == ndev->num_tx_queues) {
2690                netdev_info(ndev,
2691                            "no netdev TX timeout queue found, timeout count: %llu\n",
2692                            priv->tx_timeout_count);
2693                return false;
2694        }
2695
2696        priv->tx_timeout_count++;
2697
2698        tx_ring = &priv->ring[timeout_queue];
2699        napi = &tx_ring->tqp_vector->napi;
2700
2701        netdev_info(ndev,
2702                    "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",
2703                    priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use,
2704                    tx_ring->next_to_clean, napi->state);
2705
2706        netdev_info(ndev,
2707                    "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n",
2708                    tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes,
2709                    tx_ring->stats.sw_err_cnt, tx_ring->pending_buf);
2710
2711        netdev_info(ndev,
2712                    "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n",
2713                    tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more,
2714                    tx_ring->stats.restart_queue, tx_ring->stats.tx_busy);
2715
2716        /* When mac received many pause frames continuous, it's unable to send
2717         * packets, which may cause tx timeout
2718         */
2719        if (h->ae_algo->ops->get_mac_stats) {
2720                struct hns3_mac_stats mac_stats;
2721
2722                h->ae_algo->ops->get_mac_stats(h, &mac_stats);
2723                netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",
2724                            mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt);
2725        }
2726
2727        hw_head = readl_relaxed(tx_ring->tqp->io_base +
2728                                HNS3_RING_TX_RING_HEAD_REG);
2729        hw_tail = readl_relaxed(tx_ring->tqp->io_base +
2730                                HNS3_RING_TX_RING_TAIL_REG);
2731        fbd_num = readl_relaxed(tx_ring->tqp->io_base +
2732                                HNS3_RING_TX_RING_FBDNUM_REG);
2733        fbd_oft = readl_relaxed(tx_ring->tqp->io_base +
2734                                HNS3_RING_TX_RING_OFFSET_REG);
2735        ebd_num = readl_relaxed(tx_ring->tqp->io_base +
2736                                HNS3_RING_TX_RING_EBDNUM_REG);
2737        ebd_oft = readl_relaxed(tx_ring->tqp->io_base +
2738                                HNS3_RING_TX_RING_EBD_OFFSET_REG);
2739        bd_num = readl_relaxed(tx_ring->tqp->io_base +
2740                               HNS3_RING_TX_RING_BD_NUM_REG);
2741        bd_err = readl_relaxed(tx_ring->tqp->io_base +
2742                               HNS3_RING_TX_RING_BD_ERR_REG);
2743        ring_en = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_EN_REG);
2744        tc = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_TX_RING_TC_REG);
2745
2746        netdev_info(ndev,
2747                    "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",
2748                    bd_num, hw_head, hw_tail, bd_err,
2749                    readl(tx_ring->tqp_vector->mask_addr));
2750        netdev_info(ndev,
2751                    "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",
2752                    ring_en, tc, fbd_num, fbd_oft, ebd_num, ebd_oft);
2753
2754        return true;
2755}
2756
2757static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)
2758{
2759        struct hns3_nic_priv *priv = netdev_priv(ndev);
2760        struct hnae3_handle *h = priv->ae_handle;
2761
2762        if (!hns3_get_tx_timeo_queue_info(ndev))
2763                return;
2764
2765        /* request the reset, and let the hclge to determine
2766         * which reset level should be done
2767         */
2768        if (h->ae_algo->ops->reset_event)
2769                h->ae_algo->ops->reset_event(h->pdev, h);
2770}
2771
2772#ifdef CONFIG_RFS_ACCEL
2773static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
2774                              u16 rxq_index, u32 flow_id)
2775{
2776        struct hnae3_handle *h = hns3_get_handle(dev);
2777        struct flow_keys fkeys;
2778
2779        if (!h->ae_algo->ops->add_arfs_entry)
2780                return -EOPNOTSUPP;
2781
2782        if (skb->encapsulation)
2783                return -EPROTONOSUPPORT;
2784
2785        if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0))
2786                return -EPROTONOSUPPORT;
2787
2788        if ((fkeys.basic.n_proto != htons(ETH_P_IP) &&
2789             fkeys.basic.n_proto != htons(ETH_P_IPV6)) ||
2790            (fkeys.basic.ip_proto != IPPROTO_TCP &&
2791             fkeys.basic.ip_proto != IPPROTO_UDP))
2792                return -EPROTONOSUPPORT;
2793
2794        return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys);
2795}
2796#endif
2797
2798static int hns3_nic_get_vf_config(struct net_device *ndev, int vf,
2799                                  struct ifla_vf_info *ivf)
2800{
2801        struct hnae3_handle *h = hns3_get_handle(ndev);
2802
2803        if (!h->ae_algo->ops->get_vf_config)
2804                return -EOPNOTSUPP;
2805
2806        return h->ae_algo->ops->get_vf_config(h, vf, ivf);
2807}
2808
2809static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf,
2810                                      int link_state)
2811{
2812        struct hnae3_handle *h = hns3_get_handle(ndev);
2813
2814        if (!h->ae_algo->ops->set_vf_link_state)
2815                return -EOPNOTSUPP;
2816
2817        return h->ae_algo->ops->set_vf_link_state(h, vf, link_state);
2818}
2819
2820static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,
2821                                int min_tx_rate, int max_tx_rate)
2822{
2823        struct hnae3_handle *h = hns3_get_handle(ndev);
2824
2825        if (!h->ae_algo->ops->set_vf_rate)
2826                return -EOPNOTSUPP;
2827
2828        return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate,
2829                                            false);
2830}
2831
2832static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
2833{
2834        struct hnae3_handle *h = hns3_get_handle(netdev);
2835
2836        if (!h->ae_algo->ops->set_vf_mac)
2837                return -EOPNOTSUPP;
2838
2839        if (is_multicast_ether_addr(mac)) {
2840                netdev_err(netdev,
2841                           "Invalid MAC:%pM specified. Could not set MAC\n",
2842                           mac);
2843                return -EINVAL;
2844        }
2845
2846        return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);
2847}
2848
2849static const struct net_device_ops hns3_nic_netdev_ops = {
2850        .ndo_open               = hns3_nic_net_open,
2851        .ndo_stop               = hns3_nic_net_stop,
2852        .ndo_start_xmit         = hns3_nic_net_xmit,
2853        .ndo_tx_timeout         = hns3_nic_net_timeout,
2854        .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
2855        .ndo_do_ioctl           = hns3_nic_do_ioctl,
2856        .ndo_change_mtu         = hns3_nic_change_mtu,
2857        .ndo_set_features       = hns3_nic_set_features,
2858        .ndo_features_check     = hns3_features_check,
2859        .ndo_get_stats64        = hns3_nic_get_stats64,
2860        .ndo_setup_tc           = hns3_nic_setup_tc,
2861        .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
2862        .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
2863        .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
2864        .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
2865        .ndo_set_vf_spoofchk    = hns3_set_vf_spoofchk,
2866        .ndo_set_vf_trust       = hns3_set_vf_trust,
2867#ifdef CONFIG_RFS_ACCEL
2868        .ndo_rx_flow_steer      = hns3_rx_flow_steer,
2869#endif
2870        .ndo_get_vf_config      = hns3_nic_get_vf_config,
2871        .ndo_set_vf_link_state  = hns3_nic_set_vf_link_state,
2872        .ndo_set_vf_rate        = hns3_nic_set_vf_rate,
2873        .ndo_set_vf_mac         = hns3_nic_set_vf_mac,
2874};
2875
2876bool hns3_is_phys_func(struct pci_dev *pdev)
2877{
2878        u32 dev_id = pdev->device;
2879
2880        switch (dev_id) {
2881        case HNAE3_DEV_ID_GE:
2882        case HNAE3_DEV_ID_25GE:
2883        case HNAE3_DEV_ID_25GE_RDMA:
2884        case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
2885        case HNAE3_DEV_ID_50GE_RDMA:
2886        case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
2887        case HNAE3_DEV_ID_100G_RDMA_MACSEC:
2888        case HNAE3_DEV_ID_200G_RDMA:
2889                return true;
2890        case HNAE3_DEV_ID_VF:
2891        case HNAE3_DEV_ID_RDMA_DCB_PFC_VF:
2892                return false;
2893        default:
2894                dev_warn(&pdev->dev, "un-recognized pci device-id %u",
2895                         dev_id);
2896        }
2897
2898        return false;
2899}
2900
2901static void hns3_disable_sriov(struct pci_dev *pdev)
2902{
2903        /* If our VFs are assigned we cannot shut down SR-IOV
2904         * without causing issues, so just leave the hardware
2905         * available but disabled
2906         */
2907        if (pci_vfs_assigned(pdev)) {
2908                dev_warn(&pdev->dev,
2909                         "disabling driver while VFs are assigned\n");
2910                return;
2911        }
2912
2913        pci_disable_sriov(pdev);
2914}
2915
2916/* hns3_probe - Device initialization routine
2917 * @pdev: PCI device information struct
2918 * @ent: entry in hns3_pci_tbl
2919 *
2920 * hns3_probe initializes a PF identified by a pci_dev structure.
2921 * The OS initialization, configuring of the PF private structure,
2922 * and a hardware reset occur.
2923 *
2924 * Returns 0 on success, negative on failure
2925 */
2926static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2927{
2928        struct hnae3_ae_dev *ae_dev;
2929        int ret;
2930
2931        ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL);
2932        if (!ae_dev)
2933                return -ENOMEM;
2934
2935        ae_dev->pdev = pdev;
2936        ae_dev->flag = ent->driver_data;
2937        pci_set_drvdata(pdev, ae_dev);
2938
2939        ret = hnae3_register_ae_dev(ae_dev);
2940        if (ret)
2941                pci_set_drvdata(pdev, NULL);
2942
2943        return ret;
2944}
2945
2946/* hns3_remove - Device removal routine
2947 * @pdev: PCI device information struct
2948 */
2949static void hns3_remove(struct pci_dev *pdev)
2950{
2951        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2952
2953        if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
2954                hns3_disable_sriov(pdev);
2955
2956        hnae3_unregister_ae_dev(ae_dev);
2957        pci_set_drvdata(pdev, NULL);
2958}
2959
2960/**
2961 * hns3_pci_sriov_configure
2962 * @pdev: pointer to a pci_dev structure
2963 * @num_vfs: number of VFs to allocate
2964 *
2965 * Enable or change the number of VFs. Called when the user updates the number
2966 * of VFs in sysfs.
2967 **/
2968static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
2969{
2970        int ret;
2971
2972        if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
2973                dev_warn(&pdev->dev, "Can not config SRIOV\n");
2974                return -EINVAL;
2975        }
2976
2977        if (num_vfs) {
2978                ret = pci_enable_sriov(pdev, num_vfs);
2979                if (ret)
2980                        dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
2981                else
2982                        return num_vfs;
2983        } else if (!pci_vfs_assigned(pdev)) {
2984                pci_disable_sriov(pdev);
2985        } else {
2986                dev_warn(&pdev->dev,
2987                         "Unable to free VFs because some are assigned to VMs.\n");
2988        }
2989
2990        return 0;
2991}
2992
2993static void hns3_shutdown(struct pci_dev *pdev)
2994{
2995        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2996
2997        hnae3_unregister_ae_dev(ae_dev);
2998        pci_set_drvdata(pdev, NULL);
2999
3000        if (system_state == SYSTEM_POWER_OFF)
3001                pci_set_power_state(pdev, PCI_D3hot);
3002}
3003
3004static int __maybe_unused hns3_suspend(struct device *dev)
3005{
3006        struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
3007
3008        if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
3009                dev_info(dev, "Begin to suspend.\n");
3010                if (ae_dev->ops && ae_dev->ops->reset_prepare)
3011                        ae_dev->ops->reset_prepare(ae_dev, HNAE3_FUNC_RESET);
3012        }
3013
3014        return 0;
3015}
3016
3017static int __maybe_unused hns3_resume(struct device *dev)
3018{
3019        struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
3020
3021        if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
3022                dev_info(dev, "Begin to resume.\n");
3023                if (ae_dev->ops && ae_dev->ops->reset_done)
3024                        ae_dev->ops->reset_done(ae_dev);
3025        }
3026
3027        return 0;
3028}
3029
3030static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
3031                                            pci_channel_state_t state)
3032{
3033        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3034        pci_ers_result_t ret;
3035
3036        dev_info(&pdev->dev, "PCI error detected, state(=%u)!!\n", state);
3037
3038        if (state == pci_channel_io_perm_failure)
3039                return PCI_ERS_RESULT_DISCONNECT;
3040
3041        if (!ae_dev || !ae_dev->ops) {
3042                dev_err(&pdev->dev,
3043                        "Can't recover - error happened before device initialized\n");
3044                return PCI_ERS_RESULT_NONE;
3045        }
3046
3047        if (ae_dev->ops->handle_hw_ras_error)
3048                ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
3049        else
3050                return PCI_ERS_RESULT_NONE;
3051
3052        return ret;
3053}
3054
3055static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
3056{
3057        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3058        const struct hnae3_ae_ops *ops;
3059        enum hnae3_reset_type reset_type;
3060        struct device *dev = &pdev->dev;
3061
3062        if (!ae_dev || !ae_dev->ops)
3063                return PCI_ERS_RESULT_NONE;
3064
3065        ops = ae_dev->ops;
3066        /* request the reset */
3067        if (ops->reset_event && ops->get_reset_level &&
3068            ops->set_default_reset_request) {
3069                if (ae_dev->hw_err_reset_req) {
3070                        reset_type = ops->get_reset_level(ae_dev,
3071                                                &ae_dev->hw_err_reset_req);
3072                        ops->set_default_reset_request(ae_dev, reset_type);
3073                        dev_info(dev, "requesting reset due to PCI error\n");
3074                        ops->reset_event(pdev, NULL);
3075                }
3076
3077                return PCI_ERS_RESULT_RECOVERED;
3078        }
3079
3080        return PCI_ERS_RESULT_DISCONNECT;
3081}
3082
3083static void hns3_reset_prepare(struct pci_dev *pdev)
3084{
3085        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3086
3087        dev_info(&pdev->dev, "FLR prepare\n");
3088        if (ae_dev && ae_dev->ops && ae_dev->ops->reset_prepare)
3089                ae_dev->ops->reset_prepare(ae_dev, HNAE3_FLR_RESET);
3090}
3091
3092static void hns3_reset_done(struct pci_dev *pdev)
3093{
3094        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3095
3096        dev_info(&pdev->dev, "FLR done\n");
3097        if (ae_dev && ae_dev->ops && ae_dev->ops->reset_done)
3098                ae_dev->ops->reset_done(ae_dev);
3099}
3100
3101static const struct pci_error_handlers hns3_err_handler = {
3102        .error_detected = hns3_error_detected,
3103        .slot_reset     = hns3_slot_reset,
3104        .reset_prepare  = hns3_reset_prepare,
3105        .reset_done     = hns3_reset_done,
3106};
3107
3108static SIMPLE_DEV_PM_OPS(hns3_pm_ops, hns3_suspend, hns3_resume);
3109
3110static struct pci_driver hns3_driver = {
3111        .name     = hns3_driver_name,
3112        .id_table = hns3_pci_tbl,
3113        .probe    = hns3_probe,
3114        .remove   = hns3_remove,
3115        .shutdown = hns3_shutdown,
3116        .driver.pm  = &hns3_pm_ops,
3117        .sriov_configure = hns3_pci_sriov_configure,
3118        .err_handler    = &hns3_err_handler,
3119};
3120
3121/* set default feature to hns3 */
3122static void hns3_set_default_feature(struct net_device *netdev)
3123{
3124        struct hnae3_handle *h = hns3_get_handle(netdev);
3125        struct pci_dev *pdev = h->pdev;
3126        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3127
3128        netdev->priv_flags |= IFF_UNICAST_FLT;
3129
3130        netdev->hw_enc_features |= NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
3131                NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
3132                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
3133                NETIF_F_SCTP_CRC | NETIF_F_TSO_MANGLEID | NETIF_F_FRAGLIST;
3134
3135        netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
3136
3137        netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
3138                NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
3139                NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
3140                NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
3141                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
3142                NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
3143
3144        netdev->vlan_features |= NETIF_F_RXCSUM |
3145                NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
3146                NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
3147                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
3148                NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
3149
3150        netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
3151                NETIF_F_HW_VLAN_CTAG_RX |
3152                NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
3153                NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
3154                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
3155                NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
3156
3157        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) {
3158                netdev->hw_features |= NETIF_F_GRO_HW;
3159                netdev->features |= NETIF_F_GRO_HW;
3160
3161                if (!(h->flags & HNAE3_SUPPORT_VF)) {
3162                        netdev->hw_features |= NETIF_F_NTUPLE;
3163                        netdev->features |= NETIF_F_NTUPLE;
3164                }
3165        }
3166
3167        if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps)) {
3168                netdev->hw_features |= NETIF_F_GSO_UDP_L4;
3169                netdev->features |= NETIF_F_GSO_UDP_L4;
3170                netdev->vlan_features |= NETIF_F_GSO_UDP_L4;
3171                netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
3172        }
3173
3174        if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps)) {
3175                netdev->hw_features |= NETIF_F_HW_CSUM;
3176                netdev->features |= NETIF_F_HW_CSUM;
3177                netdev->vlan_features |= NETIF_F_HW_CSUM;
3178                netdev->hw_enc_features |= NETIF_F_HW_CSUM;
3179        } else {
3180                netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3181                netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3182                netdev->vlan_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3183                netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3184        }
3185
3186        if (test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps)) {
3187                netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
3188                netdev->features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
3189                netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
3190                netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
3191        }
3192
3193        if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) {
3194                netdev->hw_features |= NETIF_F_HW_TC;
3195                netdev->features |= NETIF_F_HW_TC;
3196        }
3197
3198        if (test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps))
3199                netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3200}
3201
3202static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
3203                             struct hns3_desc_cb *cb)
3204{
3205        unsigned int order = hns3_page_order(ring);
3206        struct page *p;
3207
3208        p = dev_alloc_pages(order);
3209        if (!p)
3210                return -ENOMEM;
3211
3212        cb->priv = p;
3213        cb->page_offset = 0;
3214        cb->reuse_flag = 0;
3215        cb->buf  = page_address(p);
3216        cb->length = hns3_page_size(ring);
3217        cb->type = DESC_TYPE_PAGE;
3218        page_ref_add(p, USHRT_MAX - 1);
3219        cb->pagecnt_bias = USHRT_MAX;
3220
3221        return 0;
3222}
3223
3224static void hns3_free_buffer(struct hns3_enet_ring *ring,
3225                             struct hns3_desc_cb *cb, int budget)
3226{
3227        if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_HEAD |
3228                        DESC_TYPE_BOUNCE_ALL | DESC_TYPE_SGL_SKB))
3229                napi_consume_skb(cb->priv, budget);
3230        else if (!HNAE3_IS_TX_RING(ring) && cb->pagecnt_bias)
3231                __page_frag_cache_drain(cb->priv, cb->pagecnt_bias);
3232        memset(cb, 0, sizeof(*cb));
3233}
3234
3235static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
3236{
3237        cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
3238                               cb->length, ring_to_dma_dir(ring));
3239
3240        if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
3241                return -EIO;
3242
3243        return 0;
3244}
3245
3246static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
3247                              struct hns3_desc_cb *cb)
3248{
3249        if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))
3250                dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
3251                                 ring_to_dma_dir(ring));
3252        else if ((cb->type & DESC_TYPE_PAGE) && cb->length)
3253                dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
3254                               ring_to_dma_dir(ring));
3255        else if (cb->type & (DESC_TYPE_BOUNCE_ALL | DESC_TYPE_BOUNCE_HEAD |
3256                             DESC_TYPE_SGL_SKB))
3257                hns3_tx_spare_reclaim_cb(ring, cb);
3258}
3259
3260static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
3261{
3262        hns3_unmap_buffer(ring, &ring->desc_cb[i]);
3263        ring->desc[i].addr = 0;
3264}
3265
3266static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i,
3267                                    int budget)
3268{
3269        struct hns3_desc_cb *cb = &ring->desc_cb[i];
3270
3271        if (!ring->desc_cb[i].dma)
3272                return;
3273
3274        hns3_buffer_detach(ring, i);
3275        hns3_free_buffer(ring, cb, budget);
3276}
3277
3278static void hns3_free_buffers(struct hns3_enet_ring *ring)
3279{
3280        int i;
3281
3282        for (i = 0; i < ring->desc_num; i++)
3283                hns3_free_buffer_detach(ring, i, 0);
3284}
3285
3286/* free desc along with its attached buffer */
3287static void hns3_free_desc(struct hns3_enet_ring *ring)
3288{
3289        int size = ring->desc_num * sizeof(ring->desc[0]);
3290
3291        hns3_free_buffers(ring);
3292
3293        if (ring->desc) {
3294                dma_free_coherent(ring_to_dev(ring), size,
3295                                  ring->desc, ring->desc_dma_addr);
3296                ring->desc = NULL;
3297        }
3298}
3299
3300static int hns3_alloc_desc(struct hns3_enet_ring *ring)
3301{
3302        int size = ring->desc_num * sizeof(ring->desc[0]);
3303
3304        ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,
3305                                        &ring->desc_dma_addr, GFP_KERNEL);
3306        if (!ring->desc)
3307                return -ENOMEM;
3308
3309        return 0;
3310}
3311
3312static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring,
3313                                   struct hns3_desc_cb *cb)
3314{
3315        int ret;
3316
3317        ret = hns3_alloc_buffer(ring, cb);
3318        if (ret)
3319                goto out;
3320
3321        ret = hns3_map_buffer(ring, cb);
3322        if (ret)
3323                goto out_with_buf;
3324
3325        return 0;
3326
3327out_with_buf:
3328        hns3_free_buffer(ring, cb, 0);
3329out:
3330        return ret;
3331}
3332
3333static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i)
3334{
3335        int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]);
3336
3337        if (ret)
3338                return ret;
3339
3340        ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
3341
3342        return 0;
3343}
3344
3345/* Allocate memory for raw pkg, and map with dma */
3346static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
3347{
3348        int i, j, ret;
3349
3350        for (i = 0; i < ring->desc_num; i++) {
3351                ret = hns3_alloc_and_attach_buffer(ring, i);
3352                if (ret)
3353                        goto out_buffer_fail;
3354        }
3355
3356        return 0;
3357
3358out_buffer_fail:
3359        for (j = i - 1; j >= 0; j--)
3360                hns3_free_buffer_detach(ring, j, 0);
3361        return ret;
3362}
3363
3364/* detach a in-used buffer and replace with a reserved one */
3365static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
3366                                struct hns3_desc_cb *res_cb)
3367{
3368        hns3_unmap_buffer(ring, &ring->desc_cb[i]);
3369        ring->desc_cb[i] = *res_cb;
3370        ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
3371        ring->desc[i].rx.bd_base_info = 0;
3372}
3373
3374static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
3375{
3376        ring->desc_cb[i].reuse_flag = 0;
3377        ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
3378                                         ring->desc_cb[i].page_offset);
3379        ring->desc[i].rx.bd_base_info = 0;
3380
3381        dma_sync_single_for_device(ring_to_dev(ring),
3382                        ring->desc_cb[i].dma + ring->desc_cb[i].page_offset,
3383                        hns3_buf_size(ring),
3384                        DMA_FROM_DEVICE);
3385}
3386
3387static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring,
3388                                  int *bytes, int *pkts, int budget)
3389{
3390        /* pair with ring->last_to_use update in hns3_tx_doorbell(),
3391         * smp_store_release() is not used in hns3_tx_doorbell() because
3392         * the doorbell operation already have the needed barrier operation.
3393         */
3394        int ltu = smp_load_acquire(&ring->last_to_use);
3395        int ntc = ring->next_to_clean;
3396        struct hns3_desc_cb *desc_cb;
3397        bool reclaimed = false;
3398        struct hns3_desc *desc;
3399
3400        while (ltu != ntc) {
3401                desc = &ring->desc[ntc];
3402
3403                if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) &
3404                                BIT(HNS3_TXD_VLD_B))
3405                        break;
3406
3407                desc_cb = &ring->desc_cb[ntc];
3408
3409                if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_ALL |
3410                                     DESC_TYPE_BOUNCE_HEAD |
3411                                     DESC_TYPE_SGL_SKB)) {
3412                        (*pkts)++;
3413                        (*bytes) += desc_cb->send_bytes;
3414                }
3415
3416                /* desc_cb will be cleaned, after hnae3_free_buffer_detach */
3417                hns3_free_buffer_detach(ring, ntc, budget);
3418
3419                if (++ntc == ring->desc_num)
3420                        ntc = 0;
3421
3422                /* Issue prefetch for next Tx descriptor */
3423                prefetch(&ring->desc_cb[ntc]);
3424                reclaimed = true;
3425        }
3426
3427        if (unlikely(!reclaimed))
3428                return false;
3429
3430        /* This smp_store_release() pairs with smp_load_acquire() in
3431         * ring_space called by hns3_nic_net_xmit.
3432         */
3433        smp_store_release(&ring->next_to_clean, ntc);
3434
3435        hns3_tx_spare_update(ring);
3436
3437        return true;
3438}
3439
3440void hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
3441{
3442        struct net_device *netdev = ring_to_netdev(ring);
3443        struct hns3_nic_priv *priv = netdev_priv(netdev);
3444        struct netdev_queue *dev_queue;
3445        int bytes, pkts;
3446
3447        bytes = 0;
3448        pkts = 0;
3449
3450        if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts, budget)))
3451                return;
3452
3453        ring->tqp_vector->tx_group.total_bytes += bytes;
3454        ring->tqp_vector->tx_group.total_packets += pkts;
3455
3456        u64_stats_update_begin(&ring->syncp);
3457        ring->stats.tx_bytes += bytes;
3458        ring->stats.tx_pkts += pkts;
3459        u64_stats_update_end(&ring->syncp);
3460
3461        dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
3462        netdev_tx_completed_queue(dev_queue, pkts, bytes);
3463
3464        if (unlikely(netif_carrier_ok(netdev) &&
3465                     ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) {
3466                /* Make sure that anybody stopping the queue after this
3467                 * sees the new next_to_clean.
3468                 */
3469                smp_mb();
3470                if (netif_tx_queue_stopped(dev_queue) &&
3471                    !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
3472                        netif_tx_wake_queue(dev_queue);
3473                        ring->stats.restart_queue++;
3474                }
3475        }
3476}
3477
3478static int hns3_desc_unused(struct hns3_enet_ring *ring)
3479{
3480        int ntc = ring->next_to_clean;
3481        int ntu = ring->next_to_use;
3482
3483        return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
3484}
3485
3486static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring,
3487                                      int cleand_count)
3488{
3489        struct hns3_desc_cb *desc_cb;
3490        struct hns3_desc_cb res_cbs;
3491        int i, ret;
3492
3493        for (i = 0; i < cleand_count; i++) {
3494                desc_cb = &ring->desc_cb[ring->next_to_use];
3495                if (desc_cb->reuse_flag) {
3496                        u64_stats_update_begin(&ring->syncp);
3497                        ring->stats.reuse_pg_cnt++;
3498                        u64_stats_update_end(&ring->syncp);
3499
3500                        hns3_reuse_buffer(ring, ring->next_to_use);
3501                } else {
3502                        ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
3503                        if (ret) {
3504                                u64_stats_update_begin(&ring->syncp);
3505                                ring->stats.sw_err_cnt++;
3506                                u64_stats_update_end(&ring->syncp);
3507
3508                                hns3_rl_err(ring_to_netdev(ring),
3509                                            "alloc rx buffer failed: %d\n",
3510                                            ret);
3511                                break;
3512                        }
3513                        hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
3514
3515                        u64_stats_update_begin(&ring->syncp);
3516                        ring->stats.non_reuse_pg++;
3517                        u64_stats_update_end(&ring->syncp);
3518                }
3519
3520                ring_ptr_move_fw(ring, next_to_use);
3521        }
3522
3523        writel(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
3524}
3525
3526static bool hns3_can_reuse_page(struct hns3_desc_cb *cb)
3527{
3528        return page_count(cb->priv) == cb->pagecnt_bias;
3529}
3530
3531static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
3532                                struct hns3_enet_ring *ring, int pull_len,
3533                                struct hns3_desc_cb *desc_cb)
3534{
3535        struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
3536        u32 frag_offset = desc_cb->page_offset + pull_len;
3537        int size = le16_to_cpu(desc->rx.size);
3538        u32 truesize = hns3_buf_size(ring);
3539        u32 frag_size = size - pull_len;
3540        bool reused;
3541
3542        /* Avoid re-using remote or pfmem page */
3543        if (unlikely(!dev_page_is_reusable(desc_cb->priv)))
3544                goto out;
3545
3546        reused = hns3_can_reuse_page(desc_cb);
3547
3548        /* Rx page can be reused when:
3549         * 1. Rx page is only owned by the driver when page_offset
3550         *    is zero, which means 0 @ truesize will be used by
3551         *    stack after skb_add_rx_frag() is called, and the rest
3552         *    of rx page can be reused by driver.
3553         * Or
3554         * 2. Rx page is only owned by the driver when page_offset
3555         *    is non-zero, which means page_offset @ truesize will
3556         *    be used by stack after skb_add_rx_frag() is called,
3557         *    and 0 @ truesize can be reused by driver.
3558         */
3559        if ((!desc_cb->page_offset && reused) ||
3560            ((desc_cb->page_offset + truesize + truesize) <=
3561             hns3_page_size(ring) && desc_cb->page_offset)) {
3562                desc_cb->page_offset += truesize;
3563                desc_cb->reuse_flag = 1;
3564        } else if (desc_cb->page_offset && reused) {
3565                desc_cb->page_offset = 0;
3566                desc_cb->reuse_flag = 1;
3567        } else if (frag_size <= ring->rx_copybreak) {
3568                void *frag = napi_alloc_frag(frag_size);
3569
3570                if (unlikely(!frag)) {
3571                        u64_stats_update_begin(&ring->syncp);
3572                        ring->stats.frag_alloc_err++;
3573                        u64_stats_update_end(&ring->syncp);
3574
3575                        hns3_rl_err(ring_to_netdev(ring),
3576                                    "failed to allocate rx frag\n");
3577                        goto out;
3578                }
3579
3580                desc_cb->reuse_flag = 1;
3581                memcpy(frag, desc_cb->buf + frag_offset, frag_size);
3582                skb_add_rx_frag(skb, i, virt_to_page(frag),
3583                                offset_in_page(frag), frag_size, frag_size);
3584
3585                u64_stats_update_begin(&ring->syncp);
3586                ring->stats.frag_alloc++;
3587                u64_stats_update_end(&ring->syncp);
3588                return;
3589        }
3590
3591out:
3592        desc_cb->pagecnt_bias--;
3593
3594        if (unlikely(!desc_cb->pagecnt_bias)) {
3595                page_ref_add(desc_cb->priv, USHRT_MAX);
3596                desc_cb->pagecnt_bias = USHRT_MAX;
3597        }
3598
3599        skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset,
3600                        frag_size, truesize);
3601
3602        if (unlikely(!desc_cb->reuse_flag))
3603                __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
3604}
3605
3606static int hns3_gro_complete(struct sk_buff *skb, u32 l234info)
3607{
3608        __be16 type = skb->protocol;
3609        struct tcphdr *th;
3610        int depth = 0;
3611
3612        while (eth_type_vlan(type)) {
3613                struct vlan_hdr *vh;
3614
3615                if ((depth + VLAN_HLEN) > skb_headlen(skb))
3616                        return -EFAULT;
3617
3618                vh = (struct vlan_hdr *)(skb->data + depth);
3619                type = vh->h_vlan_encapsulated_proto;
3620                depth += VLAN_HLEN;
3621        }
3622
3623        skb_set_network_header(skb, depth);
3624
3625        if (type == htons(ETH_P_IP)) {
3626                const struct iphdr *iph = ip_hdr(skb);
3627
3628                depth += sizeof(struct iphdr);
3629                skb_set_transport_header(skb, depth);
3630                th = tcp_hdr(skb);
3631                th->check = ~tcp_v4_check(skb->len - depth, iph->saddr,
3632                                          iph->daddr, 0);
3633        } else if (type == htons(ETH_P_IPV6)) {
3634                const struct ipv6hdr *iph = ipv6_hdr(skb);
3635
3636                depth += sizeof(struct ipv6hdr);
3637                skb_set_transport_header(skb, depth);
3638                th = tcp_hdr(skb);
3639                th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr,
3640                                          &iph->daddr, 0);
3641        } else {
3642                hns3_rl_err(skb->dev,
3643                            "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",
3644                            be16_to_cpu(type), depth);
3645                return -EFAULT;
3646        }
3647
3648        skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
3649        if (th->cwr)
3650                skb_shinfo(skb)->gso_