linux/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2// Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4#include <linux/dma-mapping.h>
   5#include <linux/etherdevice.h>
   6#include <linux/interrupt.h>
   7#ifdef CONFIG_RFS_ACCEL
   8#include <linux/cpu_rmap.h>
   9#endif
  10#include <linux/if_vlan.h>
  11#include <linux/irq.h>
  12#include <linux/ip.h>
  13#include <linux/ipv6.h>
  14#include <linux/module.h>
  15#include <linux/pci.h>
  16#include <linux/aer.h>
  17#include <linux/skbuff.h>
  18#include <linux/sctp.h>
  19#include <net/gre.h>
  20#include <net/ip6_checksum.h>
  21#include <net/pkt_cls.h>
  22#include <net/tcp.h>
  23#include <net/vxlan.h>
  24#include <net/geneve.h>
  25
  26#include "hnae3.h"
  27#include "hns3_enet.h"
  28/* All hns3 tracepoints are defined by the include below, which
  29 * must be included exactly once across the whole kernel with
  30 * CREATE_TRACE_POINTS defined
  31 */
  32#define CREATE_TRACE_POINTS
  33#include "hns3_trace.h"
  34
  35#define hns3_set_field(origin, shift, val)      ((origin) |= (val) << (shift))
  36#define hns3_tx_bd_count(S)     DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)
  37
  38#define hns3_rl_err(fmt, ...)                                           \
  39        do {                                                            \
  40                if (net_ratelimit())                                    \
  41                        netdev_err(fmt, ##__VA_ARGS__);                 \
  42        } while (0)
  43
  44static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);
  45
  46static const char hns3_driver_name[] = "hns3";
  47static const char hns3_driver_string[] =
  48                        "Hisilicon Ethernet Network Driver for Hip08 Family";
  49static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  50static struct hnae3_client client;
  51
  52static int debug = -1;
  53module_param(debug, int, 0);
  54MODULE_PARM_DESC(debug, " Network interface message level setting");
  55
  56#define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  57                           NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
  58
  59#define HNS3_INNER_VLAN_TAG     1
  60#define HNS3_OUTER_VLAN_TAG     2
  61
  62#define HNS3_MIN_TX_LEN         33U
  63
  64/* hns3_pci_tbl - PCI Device ID Table
  65 *
  66 * Last entry must be all 0s
  67 *
  68 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  69 *   Class, Class Mask, private data (not used) }
  70 */
  71static const struct pci_device_id hns3_pci_tbl[] = {
  72        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  73        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  74        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  75         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  76        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  77         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  78        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  79         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  80        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  81         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  82        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  83         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  84        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
  85         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  86        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},
  87        {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
  88         HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  89        /* required last entry */
  90        {0, }
  91};
  92MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  93
  94static irqreturn_t hns3_irq_handle(int irq, void *vector)
  95{
  96        struct hns3_enet_tqp_vector *tqp_vector = vector;
  97
  98        napi_schedule_irqoff(&tqp_vector->napi);
  99
 100        return IRQ_HANDLED;
 101}
 102
 103static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
 104{
 105        struct hns3_enet_tqp_vector *tqp_vectors;
 106        unsigned int i;
 107
 108        for (i = 0; i < priv->vector_num; i++) {
 109                tqp_vectors = &priv->tqp_vector[i];
 110
 111                if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
 112                        continue;
 113
 114                /* clear the affinity mask */
 115                irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
 116
 117                /* release the irq resource */
 118                free_irq(tqp_vectors->vector_irq, tqp_vectors);
 119                tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
 120        }
 121}
 122
 123static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
 124{
 125        struct hns3_enet_tqp_vector *tqp_vectors;
 126        int txrx_int_idx = 0;
 127        int rx_int_idx = 0;
 128        int tx_int_idx = 0;
 129        unsigned int i;
 130        int ret;
 131
 132        for (i = 0; i < priv->vector_num; i++) {
 133                tqp_vectors = &priv->tqp_vector[i];
 134
 135                if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 136                        continue;
 137
 138                if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 139                        snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 140                                 "%s-%s-%s-%d", hns3_driver_name,
 141                                 pci_name(priv->ae_handle->pdev),
 142                                 "TxRx", txrx_int_idx++);
 143                        txrx_int_idx++;
 144                } else if (tqp_vectors->rx_group.ring) {
 145                        snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 146                                 "%s-%s-%s-%d", hns3_driver_name,
 147                                 pci_name(priv->ae_handle->pdev),
 148                                 "Rx", rx_int_idx++);
 149                } else if (tqp_vectors->tx_group.ring) {
 150                        snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,
 151                                 "%s-%s-%s-%d", hns3_driver_name,
 152                                 pci_name(priv->ae_handle->pdev),
 153                                 "Tx", tx_int_idx++);
 154                } else {
 155                        /* Skip this unused q_vector */
 156                        continue;
 157                }
 158
 159                tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 160
 161                irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN);
 162                ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 163                                  tqp_vectors->name, tqp_vectors);
 164                if (ret) {
 165                        netdev_err(priv->netdev, "request irq(%d) fail\n",
 166                                   tqp_vectors->vector_irq);
 167                        hns3_nic_uninit_irq(priv);
 168                        return ret;
 169                }
 170
 171                irq_set_affinity_hint(tqp_vectors->vector_irq,
 172                                      &tqp_vectors->affinity_mask);
 173
 174                tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 175        }
 176
 177        return 0;
 178}
 179
 180static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 181                                 u32 mask_en)
 182{
 183        writel(mask_en, tqp_vector->mask_addr);
 184}
 185
 186static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 187{
 188        napi_enable(&tqp_vector->napi);
 189        enable_irq(tqp_vector->vector_irq);
 190
 191        /* enable vector */
 192        hns3_mask_vector_irq(tqp_vector, 1);
 193}
 194
 195static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 196{
 197        /* disable vector */
 198        hns3_mask_vector_irq(tqp_vector, 0);
 199
 200        disable_irq(tqp_vector->vector_irq);
 201        napi_disable(&tqp_vector->napi);
 202}
 203
 204void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 205                                 u32 rl_value)
 206{
 207        u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 208
 209        /* this defines the configuration for RL (Interrupt Rate Limiter).
 210         * Rl defines rate of interrupts i.e. number of interrupts-per-second
 211         * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 212         */
 213        if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&
 214            !tqp_vector->rx_group.coal.adapt_enable)
 215                /* According to the hardware, the range of rl_reg is
 216                 * 0-59 and the unit is 4.
 217                 */
 218                rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 219
 220        writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 221}
 222
 223void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 224                                    u32 gl_value)
 225{
 226        u32 new_val;
 227
 228        if (tqp_vector->rx_group.coal.unit_1us)
 229                new_val = gl_value | HNS3_INT_GL_1US;
 230        else
 231                new_val = hns3_gl_usec_to_reg(gl_value);
 232
 233        writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 234}
 235
 236void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 237                                    u32 gl_value)
 238{
 239        u32 new_val;
 240
 241        if (tqp_vector->tx_group.coal.unit_1us)
 242                new_val = gl_value | HNS3_INT_GL_1US;
 243        else
 244                new_val = hns3_gl_usec_to_reg(gl_value);
 245
 246        writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 247}
 248
 249void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,
 250                                    u32 ql_value)
 251{
 252        writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);
 253}
 254
 255void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,
 256                                    u32 ql_value)
 257{
 258        writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);
 259}
 260
 261static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,
 262                                      struct hns3_nic_priv *priv)
 263{
 264        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
 265        struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
 266        struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
 267        struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal;
 268        struct hns3_enet_coalesce *prx_coal = &priv->rx_coal;
 269
 270        tx_coal->adapt_enable = ptx_coal->adapt_enable;
 271        rx_coal->adapt_enable = prx_coal->adapt_enable;
 272
 273        tx_coal->int_gl = ptx_coal->int_gl;
 274        rx_coal->int_gl = prx_coal->int_gl;
 275
 276        rx_coal->flow_level = prx_coal->flow_level;
 277        tx_coal->flow_level = ptx_coal->flow_level;
 278
 279        /* device version above V3(include V3), GL can configure 1us
 280         * unit, so uses 1us unit.
 281         */
 282        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {
 283                tx_coal->unit_1us = 1;
 284                rx_coal->unit_1us = 1;
 285        }
 286
 287        if (ae_dev->dev_specs.int_ql_max) {
 288                tx_coal->ql_enable = 1;
 289                rx_coal->ql_enable = 1;
 290                tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
 291                rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;
 292                tx_coal->int_ql = ptx_coal->int_ql;
 293                rx_coal->int_ql = prx_coal->int_ql;
 294        }
 295}
 296
 297static void
 298hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 299                             struct hns3_nic_priv *priv)
 300{
 301        struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;
 302        struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;
 303        struct hnae3_handle *h = priv->ae_handle;
 304
 305        hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);
 306        hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);
 307        hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 308
 309        if (tx_coal->ql_enable)
 310                hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);
 311
 312        if (rx_coal->ql_enable)
 313                hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);
 314}
 315
 316static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 317{
 318        struct hnae3_handle *h = hns3_get_handle(netdev);
 319        struct hnae3_knic_private_info *kinfo = &h->kinfo;
 320        struct hnae3_tc_info *tc_info = &kinfo->tc_info;
 321        unsigned int queue_size = kinfo->num_tqps;
 322        int i, ret;
 323
 324        if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) {
 325                netdev_reset_tc(netdev);
 326        } else {
 327                ret = netdev_set_num_tc(netdev, tc_info->num_tc);
 328                if (ret) {
 329                        netdev_err(netdev,
 330                                   "netdev_set_num_tc fail, ret=%d!\n", ret);
 331                        return ret;
 332                }
 333
 334                for (i = 0; i < HNAE3_MAX_TC; i++) {
 335                        if (!test_bit(i, &tc_info->tc_en))
 336                                continue;
 337
 338                        netdev_set_tc_queue(netdev, i, tc_info->tqp_count[i],
 339                                            tc_info->tqp_offset[i]);
 340                }
 341        }
 342
 343        ret = netif_set_real_num_tx_queues(netdev, queue_size);
 344        if (ret) {
 345                netdev_err(netdev,
 346                           "netif_set_real_num_tx_queues fail, ret=%d!\n", ret);
 347                return ret;
 348        }
 349
 350        ret = netif_set_real_num_rx_queues(netdev, queue_size);
 351        if (ret) {
 352                netdev_err(netdev,
 353                           "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 354                return ret;
 355        }
 356
 357        return 0;
 358}
 359
 360static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 361{
 362        u16 alloc_tqps, max_rss_size, rss_size;
 363
 364        h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
 365        rss_size = alloc_tqps / h->kinfo.tc_info.num_tc;
 366
 367        return min_t(u16, rss_size, max_rss_size);
 368}
 369
 370static void hns3_tqp_enable(struct hnae3_queue *tqp)
 371{
 372        u32 rcb_reg;
 373
 374        rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 375        rcb_reg |= BIT(HNS3_RING_EN_B);
 376        hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 377}
 378
 379static void hns3_tqp_disable(struct hnae3_queue *tqp)
 380{
 381        u32 rcb_reg;
 382
 383        rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
 384        rcb_reg &= ~BIT(HNS3_RING_EN_B);
 385        hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
 386}
 387
 388static void hns3_free_rx_cpu_rmap(struct net_device *netdev)
 389{
 390#ifdef CONFIG_RFS_ACCEL
 391        free_irq_cpu_rmap(netdev->rx_cpu_rmap);
 392        netdev->rx_cpu_rmap = NULL;
 393#endif
 394}
 395
 396static int hns3_set_rx_cpu_rmap(struct net_device *netdev)
 397{
 398#ifdef CONFIG_RFS_ACCEL
 399        struct hns3_nic_priv *priv = netdev_priv(netdev);
 400        struct hns3_enet_tqp_vector *tqp_vector;
 401        int i, ret;
 402
 403        if (!netdev->rx_cpu_rmap) {
 404                netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num);
 405                if (!netdev->rx_cpu_rmap)
 406                        return -ENOMEM;
 407        }
 408
 409        for (i = 0; i < priv->vector_num; i++) {
 410                tqp_vector = &priv->tqp_vector[i];
 411                ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap,
 412                                       tqp_vector->vector_irq);
 413                if (ret) {
 414                        hns3_free_rx_cpu_rmap(netdev);
 415                        return ret;
 416                }
 417        }
 418#endif
 419        return 0;
 420}
 421
 422static int hns3_nic_net_up(struct net_device *netdev)
 423{
 424        struct hns3_nic_priv *priv = netdev_priv(netdev);
 425        struct hnae3_handle *h = priv->ae_handle;
 426        int i, j;
 427        int ret;
 428
 429        ret = hns3_nic_reset_all_ring(h);
 430        if (ret)
 431                return ret;
 432
 433        clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 434
 435        /* enable the vectors */
 436        for (i = 0; i < priv->vector_num; i++)
 437                hns3_vector_enable(&priv->tqp_vector[i]);
 438
 439        /* enable rcb */
 440        for (j = 0; j < h->kinfo.num_tqps; j++)
 441                hns3_tqp_enable(h->kinfo.tqp[j]);
 442
 443        /* start the ae_dev */
 444        ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 445        if (ret) {
 446                set_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 447                while (j--)
 448                        hns3_tqp_disable(h->kinfo.tqp[j]);
 449
 450                for (j = i - 1; j >= 0; j--)
 451                        hns3_vector_disable(&priv->tqp_vector[j]);
 452        }
 453
 454        return ret;
 455}
 456
 457static void hns3_config_xps(struct hns3_nic_priv *priv)
 458{
 459        int i;
 460
 461        for (i = 0; i < priv->vector_num; i++) {
 462                struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];
 463                struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;
 464
 465                while (ring) {
 466                        int ret;
 467
 468                        ret = netif_set_xps_queue(priv->netdev,
 469                                                  &tqp_vector->affinity_mask,
 470                                                  ring->tqp->tqp_index);
 471                        if (ret)
 472                                netdev_warn(priv->netdev,
 473                                            "set xps queue failed: %d", ret);
 474
 475                        ring = ring->next;
 476                }
 477        }
 478}
 479
 480static int hns3_nic_net_open(struct net_device *netdev)
 481{
 482        struct hns3_nic_priv *priv = netdev_priv(netdev);
 483        struct hnae3_handle *h = hns3_get_handle(netdev);
 484        struct hnae3_knic_private_info *kinfo;
 485        int i, ret;
 486
 487        if (hns3_nic_resetting(netdev))
 488                return -EBUSY;
 489
 490        netif_carrier_off(netdev);
 491
 492        ret = hns3_nic_set_real_num_queue(netdev);
 493        if (ret)
 494                return ret;
 495
 496        ret = hns3_nic_net_up(netdev);
 497        if (ret) {
 498                netdev_err(netdev, "net up fail, ret=%d!\n", ret);
 499                return ret;
 500        }
 501
 502        kinfo = &h->kinfo;
 503        for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)
 504                netdev_set_prio_tc_map(netdev, i, kinfo->tc_info.prio_tc[i]);
 505
 506        if (h->ae_algo->ops->set_timer_task)
 507                h->ae_algo->ops->set_timer_task(priv->ae_handle, true);
 508
 509        hns3_config_xps(priv);
 510
 511        netif_dbg(h, drv, netdev, "net open\n");
 512
 513        return 0;
 514}
 515
 516static void hns3_reset_tx_queue(struct hnae3_handle *h)
 517{
 518        struct net_device *ndev = h->kinfo.netdev;
 519        struct hns3_nic_priv *priv = netdev_priv(ndev);
 520        struct netdev_queue *dev_queue;
 521        u32 i;
 522
 523        for (i = 0; i < h->kinfo.num_tqps; i++) {
 524                dev_queue = netdev_get_tx_queue(ndev,
 525                                                priv->ring[i].queue_index);
 526                netdev_tx_reset_queue(dev_queue);
 527        }
 528}
 529
 530static void hns3_nic_net_down(struct net_device *netdev)
 531{
 532        struct hns3_nic_priv *priv = netdev_priv(netdev);
 533        struct hnae3_handle *h = hns3_get_handle(netdev);
 534        const struct hnae3_ae_ops *ops;
 535        int i;
 536
 537        /* disable vectors */
 538        for (i = 0; i < priv->vector_num; i++)
 539                hns3_vector_disable(&priv->tqp_vector[i]);
 540
 541        /* disable rcb */
 542        for (i = 0; i < h->kinfo.num_tqps; i++)
 543                hns3_tqp_disable(h->kinfo.tqp[i]);
 544
 545        /* stop ae_dev */
 546        ops = priv->ae_handle->ae_algo->ops;
 547        if (ops->stop)
 548                ops->stop(priv->ae_handle);
 549
 550        /* delay ring buffer clearing to hns3_reset_notify_uninit_enet
 551         * during reset process, because driver may not be able
 552         * to disable the ring through firmware when downing the netdev.
 553         */
 554        if (!hns3_nic_resetting(netdev))
 555                hns3_clear_all_ring(priv->ae_handle, false);
 556
 557        hns3_reset_tx_queue(priv->ae_handle);
 558}
 559
 560static int hns3_nic_net_stop(struct net_device *netdev)
 561{
 562        struct hns3_nic_priv *priv = netdev_priv(netdev);
 563        struct hnae3_handle *h = hns3_get_handle(netdev);
 564
 565        if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 566                return 0;
 567
 568        netif_dbg(h, drv, netdev, "net stop\n");
 569
 570        if (h->ae_algo->ops->set_timer_task)
 571                h->ae_algo->ops->set_timer_task(priv->ae_handle, false);
 572
 573        netif_carrier_off(netdev);
 574        netif_tx_disable(netdev);
 575
 576        hns3_nic_net_down(netdev);
 577
 578        return 0;
 579}
 580
 581static int hns3_nic_uc_sync(struct net_device *netdev,
 582                            const unsigned char *addr)
 583{
 584        struct hnae3_handle *h = hns3_get_handle(netdev);
 585
 586        if (h->ae_algo->ops->add_uc_addr)
 587                return h->ae_algo->ops->add_uc_addr(h, addr);
 588
 589        return 0;
 590}
 591
 592static int hns3_nic_uc_unsync(struct net_device *netdev,
 593                              const unsigned char *addr)
 594{
 595        struct hnae3_handle *h = hns3_get_handle(netdev);
 596
 597        /* need ignore the request of removing device address, because
 598         * we store the device address and other addresses of uc list
 599         * in the function's mac filter list.
 600         */
 601        if (ether_addr_equal(addr, netdev->dev_addr))
 602                return 0;
 603
 604        if (h->ae_algo->ops->rm_uc_addr)
 605                return h->ae_algo->ops->rm_uc_addr(h, addr);
 606
 607        return 0;
 608}
 609
 610static int hns3_nic_mc_sync(struct net_device *netdev,
 611                            const unsigned char *addr)
 612{
 613        struct hnae3_handle *h = hns3_get_handle(netdev);
 614
 615        if (h->ae_algo->ops->add_mc_addr)
 616                return h->ae_algo->ops->add_mc_addr(h, addr);
 617
 618        return 0;
 619}
 620
 621static int hns3_nic_mc_unsync(struct net_device *netdev,
 622                              const unsigned char *addr)
 623{
 624        struct hnae3_handle *h = hns3_get_handle(netdev);
 625
 626        if (h->ae_algo->ops->rm_mc_addr)
 627                return h->ae_algo->ops->rm_mc_addr(h, addr);
 628
 629        return 0;
 630}
 631
 632static u8 hns3_get_netdev_flags(struct net_device *netdev)
 633{
 634        u8 flags = 0;
 635
 636        if (netdev->flags & IFF_PROMISC) {
 637                flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE;
 638        } else {
 639                flags |= HNAE3_VLAN_FLTR;
 640                if (netdev->flags & IFF_ALLMULTI)
 641                        flags |= HNAE3_USER_MPE;
 642        }
 643
 644        return flags;
 645}
 646
 647static void hns3_nic_set_rx_mode(struct net_device *netdev)
 648{
 649        struct hnae3_handle *h = hns3_get_handle(netdev);
 650        u8 new_flags;
 651
 652        new_flags = hns3_get_netdev_flags(netdev);
 653
 654        __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
 655        __dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync);
 656
 657        /* User mode Promisc mode enable and vlan filtering is disabled to
 658         * let all packets in.
 659         */
 660        h->netdev_flags = new_flags;
 661        hns3_request_update_promisc_mode(h);
 662}
 663
 664void hns3_request_update_promisc_mode(struct hnae3_handle *handle)
 665{
 666        const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 667
 668        if (ops->request_update_promisc_mode)
 669                ops->request_update_promisc_mode(handle);
 670}
 671
 672void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
 673{
 674        struct hns3_nic_priv *priv = netdev_priv(netdev);
 675        struct hnae3_handle *h = priv->ae_handle;
 676        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev);
 677        bool last_state;
 678
 679        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2 &&
 680            h->ae_algo->ops->enable_vlan_filter) {
 681                last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
 682                if (enable != last_state) {
 683                        netdev_info(netdev,
 684                                    "%s vlan filter\n",
 685                                    enable ? "enable" : "disable");
 686                        h->ae_algo->ops->enable_vlan_filter(h, enable);
 687                }
 688        }
 689}
 690
 691static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs,
 692                        u16 *mss, u32 *type_cs_vlan_tso, u32 *send_bytes)
 693{
 694        u32 l4_offset, hdr_len;
 695        union l3_hdr_info l3;
 696        union l4_hdr_info l4;
 697        u32 l4_paylen;
 698        int ret;
 699
 700        if (!skb_is_gso(skb))
 701                return 0;
 702
 703        ret = skb_cow_head(skb, 0);
 704        if (unlikely(ret < 0))
 705                return ret;
 706
 707        l3.hdr = skb_network_header(skb);
 708        l4.hdr = skb_transport_header(skb);
 709
 710        /* Software should clear the IPv4's checksum field when tso is
 711         * needed.
 712         */
 713        if (l3.v4->version == 4)
 714                l3.v4->check = 0;
 715
 716        /* tunnel packet */
 717        if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 718                                         SKB_GSO_GRE_CSUM |
 719                                         SKB_GSO_UDP_TUNNEL |
 720                                         SKB_GSO_UDP_TUNNEL_CSUM)) {
 721                /* reset l3&l4 pointers from outer to inner headers */
 722                l3.hdr = skb_inner_network_header(skb);
 723                l4.hdr = skb_inner_transport_header(skb);
 724
 725                /* Software should clear the IPv4's checksum field when
 726                 * tso is needed.
 727                 */
 728                if (l3.v4->version == 4)
 729                        l3.v4->check = 0;
 730        }
 731
 732        /* normal or tunnel packet */
 733        l4_offset = l4.hdr - skb->data;
 734
 735        /* remove payload length from inner pseudo checksum when tso */
 736        l4_paylen = skb->len - l4_offset;
 737
 738        if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
 739                hdr_len = sizeof(*l4.udp) + l4_offset;
 740                csum_replace_by_diff(&l4.udp->check,
 741                                     (__force __wsum)htonl(l4_paylen));
 742        } else {
 743                hdr_len = (l4.tcp->doff << 2) + l4_offset;
 744                csum_replace_by_diff(&l4.tcp->check,
 745                                     (__force __wsum)htonl(l4_paylen));
 746        }
 747
 748        *send_bytes = (skb_shinfo(skb)->gso_segs - 1) * hdr_len + skb->len;
 749
 750        /* find the txbd field values */
 751        *paylen_fdop_ol4cs = skb->len - hdr_len;
 752        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1);
 753
 754        /* offload outer UDP header checksum */
 755        if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
 756                hns3_set_field(*paylen_fdop_ol4cs, HNS3_TXD_OL4CS_B, 1);
 757
 758        /* get MSS for TSO */
 759        *mss = skb_shinfo(skb)->gso_size;
 760
 761        trace_hns3_tso(skb);
 762
 763        return 0;
 764}
 765
 766static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 767                                u8 *il4_proto)
 768{
 769        union l3_hdr_info l3;
 770        unsigned char *l4_hdr;
 771        unsigned char *exthdr;
 772        u8 l4_proto_tmp;
 773        __be16 frag_off;
 774
 775        /* find outer header point */
 776        l3.hdr = skb_network_header(skb);
 777        l4_hdr = skb_transport_header(skb);
 778
 779        if (skb->protocol == htons(ETH_P_IPV6)) {
 780                exthdr = l3.hdr + sizeof(*l3.v6);
 781                l4_proto_tmp = l3.v6->nexthdr;
 782                if (l4_hdr != exthdr)
 783                        ipv6_skip_exthdr(skb, exthdr - skb->data,
 784                                         &l4_proto_tmp, &frag_off);
 785        } else if (skb->protocol == htons(ETH_P_IP)) {
 786                l4_proto_tmp = l3.v4->protocol;
 787        } else {
 788                return -EINVAL;
 789        }
 790
 791        *ol4_proto = l4_proto_tmp;
 792
 793        /* tunnel packet */
 794        if (!skb->encapsulation) {
 795                *il4_proto = 0;
 796                return 0;
 797        }
 798
 799        /* find inner header point */
 800        l3.hdr = skb_inner_network_header(skb);
 801        l4_hdr = skb_inner_transport_header(skb);
 802
 803        if (l3.v6->version == 6) {
 804                exthdr = l3.hdr + sizeof(*l3.v6);
 805                l4_proto_tmp = l3.v6->nexthdr;
 806                if (l4_hdr != exthdr)
 807                        ipv6_skip_exthdr(skb, exthdr - skb->data,
 808                                         &l4_proto_tmp, &frag_off);
 809        } else if (l3.v4->version == 4) {
 810                l4_proto_tmp = l3.v4->protocol;
 811        }
 812
 813        *il4_proto = l4_proto_tmp;
 814
 815        return 0;
 816}
 817
 818/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
 819 * and it is udp packet, which has a dest port as the IANA assigned.
 820 * the hardware is expected to do the checksum offload, but the
 821 * hardware will not do the checksum offload when udp dest port is
 822 * 4789, 4790 or 6081.
 823 */
 824static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
 825{
 826        struct hns3_nic_priv *priv = netdev_priv(skb->dev);
 827        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);
 828        union l4_hdr_info l4;
 829
 830        /* device version above V3(include V3), the hardware can
 831         * do this checksum offload.
 832         */
 833        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
 834                return false;
 835
 836        l4.hdr = skb_transport_header(skb);
 837
 838        if (!(!skb->encapsulation &&
 839              (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) ||
 840              l4.udp->dest == htons(GENEVE_UDP_PORT) ||
 841              l4.udp->dest == htons(4790))))
 842                return false;
 843
 844        return true;
 845}
 846
 847static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 848                                  u32 *ol_type_vlan_len_msec)
 849{
 850        u32 l2_len, l3_len, l4_len;
 851        unsigned char *il2_hdr;
 852        union l3_hdr_info l3;
 853        union l4_hdr_info l4;
 854
 855        l3.hdr = skb_network_header(skb);
 856        l4.hdr = skb_transport_header(skb);
 857
 858        /* compute OL2 header size, defined in 2 Bytes */
 859        l2_len = l3.hdr - skb->data;
 860        hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1);
 861
 862        /* compute OL3 header size, defined in 4 Bytes */
 863        l3_len = l4.hdr - l3.hdr;
 864        hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2);
 865
 866        il2_hdr = skb_inner_mac_header(skb);
 867        /* compute OL4 header size, defined in 4 Bytes */
 868        l4_len = il2_hdr - l4.hdr;
 869        hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2);
 870
 871        /* define outer network header type */
 872        if (skb->protocol == htons(ETH_P_IP)) {
 873                if (skb_is_gso(skb))
 874                        hns3_set_field(*ol_type_vlan_len_msec,
 875                                       HNS3_TXD_OL3T_S,
 876                                       HNS3_OL3T_IPV4_CSUM);
 877                else
 878                        hns3_set_field(*ol_type_vlan_len_msec,
 879                                       HNS3_TXD_OL3T_S,
 880                                       HNS3_OL3T_IPV4_NO_CSUM);
 881        } else if (skb->protocol == htons(ETH_P_IPV6)) {
 882                hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S,
 883                               HNS3_OL3T_IPV6);
 884        }
 885
 886        if (ol4_proto == IPPROTO_UDP)
 887                hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
 888                               HNS3_TUN_MAC_IN_UDP);
 889        else if (ol4_proto == IPPROTO_GRE)
 890                hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,
 891                               HNS3_TUN_NVGRE);
 892}
 893
 894static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 895                           u8 il4_proto, u32 *type_cs_vlan_tso,
 896                           u32 *ol_type_vlan_len_msec)
 897{
 898        unsigned char *l2_hdr = skb->data;
 899        u32 l4_proto = ol4_proto;
 900        union l4_hdr_info l4;
 901        union l3_hdr_info l3;
 902        u32 l2_len, l3_len;
 903
 904        l4.hdr = skb_transport_header(skb);
 905        l3.hdr = skb_network_header(skb);
 906
 907        /* handle encapsulation skb */
 908        if (skb->encapsulation) {
 909                /* If this is a not UDP/GRE encapsulation skb */
 910                if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {
 911                        /* drop the skb tunnel packet if hardware don't support,
 912                         * because hardware can't calculate csum when TSO.
 913                         */
 914                        if (skb_is_gso(skb))
 915                                return -EDOM;
 916
 917                        /* the stack computes the IP header already,
 918                         * driver calculate l4 checksum when not TSO.
 919                         */
 920                        return skb_checksum_help(skb);
 921                }
 922
 923                hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec);
 924
 925                /* switch to inner header */
 926                l2_hdr = skb_inner_mac_header(skb);
 927                l3.hdr = skb_inner_network_header(skb);
 928                l4.hdr = skb_inner_transport_header(skb);
 929                l4_proto = il4_proto;
 930        }
 931
 932        if (l3.v4->version == 4) {
 933                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
 934                               HNS3_L3T_IPV4);
 935
 936                /* the stack computes the IP header already, the only time we
 937                 * need the hardware to recompute it is in the case of TSO.
 938                 */
 939                if (skb_is_gso(skb))
 940                        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 941        } else if (l3.v6->version == 6) {
 942                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,
 943                               HNS3_L3T_IPV6);
 944        }
 945
 946        /* compute inner(/normal) L2 header size, defined in 2 Bytes */
 947        l2_len = l3.hdr - l2_hdr;
 948        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1);
 949
 950        /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 951        l3_len = l4.hdr - l3.hdr;
 952        hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2);
 953
 954        /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 955        switch (l4_proto) {
 956        case IPPROTO_TCP:
 957                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 958                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 959                               HNS3_L4T_TCP);
 960                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 961                               l4.tcp->doff);
 962                break;
 963        case IPPROTO_UDP:
 964                if (hns3_tunnel_csum_bug(skb))
 965                        return skb_checksum_help(skb);
 966
 967                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 968                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 969                               HNS3_L4T_UDP);
 970                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 971                               (sizeof(struct udphdr) >> 2));
 972                break;
 973        case IPPROTO_SCTP:
 974                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 975                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,
 976                               HNS3_L4T_SCTP);
 977                hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,
 978                               (sizeof(struct sctphdr) >> 2));
 979                break;
 980        default:
 981                /* drop the skb tunnel packet if hardware don't support,
 982                 * because hardware can't calculate csum when TSO.
 983                 */
 984                if (skb_is_gso(skb))
 985                        return -EDOM;
 986
 987                /* the stack computes the IP header already,
 988                 * driver calculate l4 checksum when not TSO.
 989                 */
 990                return skb_checksum_help(skb);
 991        }
 992
 993        return 0;
 994}
 995
 996static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,
 997                             struct sk_buff *skb)
 998{
 999        struct hnae3_handle *handle = tx_ring->tqp->handle;
1000        struct hnae3_ae_dev *ae_dev;
1001        struct vlan_ethhdr *vhdr;
1002        int rc;
1003
1004        if (!(skb->protocol == htons(ETH_P_8021Q) ||
1005              skb_vlan_tag_present(skb)))
1006                return 0;
1007
1008        /* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert
1009         * VLAN enabled, only one VLAN header is allowed in skb, otherwise it
1010         * will cause RAS error.
1011         */
1012        ae_dev = pci_get_drvdata(handle->pdev);
1013        if (unlikely(skb_vlan_tagged_multi(skb) &&
1014                     ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2 &&
1015                     handle->port_base_vlan_state ==
1016                     HNAE3_PORT_BASE_VLAN_ENABLE))
1017                return -EINVAL;
1018
1019        if (skb->protocol == htons(ETH_P_8021Q) &&
1020            !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1021                /* When HW VLAN acceleration is turned off, and the stack
1022                 * sets the protocol to 802.1q, the driver just need to
1023                 * set the protocol to the encapsulated ethertype.
1024                 */
1025                skb->protocol = vlan_get_protocol(skb);
1026                return 0;
1027        }
1028
1029        if (skb_vlan_tag_present(skb)) {
1030                /* Based on hw strategy, use out_vtag in two layer tag case,
1031                 * and use inner_vtag in one tag case.
1032                 */
1033                if (skb->protocol == htons(ETH_P_8021Q) &&
1034                    handle->port_base_vlan_state ==
1035                    HNAE3_PORT_BASE_VLAN_DISABLE)
1036                        rc = HNS3_OUTER_VLAN_TAG;
1037                else
1038                        rc = HNS3_INNER_VLAN_TAG;
1039
1040                skb->protocol = vlan_get_protocol(skb);
1041                return rc;
1042        }
1043
1044        rc = skb_cow_head(skb, 0);
1045        if (unlikely(rc < 0))
1046                return rc;
1047
1048        vhdr = (struct vlan_ethhdr *)skb->data;
1049        vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)
1050                                         & VLAN_PRIO_MASK);
1051
1052        skb->protocol = vlan_get_protocol(skb);
1053        return 0;
1054}
1055
1056/* check if the hardware is capable of checksum offloading */
1057static bool hns3_check_hw_tx_csum(struct sk_buff *skb)
1058{
1059        struct hns3_nic_priv *priv = netdev_priv(skb->dev);
1060
1061        /* Kindly note, due to backward compatibility of the TX descriptor,
1062         * HW checksum of the non-IP packets and GSO packets is handled at
1063         * different place in the following code
1064         */
1065        if (skb_csum_is_sctp(skb) || skb_is_gso(skb) ||
1066            !test_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state))
1067                return false;
1068
1069        return true;
1070}
1071
1072static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,
1073                              struct sk_buff *skb, struct hns3_desc *desc,
1074                              struct hns3_desc_cb *desc_cb)
1075{
1076        u32 ol_type_vlan_len_msec = 0;
1077        u32 paylen_ol4cs = skb->len;
1078        u32 type_cs_vlan_tso = 0;
1079        u16 mss_hw_csum = 0;
1080        u16 inner_vtag = 0;
1081        u16 out_vtag = 0;
1082        int ret;
1083
1084        ret = hns3_handle_vtags(ring, skb);
1085        if (unlikely(ret < 0)) {
1086                u64_stats_update_begin(&ring->syncp);
1087                ring->stats.tx_vlan_err++;
1088                u64_stats_update_end(&ring->syncp);
1089                return ret;
1090        } else if (ret == HNS3_INNER_VLAN_TAG) {
1091                inner_vtag = skb_vlan_tag_get(skb);
1092                inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1093                                VLAN_PRIO_MASK;
1094                hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1);
1095        } else if (ret == HNS3_OUTER_VLAN_TAG) {
1096                out_vtag = skb_vlan_tag_get(skb);
1097                out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &
1098                                VLAN_PRIO_MASK;
1099                hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B,
1100                               1);
1101        }
1102
1103        desc_cb->send_bytes = skb->len;
1104
1105        if (skb->ip_summed == CHECKSUM_PARTIAL) {
1106                u8 ol4_proto, il4_proto;
1107
1108                if (hns3_check_hw_tx_csum(skb)) {
1109                        /* set checksum start and offset, defined in 2 Bytes */
1110                        hns3_set_field(type_cs_vlan_tso, HNS3_TXD_CSUM_START_S,
1111                                       skb_checksum_start_offset(skb) >> 1);
1112                        hns3_set_field(ol_type_vlan_len_msec,
1113                                       HNS3_TXD_CSUM_OFFSET_S,
1114                                       skb->csum_offset >> 1);
1115                        mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B);
1116                        goto out_hw_tx_csum;
1117                }
1118
1119                skb_reset_mac_len(skb);
1120
1121                ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1122                if (unlikely(ret < 0)) {
1123                        u64_stats_update_begin(&ring->syncp);
1124                        ring->stats.tx_l4_proto_err++;
1125                        u64_stats_update_end(&ring->syncp);
1126                        return ret;
1127                }
1128
1129                ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto,
1130                                      &type_cs_vlan_tso,
1131                                      &ol_type_vlan_len_msec);
1132                if (unlikely(ret < 0)) {
1133                        u64_stats_update_begin(&ring->syncp);
1134                        ring->stats.tx_l2l3l4_err++;
1135                        u64_stats_update_end(&ring->syncp);
1136                        return ret;
1137                }
1138
1139                ret = hns3_set_tso(skb, &paylen_ol4cs, &mss_hw_csum,
1140                                   &type_cs_vlan_tso, &desc_cb->send_bytes);
1141                if (unlikely(ret < 0)) {
1142                        u64_stats_update_begin(&ring->syncp);
1143                        ring->stats.tx_tso_err++;
1144                        u64_stats_update_end(&ring->syncp);
1145                        return ret;
1146                }
1147        }
1148
1149out_hw_tx_csum:
1150        /* Set txbd */
1151        desc->tx.ol_type_vlan_len_msec =
1152                cpu_to_le32(ol_type_vlan_len_msec);
1153        desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso);
1154        desc->tx.paylen_ol4cs = cpu_to_le32(paylen_ol4cs);
1155        desc->tx.mss_hw_csum = cpu_to_le16(mss_hw_csum);
1156        desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1157        desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1158
1159        return 0;
1160}
1161
1162static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
1163                          unsigned int size, enum hns_desc_type type)
1164{
1165#define HNS3_LIKELY_BD_NUM      1
1166
1167        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1168        struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1169        struct device *dev = ring_to_dev(ring);
1170        skb_frag_t *frag;
1171        unsigned int frag_buf_num;
1172        int k, sizeoflast;
1173        dma_addr_t dma;
1174
1175        if (type == DESC_TYPE_FRAGLIST_SKB ||
1176            type == DESC_TYPE_SKB) {
1177                struct sk_buff *skb = (struct sk_buff *)priv;
1178
1179                dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1180        } else {
1181                frag = (skb_frag_t *)priv;
1182                dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1183        }
1184
1185        if (unlikely(dma_mapping_error(dev, dma))) {
1186                u64_stats_update_begin(&ring->syncp);
1187                ring->stats.sw_err_cnt++;
1188                u64_stats_update_end(&ring->syncp);
1189                return -ENOMEM;
1190        }
1191
1192        desc_cb->priv = priv;
1193        desc_cb->length = size;
1194        desc_cb->dma = dma;
1195        desc_cb->type = type;
1196
1197        if (likely(size <= HNS3_MAX_BD_SIZE)) {
1198                desc->addr = cpu_to_le64(dma);
1199                desc->tx.send_size = cpu_to_le16(size);
1200                desc->tx.bdtp_fe_sc_vld_ra_ri =
1201                        cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1202
1203                trace_hns3_tx_desc(ring, ring->next_to_use);
1204                ring_ptr_move_fw(ring, next_to_use);
1205                return HNS3_LIKELY_BD_NUM;
1206        }
1207
1208        frag_buf_num = hns3_tx_bd_count(size);
1209        sizeoflast = size % HNS3_MAX_BD_SIZE;
1210        sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1211
1212        /* When frag size is bigger than hardware limit, split this frag */
1213        for (k = 0; k < frag_buf_num; k++) {
1214                /* now, fill the descriptor */
1215                desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1216                desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1217                                     (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1218                desc->tx.bdtp_fe_sc_vld_ra_ri =
1219                                cpu_to_le16(BIT(HNS3_TXD_VLD_B));
1220
1221                trace_hns3_tx_desc(ring, ring->next_to_use);
1222                /* move ring pointer to next */
1223                ring_ptr_move_fw(ring, next_to_use);
1224
1225                desc = &ring->desc[ring->next_to_use];
1226        }
1227
1228        return frag_buf_num;
1229}
1230
1231static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1232                                    unsigned int bd_num)
1233{
1234        unsigned int size;
1235        int i;
1236
1237        size = skb_headlen(skb);
1238        while (size > HNS3_MAX_BD_SIZE) {
1239                bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1240                size -= HNS3_MAX_BD_SIZE;
1241
1242                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1243                        return bd_num;
1244        }
1245
1246        if (size) {
1247                bd_size[bd_num++] = size;
1248                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1249                        return bd_num;
1250        }
1251
1252        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1253                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1254                size = skb_frag_size(frag);
1255                if (!size)
1256                        continue;
1257
1258                while (size > HNS3_MAX_BD_SIZE) {
1259                        bd_size[bd_num++] = HNS3_MAX_BD_SIZE;
1260                        size -= HNS3_MAX_BD_SIZE;
1261
1262                        if (bd_num > HNS3_MAX_TSO_BD_NUM)
1263                                return bd_num;
1264                }
1265
1266                bd_size[bd_num++] = size;
1267                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1268                        return bd_num;
1269        }
1270
1271        return bd_num;
1272}
1273
1274static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size,
1275                                   u8 max_non_tso_bd_num, unsigned int bd_num,
1276                                   unsigned int recursion_level)
1277{
1278#define HNS3_MAX_RECURSION_LEVEL        24
1279
1280        struct sk_buff *frag_skb;
1281
1282        /* If the total len is within the max bd limit */
1283        if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level &&
1284                   !skb_has_frag_list(skb) &&
1285                   skb_shinfo(skb)->nr_frags < max_non_tso_bd_num))
1286                return skb_shinfo(skb)->nr_frags + 1U;
1287
1288        if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL))
1289                return UINT_MAX;
1290
1291        bd_num = hns3_skb_bd_num(skb, bd_size, bd_num);
1292        if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)
1293                return bd_num;
1294
1295        skb_walk_frags(skb, frag_skb) {
1296                bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num,
1297                                        bd_num, recursion_level + 1);
1298                if (bd_num > HNS3_MAX_TSO_BD_NUM)
1299                        return bd_num;
1300        }
1301
1302        return bd_num;
1303}
1304
1305static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)
1306{
1307        if (!skb->encapsulation)
1308                return skb_transport_offset(skb) + tcp_hdrlen(skb);
1309
1310        return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
1311}
1312
1313/* HW need every continuous max_non_tso_bd_num buffer data to be larger
1314 * than MSS, we simplify it by ensuring skb_headlen + the first continuous
1315 * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss,
1316 * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger
1317 * than MSS except the last max_non_tso_bd_num - 1 frags.
1318 */
1319static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,
1320                                     unsigned int bd_num, u8 max_non_tso_bd_num)
1321{
1322        unsigned int tot_len = 0;
1323        int i;
1324
1325        for (i = 0; i < max_non_tso_bd_num - 1U; i++)
1326                tot_len += bd_size[i];
1327
1328        /* ensure the first max_non_tso_bd_num frags is greater than
1329         * mss + header
1330         */
1331        if (tot_len + bd_size[max_non_tso_bd_num - 1U] <
1332            skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb))
1333                return true;
1334
1335        /* ensure every continuous max_non_tso_bd_num - 1 buffer is greater
1336         * than mss except the last one.
1337         */
1338        for (i = 0; i < bd_num - max_non_tso_bd_num; i++) {
1339                tot_len -= bd_size[i];
1340                tot_len += bd_size[i + max_non_tso_bd_num - 1U];
1341
1342                if (tot_len < skb_shinfo(skb)->gso_size)
1343                        return true;
1344        }
1345
1346        return false;
1347}
1348
1349void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)
1350{
1351        int i;
1352
1353        for (i = 0; i < MAX_SKB_FRAGS; i++)
1354                size[i] = skb_frag_size(&shinfo->frags[i]);
1355}
1356
1357static int hns3_skb_linearize(struct hns3_enet_ring *ring,
1358                              struct sk_buff *skb,
1359                              u8 max_non_tso_bd_num,
1360                              unsigned int bd_num)
1361{
1362        /* 'bd_num == UINT_MAX' means the skb' fraglist has a
1363         * recursion level of over HNS3_MAX_RECURSION_LEVEL.
1364         */
1365        if (bd_num == UINT_MAX) {
1366                u64_stats_update_begin(&ring->syncp);
1367                ring->stats.over_max_recursion++;
1368                u64_stats_update_end(&ring->syncp);
1369                return -ENOMEM;
1370        }
1371
1372        /* The skb->len has exceeded the hw limitation, linearization
1373         * will not help.
1374         */
1375        if (skb->len > HNS3_MAX_TSO_SIZE ||
1376            (!skb_is_gso(skb) && skb->len >
1377             HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num))) {
1378                u64_stats_update_begin(&ring->syncp);
1379                ring->stats.hw_limitation++;
1380                u64_stats_update_end(&ring->syncp);
1381                return -ENOMEM;
1382        }
1383
1384        if (__skb_linearize(skb)) {
1385                u64_stats_update_begin(&ring->syncp);
1386                ring->stats.sw_err_cnt++;
1387                u64_stats_update_end(&ring->syncp);
1388                return -ENOMEM;
1389        }
1390
1391        return 0;
1392}
1393
1394static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,
1395                                  struct net_device *netdev,
1396                                  struct sk_buff *skb)
1397{
1398        struct hns3_nic_priv *priv = netdev_priv(netdev);
1399        u8 max_non_tso_bd_num = priv->max_non_tso_bd_num;
1400        unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];
1401        unsigned int bd_num;
1402
1403        bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0);
1404        if (unlikely(bd_num > max_non_tso_bd_num)) {
1405                if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&
1406                    !hns3_skb_need_linearized(skb, bd_size, bd_num,
1407                                              max_non_tso_bd_num)) {
1408                        trace_hns3_over_max_bd(skb);
1409                        goto out;
1410                }
1411
1412                if (hns3_skb_linearize(ring, skb, max_non_tso_bd_num,
1413                                       bd_num))
1414                        return -ENOMEM;
1415
1416                bd_num = hns3_tx_bd_count(skb->len);
1417
1418                u64_stats_update_begin(&ring->syncp);
1419                ring->stats.tx_copy++;
1420                u64_stats_update_end(&ring->syncp);
1421        }
1422
1423out:
1424        if (likely(ring_space(ring) >= bd_num))
1425                return bd_num;
1426
1427        netif_stop_subqueue(netdev, ring->queue_index);
1428        smp_mb(); /* Memory barrier before checking ring_space */
1429
1430        /* Start queue in case hns3_clean_tx_ring has just made room
1431         * available and has not seen the queue stopped state performed
1432         * by netif_stop_subqueue above.
1433         */
1434        if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&
1435            !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
1436                netif_start_subqueue(netdev, ring->queue_index);
1437                return bd_num;
1438        }
1439
1440        u64_stats_update_begin(&ring->syncp);
1441        ring->stats.tx_busy++;
1442        u64_stats_update_end(&ring->syncp);
1443
1444        return -EBUSY;
1445}
1446
1447static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1448{
1449        struct device *dev = ring_to_dev(ring);
1450        unsigned int i;
1451
1452        for (i = 0; i < ring->desc_num; i++) {
1453                struct hns3_desc *desc = &ring->desc[ring->next_to_use];
1454
1455                memset(desc, 0, sizeof(*desc));
1456
1457                /* check if this is where we started */
1458                if (ring->next_to_use == next_to_use_orig)
1459                        break;
1460
1461                /* rollback one */
1462                ring_ptr_move_bw(ring, next_to_use);
1463
1464                if (!ring->desc_cb[ring->next_to_use].dma)
1465                        continue;
1466
1467                /* unmap the descriptor dma address */
1468                if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB ||
1469                    ring->desc_cb[ring->next_to_use].type ==
1470                    DESC_TYPE_FRAGLIST_SKB)
1471                        dma_unmap_single(dev,
1472                                         ring->desc_cb[ring->next_to_use].dma,
1473                                        ring->desc_cb[ring->next_to_use].length,
1474                                        DMA_TO_DEVICE);
1475                else if (ring->desc_cb[ring->next_to_use].length)
1476                        dma_unmap_page(dev,
1477                                       ring->desc_cb[ring->next_to_use].dma,
1478                                       ring->desc_cb[ring->next_to_use].length,
1479                                       DMA_TO_DEVICE);
1480
1481                ring->desc_cb[ring->next_to_use].length = 0;
1482                ring->desc_cb[ring->next_to_use].dma = 0;
1483                ring->desc_cb[ring->next_to_use].type = DESC_TYPE_UNKNOWN;
1484        }
1485}
1486
1487static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,
1488                                 struct sk_buff *skb, enum hns_desc_type type)
1489{
1490        unsigned int size = skb_headlen(skb);
1491        struct sk_buff *frag_skb;
1492        int i, ret, bd_num = 0;
1493
1494        if (size) {
1495                ret = hns3_fill_desc(ring, skb, size, type);
1496                if (unlikely(ret < 0))
1497                        return ret;
1498
1499                bd_num += ret;
1500        }
1501
1502        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1503                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1504
1505                size = skb_frag_size(frag);
1506                if (!size)
1507                        continue;
1508
1509                ret = hns3_fill_desc(ring, frag, size, DESC_TYPE_PAGE);
1510                if (unlikely(ret < 0))
1511                        return ret;
1512
1513                bd_num += ret;
1514        }
1515
1516        skb_walk_frags(skb, frag_skb) {
1517                ret = hns3_fill_skb_to_desc(ring, frag_skb,
1518                                            DESC_TYPE_FRAGLIST_SKB);
1519                if (unlikely(ret < 0))
1520                        return ret;
1521
1522                bd_num += ret;
1523        }
1524
1525        return bd_num;
1526}
1527
1528static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,
1529                             bool doorbell)
1530{
1531        ring->pending_buf += num;
1532
1533        if (!doorbell) {
1534                u64_stats_update_begin(&ring->syncp);
1535                ring->stats.tx_more++;
1536                u64_stats_update_end(&ring->syncp);
1537                return;
1538        }
1539
1540        if (!ring->pending_buf)
1541                return;
1542
1543        writel(ring->pending_buf,
1544               ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG);
1545        ring->pending_buf = 0;
1546        WRITE_ONCE(ring->last_to_use, ring->next_to_use);
1547}
1548
1549netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1550{
1551        struct hns3_nic_priv *priv = netdev_priv(netdev);
1552        struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];
1553        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1554        struct netdev_queue *dev_queue;
1555        int pre_ntu, next_to_use_head;
1556        bool doorbell;
1557        int ret;
1558
1559        /* Hardware can only handle short frames above 32 bytes */
1560        if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {
1561                hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
1562
1563                u64_stats_update_begin(&ring->syncp);
1564                ring->stats.sw_err_cnt++;
1565                u64_stats_update_end(&ring->syncp);
1566
1567                return NETDEV_TX_OK;
1568        }
1569
1570        /* Prefetch the data used later */
1571        prefetch(skb->data);
1572
1573        ret = hns3_nic_maybe_stop_tx(ring, netdev, skb);
1574        if (unlikely(ret <= 0)) {
1575                if (ret == -EBUSY) {
1576                        hns3_tx_doorbell(ring, 0, true);
1577                        return NETDEV_TX_BUSY;
1578                }
1579
1580                hns3_rl_err(netdev, "xmit error: %d!\n", ret);
1581                goto out_err_tx_ok;
1582        }
1583
1584        next_to_use_head = ring->next_to_use;
1585
1586        ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use],
1587                                 desc_cb);
1588        if (unlikely(ret < 0))
1589                goto fill_err;
1590
1591        /* 'ret < 0' means filling error, 'ret == 0' means skb->len is
1592         * zero, which is unlikely, and 'ret > 0' means how many tx desc
1593         * need to be notified to the hw.
1594         */
1595        ret = hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);
1596        if (unlikely(ret <= 0))
1597                goto fill_err;
1598
1599        pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) :
1600                                        (ring->desc_num - 1);
1601        ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |=
1602                                cpu_to_le16(BIT(HNS3_TXD_FE_B));
1603        trace_hns3_tx_desc(ring, pre_ntu);
1604
1605        /* Complete translate all packets */
1606        dev_queue = netdev_get_tx_queue(netdev, ring->queue_index);
1607        doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes,
1608                                          netdev_xmit_more());
1609        hns3_tx_doorbell(ring, ret, doorbell);
1610
1611        return NETDEV_TX_OK;
1612
1613fill_err:
1614        hns3_clear_desc(ring, next_to_use_head);
1615
1616out_err_tx_ok:
1617        dev_kfree_skb_any(skb);
1618        hns3_tx_doorbell(ring, 0, !netdev_xmit_more());
1619        return NETDEV_TX_OK;
1620}
1621
1622static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1623{
1624        struct hnae3_handle *h = hns3_get_handle(netdev);
1625        struct sockaddr *mac_addr = p;
1626        int ret;
1627
1628        if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1629                return -EADDRNOTAVAIL;
1630
1631        if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1632                netdev_info(netdev, "already using mac address %pM\n",
1633                            mac_addr->sa_data);
1634                return 0;
1635        }
1636
1637        /* For VF device, if there is a perm_addr, then the user will not
1638         * be allowed to change the address.
1639         */
1640        if (!hns3_is_phys_func(h->pdev) &&
1641            !is_zero_ether_addr(netdev->perm_addr)) {
1642                netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n",
1643                           netdev->perm_addr, mac_addr->sa_data);
1644                return -EPERM;
1645        }
1646
1647        ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1648        if (ret) {
1649                netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1650                return ret;
1651        }
1652
1653        ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1654
1655        return 0;
1656}
1657
1658static int hns3_nic_do_ioctl(struct net_device *netdev,
1659                             struct ifreq *ifr, int cmd)
1660{
1661        struct hnae3_handle *h = hns3_get_handle(netdev);
1662
1663        if (!netif_running(netdev))
1664                return -EINVAL;
1665
1666        if (!h->ae_algo->ops->do_ioctl)
1667                return -EOPNOTSUPP;
1668
1669        return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1670}
1671
1672static int hns3_nic_set_features(struct net_device *netdev,
1673                                 netdev_features_t features)
1674{
1675        netdev_features_t changed = netdev->features ^ features;
1676        struct hns3_nic_priv *priv = netdev_priv(netdev);
1677        struct hnae3_handle *h = priv->ae_handle;
1678        bool enable;
1679        int ret;
1680
1681        if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
1682                enable = !!(features & NETIF_F_GRO_HW);
1683                ret = h->ae_algo->ops->set_gro_en(h, enable);
1684                if (ret)
1685                        return ret;
1686        }
1687
1688        if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1689            h->ae_algo->ops->enable_hw_strip_rxvtag) {
1690                enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1691                ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable);
1692                if (ret)
1693                        return ret;
1694        }
1695
1696        if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1697                enable = !!(features & NETIF_F_NTUPLE);
1698                h->ae_algo->ops->enable_fd(h, enable);
1699        }
1700
1701        if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&
1702            h->ae_algo->ops->cls_flower_active(h)) {
1703                netdev_err(netdev,
1704                           "there are offloaded TC filters active, cannot disable HW TC offload");
1705                return -EINVAL;
1706        }
1707
1708        netdev->features = features;
1709        return 0;
1710}
1711
1712static netdev_features_t hns3_features_check(struct sk_buff *skb,
1713                                             struct net_device *dev,
1714                                             netdev_features_t features)
1715{
1716#define HNS3_MAX_HDR_LEN        480U
1717#define HNS3_MAX_L4_HDR_LEN     60U
1718
1719        size_t len;
1720
1721        if (skb->ip_summed != CHECKSUM_PARTIAL)
1722                return features;
1723
1724        if (skb->encapsulation)
1725                len = skb_inner_transport_header(skb) - skb->data;
1726        else
1727                len = skb_transport_header(skb) - skb->data;
1728
1729        /* Assume L4 is 60 byte as TCP is the only protocol with a
1730         * a flexible value, and it's max len is 60 bytes.
1731         */
1732        len += HNS3_MAX_L4_HDR_LEN;
1733
1734        /* Hardware only supports checksum on the skb with a max header
1735         * len of 480 bytes.
1736         */
1737        if (len > HNS3_MAX_HDR_LEN)
1738                features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1739
1740        return features;
1741}
1742
1743static void hns3_nic_get_stats64(struct net_device *netdev,
1744                                 struct rtnl_link_stats64 *stats)
1745{
1746        struct hns3_nic_priv *priv = netdev_priv(netdev);
1747        int queue_num = priv->ae_handle->kinfo.num_tqps;
1748        struct hnae3_handle *handle = priv->ae_handle;
1749        struct hns3_enet_ring *ring;
1750        u64 rx_length_errors = 0;
1751        u64 rx_crc_errors = 0;
1752        u64 rx_multicast = 0;
1753        unsigned int start;
1754        u64 tx_errors = 0;
1755        u64 rx_errors = 0;
1756        unsigned int idx;
1757        u64 tx_bytes = 0;
1758        u64 rx_bytes = 0;
1759        u64 tx_pkts = 0;
1760        u64 rx_pkts = 0;
1761        u64 tx_drop = 0;
1762        u64 rx_drop = 0;
1763
1764        if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1765                return;
1766
1767        handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1768
1769        for (idx = 0; idx < queue_num; idx++) {
1770                /* fetch the tx stats */
1771                ring = &priv->ring[idx];
1772                do {
1773                        start = u64_stats_fetch_begin_irq(&ring->syncp);
1774                        tx_bytes += ring->stats.tx_bytes;
1775                        tx_pkts += ring->stats.tx_pkts;
1776                        tx_drop += ring->stats.sw_err_cnt;
1777                        tx_drop += ring->stats.tx_vlan_err;
1778                        tx_drop += ring->stats.tx_l4_proto_err;
1779                        tx_drop += ring->stats.tx_l2l3l4_err;
1780                        tx_drop += ring->stats.tx_tso_err;
1781                        tx_drop += ring->stats.over_max_recursion;
1782                        tx_drop += ring->stats.hw_limitation;
1783                        tx_errors += ring->stats.sw_err_cnt;
1784                        tx_errors += ring->stats.tx_vlan_err;
1785                        tx_errors += ring->stats.tx_l4_proto_err;
1786                        tx_errors += ring->stats.tx_l2l3l4_err;
1787                        tx_errors += ring->stats.tx_tso_err;
1788                        tx_errors += ring->stats.over_max_recursion;
1789                        tx_errors += ring->stats.hw_limitation;
1790                } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1791
1792                /* fetch the rx stats */
1793                ring = &priv->ring[idx + queue_num];
1794                do {
1795                        start = u64_stats_fetch_begin_irq(&ring->syncp);
1796                        rx_bytes += ring->stats.rx_bytes;
1797                        rx_pkts += ring->stats.rx_pkts;
1798                        rx_drop += ring->stats.l2_err;
1799                        rx_errors += ring->stats.l2_err;
1800                        rx_errors += ring->stats.l3l4_csum_err;
1801                        rx_crc_errors += ring->stats.l2_err;
1802                        rx_multicast += ring->stats.rx_multicast;
1803                        rx_length_errors += ring->stats.err_pkt_len;
1804                } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1805        }
1806
1807        stats->tx_bytes = tx_bytes;
1808        stats->tx_packets = tx_pkts;
1809        stats->rx_bytes = rx_bytes;
1810        stats->rx_packets = rx_pkts;
1811
1812        stats->rx_errors = rx_errors;
1813        stats->multicast = rx_multicast;
1814        stats->rx_length_errors = rx_length_errors;
1815        stats->rx_crc_errors = rx_crc_errors;
1816        stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1817
1818        stats->tx_errors = tx_errors;
1819        stats->rx_dropped = rx_drop;
1820        stats->tx_dropped = tx_drop;
1821        stats->collisions = netdev->stats.collisions;
1822        stats->rx_over_errors = netdev->stats.rx_over_errors;
1823        stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1824        stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1825        stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1826        stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1827        stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1828        stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1829        stats->tx_window_errors = netdev->stats.tx_window_errors;
1830        stats->rx_compressed = netdev->stats.rx_compressed;
1831        stats->tx_compressed = netdev->stats.tx_compressed;
1832}
1833
1834static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1835{
1836        struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1837        struct hnae3_knic_private_info *kinfo;
1838        u8 tc = mqprio_qopt->qopt.num_tc;
1839        u16 mode = mqprio_qopt->mode;
1840        u8 hw = mqprio_qopt->qopt.hw;
1841        struct hnae3_handle *h;
1842
1843        if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1844               mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1845                return -EOPNOTSUPP;
1846
1847        if (tc > HNAE3_MAX_TC)
1848                return -EINVAL;
1849
1850        if (!netdev)
1851                return -EINVAL;
1852
1853        h = hns3_get_handle(netdev);
1854        kinfo = &h->kinfo;
1855
1856        netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc);
1857
1858        return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1859                kinfo->dcb_ops->setup_tc(h, mqprio_qopt) : -EOPNOTSUPP;
1860}
1861
1862static int hns3_setup_tc_cls_flower(struct hns3_nic_priv *priv,
1863                                    struct flow_cls_offload *flow)
1864{
1865        int tc = tc_classid_to_hwtc(priv->netdev, flow->classid);
1866        struct hnae3_handle *h = hns3_get_handle(priv->netdev);
1867
1868        switch (flow->command) {
1869        case FLOW_CLS_REPLACE:
1870                if (h->ae_algo->ops->add_cls_flower)
1871                        return h->ae_algo->ops->add_cls_flower(h, flow, tc);
1872                break;
1873        case FLOW_CLS_DESTROY:
1874                if (h->ae_algo->ops->del_cls_flower)
1875                        return h->ae_algo->ops->del_cls_flower(h, flow);
1876                break;
1877        default:
1878                break;
1879        }
1880
1881        return -EOPNOTSUPP;
1882}
1883
1884static int hns3_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
1885                                  void *cb_priv)
1886{
1887        struct hns3_nic_priv *priv = cb_priv;
1888
1889        if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
1890                return -EOPNOTSUPP;
1891
1892        switch (type) {
1893        case TC_SETUP_CLSFLOWER:
1894                return hns3_setup_tc_cls_flower(priv, type_data);
1895        default:
1896                return -EOPNOTSUPP;
1897        }
1898}
1899
1900static LIST_HEAD(hns3_block_cb_list);
1901
1902static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1903                             void *type_data)
1904{
1905        struct hns3_nic_priv *priv = netdev_priv(dev);
1906        int ret;
1907
1908        switch (type) {
1909        case TC_SETUP_QDISC_MQPRIO:
1910                ret = hns3_setup_tc(dev, type_data);
1911                break;
1912        case TC_SETUP_BLOCK:
1913                ret = flow_block_cb_setup_simple(type_data,
1914                                                 &hns3_block_cb_list,
1915                                                 hns3_setup_tc_block_cb,
1916                                                 priv, priv, true);
1917                break;
1918        default:
1919                return -EOPNOTSUPP;
1920        }
1921
1922        return ret;
1923}
1924
1925static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1926                                __be16 proto, u16 vid)
1927{
1928        struct hnae3_handle *h = hns3_get_handle(netdev);
1929        int ret = -EIO;
1930
1931        if (h->ae_algo->ops->set_vlan_filter)
1932                ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1933
1934        return ret;
1935}
1936
1937static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1938                                 __be16 proto, u16 vid)
1939{
1940        struct hnae3_handle *h = hns3_get_handle(netdev);
1941        int ret = -EIO;
1942
1943        if (h->ae_algo->ops->set_vlan_filter)
1944                ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1945
1946        return ret;
1947}
1948
1949static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1950                                u8 qos, __be16 vlan_proto)
1951{
1952        struct hnae3_handle *h = hns3_get_handle(netdev);
1953        int ret = -EIO;
1954
1955        netif_dbg(h, drv, netdev,
1956                  "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",
1957                  vf, vlan, qos, ntohs(vlan_proto));
1958
1959        if (h->ae_algo->ops->set_vf_vlan_filter)
1960                ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1961                                                          qos, vlan_proto);
1962
1963        return ret;
1964}
1965
1966static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
1967{
1968        struct hnae3_handle *handle = hns3_get_handle(netdev);
1969
1970        if (hns3_nic_resetting(netdev))
1971                return -EBUSY;
1972
1973        if (!handle->ae_algo->ops->set_vf_spoofchk)
1974                return -EOPNOTSUPP;
1975
1976        return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable);
1977}
1978
1979static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable)
1980{
1981        struct hnae3_handle *handle = hns3_get_handle(netdev);
1982
1983        if (!handle->ae_algo->ops->set_vf_trust)
1984                return -EOPNOTSUPP;
1985
1986        return handle->ae_algo->ops->set_vf_trust(handle, vf, enable);
1987}
1988
1989static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1990{
1991        struct hnae3_handle *h = hns3_get_handle(netdev);
1992        int ret;
1993
1994        if (hns3_nic_resetting(netdev))
1995                return -EBUSY;
1996
1997        if (!h->ae_algo->ops->set_mtu)
1998                return -EOPNOTSUPP;
1999
2000        netif_dbg(h, drv, netdev,
2001                  "change mtu from %u to %d\n", netdev->mtu, new_mtu);
2002
2003        ret = h->ae_algo->ops->set_mtu(h, new_mtu);
2004        if (ret)
2005                netdev_err(netdev, "failed to change MTU in hardware %d\n",
2006                           ret);
2007        else
2008                netdev->mtu = new_mtu;
2009
2010        return ret;
2011}
2012
2013static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
2014{
2015        struct hns3_nic_priv *priv = netdev_priv(ndev);
2016        struct hnae3_handle *h = hns3_get_handle(ndev);
2017        struct hns3_enet_ring *tx_ring;
2018        struct napi_struct *napi;
2019        int timeout_queue = 0;
2020        int hw_head, hw_tail;
2021        int fbd_num, fbd_oft;
2022        int ebd_num, ebd_oft;
2023        int bd_num, bd_err;
2024        int ring_en, tc;
2025        int i;
2026
2027        /* Find the stopped queue the same way the stack does */
2028        for (i = 0; i < ndev->num_tx_queues; i++) {
2029                struct netdev_queue *q;
2030                unsigned long trans_start;
2031
2032                q = netdev_get_tx_queue(ndev, i);
2033                trans_start = q->trans_start;
2034                if (netif_xmit_stopped(q) &&
2035                    time_after(jiffies,
2036                               (trans_start + ndev->watchdog_timeo))) {
2037                        timeout_queue = i;
2038                        netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",
2039                                    q->state,
2040                                    jiffies_to_msecs(jiffies - trans_start));
2041                        break;
2042                }
2043        }
2044
2045        if (i == ndev->num_tx_queues) {
2046                netdev_info(ndev,
2047                            "no netdev TX timeout queue found, timeout count: %llu\n",
2048                            priv->tx_timeout_count);
2049                return false;
2050        }
2051
2052        priv->tx_timeout_count++;
2053
2054        tx_ring = &priv->ring[timeout_queue];
2055        napi = &tx_ring->tqp_vector->napi;
2056
2057        netdev_info(ndev,
2058                    "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",
2059                    priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use,
2060                    tx_ring->next_to_clean, napi->state);
2061
2062        netdev_info(ndev,
2063                    "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n",
2064                    tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes,
2065                    tx_ring->stats.sw_err_cnt, tx_ring->pending_buf);
2066
2067        netdev_info(ndev,
2068                    "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n",
2069                    tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more,
2070                    tx_ring->stats.restart_queue, tx_ring->stats.tx_busy);
2071
2072        /* When mac received many pause frames continuous, it's unable to send
2073         * packets, which may cause tx timeout
2074         */
2075        if (h->ae_algo->ops->get_mac_stats) {
2076                struct hns3_mac_stats mac_stats;
2077
2078                h->ae_algo->ops->get_mac_stats(h, &mac_stats);
2079                netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",
2080                            mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt);
2081        }
2082
2083        hw_head = readl_relaxed(tx_ring->tqp->io_base +
2084                                HNS3_RING_TX_RING_HEAD_REG);
2085        hw_tail = readl_relaxed(tx_ring->tqp->io_base +
2086                                HNS3_RING_TX_RING_TAIL_REG);
2087        fbd_num = readl_relaxed(tx_ring->tqp->io_base +
2088                                HNS3_RING_TX_RING_FBDNUM_REG);
2089        fbd_oft = readl_relaxed(tx_ring->tqp->io_base +
2090                                HNS3_RING_TX_RING_OFFSET_REG);
2091        ebd_num = readl_relaxed(tx_ring->tqp->io_base +
2092                                HNS3_RING_TX_RING_EBDNUM_REG);
2093        ebd_oft = readl_relaxed(tx_ring->tqp->io_base +
2094                                HNS3_RING_TX_RING_EBD_OFFSET_REG);
2095        bd_num = readl_relaxed(tx_ring->tqp->io_base +
2096                               HNS3_RING_TX_RING_BD_NUM_REG);
2097        bd_err = readl_relaxed(tx_ring->tqp->io_base +
2098                               HNS3_RING_TX_RING_BD_ERR_REG);
2099        ring_en = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_EN_REG);
2100        tc = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_TX_RING_TC_REG);
2101
2102        netdev_info(ndev,
2103                    "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",
2104                    bd_num, hw_head, hw_tail, bd_err,
2105                    readl(tx_ring->tqp_vector->mask_addr));
2106        netdev_info(ndev,
2107                    "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",
2108                    ring_en, tc, fbd_num, fbd_oft, ebd_num, ebd_oft);
2109
2110        return true;
2111}
2112
2113static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)
2114{
2115        struct hns3_nic_priv *priv = netdev_priv(ndev);
2116        struct hnae3_handle *h = priv->ae_handle;
2117
2118        if (!hns3_get_tx_timeo_queue_info(ndev))
2119                return;
2120
2121        /* request the reset, and let the hclge to determine
2122         * which reset level should be done
2123         */
2124        if (h->ae_algo->ops->reset_event)
2125                h->ae_algo->ops->reset_event(h->pdev, h);
2126}
2127
2128#ifdef CONFIG_RFS_ACCEL
2129static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
2130                              u16 rxq_index, u32 flow_id)
2131{
2132        struct hnae3_handle *h = hns3_get_handle(dev);
2133        struct flow_keys fkeys;
2134
2135        if (!h->ae_algo->ops->add_arfs_entry)
2136                return -EOPNOTSUPP;
2137
2138        if (skb->encapsulation)
2139                return -EPROTONOSUPPORT;
2140
2141        if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0))
2142                return -EPROTONOSUPPORT;
2143
2144        if ((fkeys.basic.n_proto != htons(ETH_P_IP) &&
2145             fkeys.basic.n_proto != htons(ETH_P_IPV6)) ||
2146            (fkeys.basic.ip_proto != IPPROTO_TCP &&
2147             fkeys.basic.ip_proto != IPPROTO_UDP))
2148                return -EPROTONOSUPPORT;
2149
2150        return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys);
2151}
2152#endif
2153
2154static int hns3_nic_get_vf_config(struct net_device *ndev, int vf,
2155                                  struct ifla_vf_info *ivf)
2156{
2157        struct hnae3_handle *h = hns3_get_handle(ndev);
2158
2159        if (!h->ae_algo->ops->get_vf_config)
2160                return -EOPNOTSUPP;
2161
2162        return h->ae_algo->ops->get_vf_config(h, vf, ivf);
2163}
2164
2165static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf,
2166                                      int link_state)
2167{
2168        struct hnae3_handle *h = hns3_get_handle(ndev);
2169
2170        if (!h->ae_algo->ops->set_vf_link_state)
2171                return -EOPNOTSUPP;
2172
2173        return h->ae_algo->ops->set_vf_link_state(h, vf, link_state);
2174}
2175
2176static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,
2177                                int min_tx_rate, int max_tx_rate)
2178{
2179        struct hnae3_handle *h = hns3_get_handle(ndev);
2180
2181        if (!h->ae_algo->ops->set_vf_rate)
2182                return -EOPNOTSUPP;
2183
2184        return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate,
2185                                            false);
2186}
2187
2188static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
2189{
2190        struct hnae3_handle *h = hns3_get_handle(netdev);
2191
2192        if (!h->ae_algo->ops->set_vf_mac)
2193                return -EOPNOTSUPP;
2194
2195        if (is_multicast_ether_addr(mac)) {
2196                netdev_err(netdev,
2197                           "Invalid MAC:%pM specified. Could not set MAC\n",
2198                           mac);
2199                return -EINVAL;
2200        }
2201
2202        return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);
2203}
2204
2205static const struct net_device_ops hns3_nic_netdev_ops = {
2206        .ndo_open               = hns3_nic_net_open,
2207        .ndo_stop               = hns3_nic_net_stop,
2208        .ndo_start_xmit         = hns3_nic_net_xmit,
2209        .ndo_tx_timeout         = hns3_nic_net_timeout,
2210        .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
2211        .ndo_do_ioctl           = hns3_nic_do_ioctl,
2212        .ndo_change_mtu         = hns3_nic_change_mtu,
2213        .ndo_set_features       = hns3_nic_set_features,
2214        .ndo_features_check     = hns3_features_check,
2215        .ndo_get_stats64        = hns3_nic_get_stats64,
2216        .ndo_setup_tc           = hns3_nic_setup_tc,
2217        .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
2218        .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
2219        .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
2220        .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
2221        .ndo_set_vf_spoofchk    = hns3_set_vf_spoofchk,
2222        .ndo_set_vf_trust       = hns3_set_vf_trust,
2223#ifdef CONFIG_RFS_ACCEL
2224        .ndo_rx_flow_steer      = hns3_rx_flow_steer,
2225#endif
2226        .ndo_get_vf_config      = hns3_nic_get_vf_config,
2227        .ndo_set_vf_link_state  = hns3_nic_set_vf_link_state,
2228        .ndo_set_vf_rate        = hns3_nic_set_vf_rate,
2229        .ndo_set_vf_mac         = hns3_nic_set_vf_mac,
2230};
2231
2232bool hns3_is_phys_func(struct pci_dev *pdev)
2233{
2234        u32 dev_id = pdev->device;
2235
2236        switch (dev_id) {
2237        case HNAE3_DEV_ID_GE:
2238        case HNAE3_DEV_ID_25GE:
2239        case HNAE3_DEV_ID_25GE_RDMA:
2240        case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
2241        case HNAE3_DEV_ID_50GE_RDMA:
2242        case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
2243        case HNAE3_DEV_ID_100G_RDMA_MACSEC:
2244        case HNAE3_DEV_ID_200G_RDMA:
2245                return true;
2246        case HNAE3_DEV_ID_VF:
2247        case HNAE3_DEV_ID_RDMA_DCB_PFC_VF:
2248                return false;
2249        default:
2250                dev_warn(&pdev->dev, "un-recognized pci device-id %u",
2251                         dev_id);
2252        }
2253
2254        return false;
2255}
2256
2257static void hns3_disable_sriov(struct pci_dev *pdev)
2258{
2259        /* If our VFs are assigned we cannot shut down SR-IOV
2260         * without causing issues, so just leave the hardware
2261         * available but disabled
2262         */
2263        if (pci_vfs_assigned(pdev)) {
2264                dev_warn(&pdev->dev,
2265                         "disabling driver while VFs are assigned\n");
2266                return;
2267        }
2268
2269        pci_disable_sriov(pdev);
2270}
2271
2272/* hns3_probe - Device initialization routine
2273 * @pdev: PCI device information struct
2274 * @ent: entry in hns3_pci_tbl
2275 *
2276 * hns3_probe initializes a PF identified by a pci_dev structure.
2277 * The OS initialization, configuring of the PF private structure,
2278 * and a hardware reset occur.
2279 *
2280 * Returns 0 on success, negative on failure
2281 */
2282static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2283{
2284        struct hnae3_ae_dev *ae_dev;
2285        int ret;
2286
2287        ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL);
2288        if (!ae_dev)
2289                return -ENOMEM;
2290
2291        ae_dev->pdev = pdev;
2292        ae_dev->flag = ent->driver_data;
2293        pci_set_drvdata(pdev, ae_dev);
2294
2295        ret = hnae3_register_ae_dev(ae_dev);
2296        if (ret)
2297                pci_set_drvdata(pdev, NULL);
2298
2299        return ret;
2300}
2301
2302/* hns3_remove - Device removal routine
2303 * @pdev: PCI device information struct
2304 */
2305static void hns3_remove(struct pci_dev *pdev)
2306{
2307        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2308
2309        if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
2310                hns3_disable_sriov(pdev);
2311
2312        hnae3_unregister_ae_dev(ae_dev);
2313        pci_set_drvdata(pdev, NULL);
2314}
2315
2316/**
2317 * hns3_pci_sriov_configure
2318 * @pdev: pointer to a pci_dev structure
2319 * @num_vfs: number of VFs to allocate
2320 *
2321 * Enable or change the number of VFs. Called when the user updates the number
2322 * of VFs in sysfs.
2323 **/
2324static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
2325{
2326        int ret;
2327
2328        if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
2329                dev_warn(&pdev->dev, "Can not config SRIOV\n");
2330                return -EINVAL;
2331        }
2332
2333        if (num_vfs) {
2334                ret = pci_enable_sriov(pdev, num_vfs);
2335                if (ret)
2336                        dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
2337                else
2338                        return num_vfs;
2339        } else if (!pci_vfs_assigned(pdev)) {
2340                pci_disable_sriov(pdev);
2341        } else {
2342                dev_warn(&pdev->dev,
2343                         "Unable to free VFs because some are assigned to VMs.\n");
2344        }
2345
2346        return 0;
2347}
2348
2349static void hns3_shutdown(struct pci_dev *pdev)
2350{
2351        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2352
2353        hnae3_unregister_ae_dev(ae_dev);
2354        pci_set_drvdata(pdev, NULL);
2355
2356        if (system_state == SYSTEM_POWER_OFF)
2357                pci_set_power_state(pdev, PCI_D3hot);
2358}
2359
2360static int __maybe_unused hns3_suspend(struct device *dev)
2361{
2362        struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
2363
2364        if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
2365                dev_info(dev, "Begin to suspend.\n");
2366                if (ae_dev->ops && ae_dev->ops->reset_prepare)
2367                        ae_dev->ops->reset_prepare(ae_dev, HNAE3_FUNC_RESET);
2368        }
2369
2370        return 0;
2371}
2372
2373static int __maybe_unused hns3_resume(struct device *dev)
2374{
2375        struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);
2376
2377        if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {
2378                dev_info(dev, "Begin to resume.\n");
2379                if (ae_dev->ops && ae_dev->ops->reset_done)
2380                        ae_dev->ops->reset_done(ae_dev);
2381        }
2382
2383        return 0;
2384}
2385
2386static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
2387                                            pci_channel_state_t state)
2388{
2389        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2390        pci_ers_result_t ret;
2391
2392        dev_info(&pdev->dev, "PCI error detected, state(=%u)!!\n", state);
2393
2394        if (state == pci_channel_io_perm_failure)
2395                return PCI_ERS_RESULT_DISCONNECT;
2396
2397        if (!ae_dev || !ae_dev->ops) {
2398                dev_err(&pdev->dev,
2399                        "Can't recover - error happened before device initialized\n");
2400                return PCI_ERS_RESULT_NONE;
2401        }
2402
2403        if (ae_dev->ops->handle_hw_ras_error)
2404                ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
2405        else
2406                return PCI_ERS_RESULT_NONE;
2407
2408        return ret;
2409}
2410
2411static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
2412{
2413        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2414        const struct hnae3_ae_ops *ops;
2415        enum hnae3_reset_type reset_type;
2416        struct device *dev = &pdev->dev;
2417
2418        if (!ae_dev || !ae_dev->ops)
2419                return PCI_ERS_RESULT_NONE;
2420
2421        ops = ae_dev->ops;
2422        /* request the reset */
2423        if (ops->reset_event && ops->get_reset_level &&
2424            ops->set_default_reset_request) {
2425                if (ae_dev->hw_err_reset_req) {
2426                        reset_type = ops->get_reset_level(ae_dev,
2427                                                &ae_dev->hw_err_reset_req);
2428                        ops->set_default_reset_request(ae_dev, reset_type);
2429                        dev_info(dev, "requesting reset due to PCI error\n");
2430                        ops->reset_event(pdev, NULL);
2431                }
2432
2433                return PCI_ERS_RESULT_RECOVERED;
2434        }
2435
2436        return PCI_ERS_RESULT_DISCONNECT;
2437}
2438
2439static void hns3_reset_prepare(struct pci_dev *pdev)
2440{
2441        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2442
2443        dev_info(&pdev->dev, "FLR prepare\n");
2444        if (ae_dev && ae_dev->ops && ae_dev->ops->reset_prepare)
2445                ae_dev->ops->reset_prepare(ae_dev, HNAE3_FLR_RESET);
2446}
2447
2448static void hns3_reset_done(struct pci_dev *pdev)
2449{
2450        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2451
2452        dev_info(&pdev->dev, "FLR done\n");
2453        if (ae_dev && ae_dev->ops && ae_dev->ops->reset_done)
2454                ae_dev->ops->reset_done(ae_dev);
2455}
2456
2457static const struct pci_error_handlers hns3_err_handler = {
2458        .error_detected = hns3_error_detected,
2459        .slot_reset     = hns3_slot_reset,
2460        .reset_prepare  = hns3_reset_prepare,
2461        .reset_done     = hns3_reset_done,
2462};
2463
2464static SIMPLE_DEV_PM_OPS(hns3_pm_ops, hns3_suspend, hns3_resume);
2465
2466static struct pci_driver hns3_driver = {
2467        .name     = hns3_driver_name,
2468        .id_table = hns3_pci_tbl,
2469        .probe    = hns3_probe,
2470        .remove   = hns3_remove,
2471        .shutdown = hns3_shutdown,
2472        .driver.pm  = &hns3_pm_ops,
2473        .sriov_configure = hns3_pci_sriov_configure,
2474        .err_handler    = &hns3_err_handler,
2475};
2476
2477/* set default feature to hns3 */
2478static void hns3_set_default_feature(struct net_device *netdev)
2479{
2480        struct hnae3_handle *h = hns3_get_handle(netdev);
2481        struct pci_dev *pdev = h->pdev;
2482        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
2483
2484        netdev->priv_flags |= IFF_UNICAST_FLT;
2485
2486        netdev->hw_enc_features |= NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2487                NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2488                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2489                NETIF_F_SCTP_CRC | NETIF_F_TSO_MANGLEID | NETIF_F_FRAGLIST;
2490
2491        netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
2492
2493        netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2494                NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2495                NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2496                NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2497                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2498                NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
2499
2500        netdev->vlan_features |= NETIF_F_RXCSUM |
2501                NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
2502                NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2503                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2504                NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
2505
2506        netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
2507                NETIF_F_HW_VLAN_CTAG_RX |
2508                NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
2509                NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
2510                NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
2511                NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
2512
2513        if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) {
2514                netdev->hw_features |= NETIF_F_GRO_HW;
2515                netdev->features |= NETIF_F_GRO_HW;
2516
2517                if (!(h->flags & HNAE3_SUPPORT_VF)) {
2518                        netdev->hw_features |= NETIF_F_NTUPLE;
2519                        netdev->features |= NETIF_F_NTUPLE;
2520                }
2521        }
2522
2523        if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps)) {
2524                netdev->hw_features |= NETIF_F_GSO_UDP_L4;
2525                netdev->features |= NETIF_F_GSO_UDP_L4;
2526                netdev->vlan_features |= NETIF_F_GSO_UDP_L4;
2527                netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
2528        }
2529
2530        if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps)) {
2531                netdev->hw_features |= NETIF_F_HW_CSUM;
2532                netdev->features |= NETIF_F_HW_CSUM;
2533                netdev->vlan_features |= NETIF_F_HW_CSUM;
2534                netdev->hw_enc_features |= NETIF_F_HW_CSUM;
2535        } else {
2536                netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2537                netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2538                netdev->vlan_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2539                netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2540        }
2541
2542        if (test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps)) {
2543                netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2544                netdev->features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2545                netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2546                netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
2547        }
2548
2549        if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) {
2550                netdev->hw_features |= NETIF_F_HW_TC;
2551                netdev->features |= NETIF_F_HW_TC;
2552        }
2553}
2554
2555static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
2556                             struct hns3_desc_cb *cb)
2557{
2558        unsigned int order = hns3_page_order(ring);
2559        struct page *p;
2560
2561        p = dev_alloc_pages(order);
2562        if (!p)
2563                return -ENOMEM;
2564
2565        cb->priv = p;
2566        cb->page_offset = 0;
2567        cb->reuse_flag = 0;
2568        cb->buf  = page_address(p);
2569        cb->length = hns3_page_size(ring);
2570        cb->type = DESC_TYPE_PAGE;
2571        page_ref_add(p, USHRT_MAX - 1);
2572        cb->pagecnt_bias = USHRT_MAX;
2573
2574        return 0;
2575}
2576
2577static void hns3_free_buffer(struct hns3_enet_ring *ring,
2578                             struct hns3_desc_cb *cb, int budget)
2579{
2580        if (cb->type == DESC_TYPE_SKB)
2581                napi_consume_skb(cb->priv, budget);
2582        else if (!HNAE3_IS_TX_RING(ring) && cb->pagecnt_bias)
2583                __page_frag_cache_drain(cb->priv, cb->pagecnt_bias);
2584        memset(cb, 0, sizeof(*cb));
2585}
2586
2587static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
2588{
2589        cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
2590                               cb->length, ring_to_dma_dir(ring));
2591
2592        if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
2593                return -EIO;
2594
2595        return 0;
2596}
2597
2598static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
2599                              struct hns3_desc_cb *cb)
2600{
2601        if (cb->type == DESC_TYPE_SKB || cb->type == DESC_TYPE_FRAGLIST_SKB)
2602                dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
2603                                 ring_to_dma_dir(ring));
2604        else if (cb->length)
2605                dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
2606                               ring_to_dma_dir(ring));
2607}
2608
2609static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
2610{
2611        hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2612        ring->desc[i].addr = 0;
2613}
2614
2615static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i,
2616                                    int budget)
2617{
2618        struct hns3_desc_cb *cb = &ring->desc_cb[i];
2619
2620        if (!ring->desc_cb[i].dma)
2621                return;
2622
2623        hns3_buffer_detach(ring, i);
2624        hns3_free_buffer(ring, cb, budget);
2625}
2626
2627static void hns3_free_buffers(struct hns3_enet_ring *ring)
2628{
2629        int i;
2630
2631        for (i = 0; i < ring->desc_num; i++)
2632                hns3_free_buffer_detach(ring, i, 0);
2633}
2634
2635/* free desc along with its attached buffer */
2636static void hns3_free_desc(struct hns3_enet_ring *ring)
2637{
2638        int size = ring->desc_num * sizeof(ring->desc[0]);
2639
2640        hns3_free_buffers(ring);
2641
2642        if (ring->desc) {
2643                dma_free_coherent(ring_to_dev(ring), size,
2644                                  ring->desc, ring->desc_dma_addr);
2645                ring->desc = NULL;
2646        }
2647}
2648
2649static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2650{
2651        int size = ring->desc_num * sizeof(ring->desc[0]);
2652
2653        ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,
2654                                        &ring->desc_dma_addr, GFP_KERNEL);
2655        if (!ring->desc)
2656                return -ENOMEM;
2657
2658        return 0;
2659}
2660
2661static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring,
2662                                   struct hns3_desc_cb *cb)
2663{
2664        int ret;
2665
2666        ret = hns3_alloc_buffer(ring, cb);
2667        if (ret)
2668                goto out;
2669
2670        ret = hns3_map_buffer(ring, cb);
2671        if (ret)
2672                goto out_with_buf;
2673
2674        return 0;
2675
2676out_with_buf:
2677        hns3_free_buffer(ring, cb, 0);
2678out:
2679        return ret;
2680}
2681
2682static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i)
2683{
2684        int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]);
2685
2686        if (ret)
2687                return ret;
2688
2689        ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2690
2691        return 0;
2692}
2693
2694/* Allocate memory for raw pkg, and map with dma */
2695static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2696{
2697        int i, j, ret;
2698
2699        for (i = 0; i < ring->desc_num; i++) {
2700                ret = hns3_alloc_and_attach_buffer(ring, i);
2701                if (ret)
2702                        goto out_buffer_fail;
2703        }
2704
2705        return 0;
2706
2707out_buffer_fail:
2708        for (j = i - 1; j >= 0; j--)
2709                hns3_free_buffer_detach(ring, j, 0);
2710        return ret;
2711}
2712
2713/* detach a in-used buffer and replace with a reserved one */
2714static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2715                                struct hns3_desc_cb *res_cb)
2716{
2717        hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2718        ring->desc_cb[i] = *res_cb;
2719        ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2720        ring->desc[i].rx.bd_base_info = 0;
2721}
2722
2723static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2724{
2725        ring->desc_cb[i].reuse_flag = 0;
2726        ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +
2727                                         ring->desc_cb[i].page_offset);
2728        ring->desc[i].rx.bd_base_info = 0;
2729
2730        dma_sync_single_for_device(ring_to_dev(ring),
2731                        ring->desc_cb[i].dma + ring->desc_cb[i].page_offset,
2732                        hns3_buf_size(ring),
2733                        DMA_FROM_DEVICE);
2734}
2735
2736static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring,
2737                                  int *bytes, int *pkts, int budget)
2738{
2739        /* pair with ring->last_to_use update in hns3_tx_doorbell(),
2740         * smp_store_release() is not used in hns3_tx_doorbell() because
2741         * the doorbell operation already have the needed barrier operation.
2742         */
2743        int ltu = smp_load_acquire(&ring->last_to_use);
2744        int ntc = ring->next_to_clean;
2745        struct hns3_desc_cb *desc_cb;
2746        bool reclaimed = false;
2747        struct hns3_desc *desc;
2748
2749        while (ltu != ntc) {
2750                desc = &ring->desc[ntc];
2751
2752                if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) &
2753                                BIT(HNS3_TXD_VLD_B))
2754                        break;
2755
2756                desc_cb = &ring->desc_cb[ntc];
2757
2758                if (desc_cb->type == DESC_TYPE_SKB) {
2759                        (*pkts)++;
2760                        (*bytes) += desc_cb->send_bytes;
2761                }
2762
2763                /* desc_cb will be cleaned, after hnae3_free_buffer_detach */
2764                hns3_free_buffer_detach(ring, ntc, budget);
2765
2766                if (++ntc == ring->desc_num)
2767                        ntc = 0;
2768
2769                /* Issue prefetch for next Tx descriptor */
2770                prefetch(&ring->desc_cb[ntc]);
2771                reclaimed = true;
2772        }
2773
2774        if (unlikely(!reclaimed))
2775                return false;
2776
2777        /* This smp_store_release() pairs with smp_load_acquire() in
2778         * ring_space called by hns3_nic_net_xmit.
2779         */
2780        smp_store_release(&ring->next_to_clean, ntc);
2781        return true;
2782}
2783
2784void hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
2785{
2786        struct net_device *netdev = ring_to_netdev(ring);
2787        struct hns3_nic_priv *priv = netdev_priv(netdev);
2788        struct netdev_queue *dev_queue;
2789        int bytes, pkts;
2790
2791        bytes = 0;
2792        pkts = 0;
2793
2794        if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts, budget)))
2795                return;
2796
2797        ring->tqp_vector->tx_group.total_bytes += bytes;
2798        ring->tqp_vector->tx_group.total_packets += pkts;
2799
2800        u64_stats_update_begin(&ring->syncp);
2801        ring->stats.tx_bytes += bytes;
2802        ring->stats.tx_pkts += pkts;
2803        u64_stats_update_end(&ring->syncp);
2804
2805        dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2806        netdev_tx_completed_queue(dev_queue, pkts, bytes);
2807
2808        if (unlikely(netif_carrier_ok(netdev) &&
2809                     ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) {
2810                /* Make sure that anybody stopping the queue after this
2811                 * sees the new next_to_clean.
2812                 */
2813                smp_mb();
2814                if (netif_tx_queue_stopped(dev_queue) &&
2815                    !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2816                        netif_tx_wake_queue(dev_queue);
2817                        ring->stats.restart_queue++;
2818                }
2819        }
2820}
2821
2822static int hns3_desc_unused(struct hns3_enet_ring *ring)
2823{
2824        int ntc = ring->next_to_clean;
2825        int ntu = ring->next_to_use;
2826
2827        return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2828}
2829
2830static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring,
2831                                      int cleand_count)
2832{
2833        struct hns3_desc_cb *desc_cb;
2834        struct hns3_desc_cb res_cbs;
2835        int i, ret;
2836
2837        for (i = 0; i < cleand_count; i++) {
2838                desc_cb = &ring->desc_cb[ring->next_to_use];
2839                if (desc_cb->reuse_flag) {
2840                        u64_stats_update_begin(&ring->syncp);
2841                        ring->stats.reuse_pg_cnt++;
2842                        u64_stats_update_end(&ring->syncp);
2843
2844                        hns3_reuse_buffer(ring, ring->next_to_use);
2845                } else {
2846                        ret = hns3_alloc_and_map_buffer(ring, &res_cbs);
2847                        if (ret) {
2848                                u64_stats_update_begin(&ring->syncp);
2849                                ring->stats.sw_err_cnt++;
2850                                u64_stats_update_end(&ring->syncp);
2851
2852                                hns3_rl_err(ring_to_netdev(ring),
2853                                            "alloc rx buffer failed: %d\n",
2854                                            ret);
2855                                break;
2856                        }
2857                        hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2858
2859                        u64_stats_update_begin(&ring->syncp);
2860                        ring->stats.non_reuse_pg++;
2861                        u64_stats_update_end(&ring->syncp);
2862                }
2863
2864                ring_ptr_move_fw(ring, next_to_use);
2865        }
2866
2867        writel(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2868}
2869
2870static bool hns3_can_reuse_page(struct hns3_desc_cb *cb)
2871{
2872        return (page_count(cb->priv) - cb->pagecnt_bias) == 1;
2873}
2874
2875static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2876                                struct hns3_enet_ring *ring, int pull_len,
2877                                struct hns3_desc_cb *desc_cb)
2878{
2879        struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2880        int size = le16_to_cpu(desc->rx.size);
2881        u32 truesize = hns3_buf_size(ring);
2882
2883        desc_cb->pagecnt_bias--;
2884        skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2885                        size - pull_len, truesize);
2886
2887        /* Avoid re-using remote and pfmemalloc pages, or the stack is still
2888         * using the page when page_offset rollback to zero, flag default
2889         * unreuse
2890         */
2891        if (!dev_page_is_reusable(desc_cb->priv) ||
2892            (!desc_cb->page_offset && !hns3_can_reuse_page(desc_cb))) {
2893                __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
2894                return;
2895        }
2896
2897        /* Move offset up to the next cache line */
2898        desc_cb->page_offset += truesize;
2899
2900        if (desc_cb->page_offset + truesize <= hns3_page_size(ring)) {
2901                desc_cb->reuse_flag = 1;
2902        } else if (hns3_can_reuse_page(desc_cb)) {
2903                desc_cb->reuse_flag = 1;
2904                desc_cb->page_offset = 0;
2905        } else if (desc_cb->pagecnt_bias) {
2906                __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);
2907                return;
2908        }
2909
2910        if (unlikely(!desc_cb->pagecnt_bias)) {
2911                page_ref_add(desc_cb->priv, USHRT_MAX);
2912                desc_cb->pagecnt_bias = USHRT_MAX;
2913        }
2914}
2915
2916static int hns3_gro_complete(struct sk_buff *skb, u32 l234info)
2917{
2918        __be16 type = skb->protocol;
2919        struct tcphdr *th;
2920        int depth = 0;
2921
2922        while (eth_type_vlan(type)) {
2923                struct vlan_hdr *vh;
2924
2925                if ((depth + VLAN_HLEN) > skb_headlen(skb))
2926                        return -EFAULT;
2927
2928                vh = (struct vlan_hdr *)(skb->data + depth);
2929                type = vh->h_vlan_encapsulated_proto;
2930                depth += VLAN_HLEN;
2931        }
2932
2933        skb_set_network_header(skb, depth);
2934
2935        if (type == htons(ETH_P_IP)) {
2936                const struct iphdr *iph = ip_hdr(skb);
2937
2938                depth += sizeof(struct iphdr);
2939                skb_set_transport_header(skb, depth);
2940                th = tcp_hdr(skb);
2941                th->check = ~tcp_v4_check(skb->len - depth, iph->saddr,
2942                                          iph->daddr, 0);
2943        } else if (type == htons(ETH_P_IPV6)) {
2944                const struct ipv6hdr *iph = ipv6_hdr(skb);
2945
2946                depth += sizeof(struct ipv6hdr);
2947                skb_set_transport_header(skb, depth);
2948                th = tcp_hdr(skb);
2949                th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr,
2950                                          &iph->daddr, 0);
2951        } else {
2952                hns3_rl_err(skb->dev,
2953                            "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",
2954                            be16_to_cpu(type), depth);
2955                return -EFAULT;
2956        }
2957
2958        skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
2959        if (th->cwr)
2960                skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
2961
2962        if (l234info & BIT(HNS3_RXD_GRO_FIXID_B))
2963                skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;
2964
2965        skb->csum_start = (unsigned char *)th - skb->head;
2966        skb->csum_offset = offsetof(struct tcphdr, check);
2967        skb->ip_summed = CHECKSUM_PARTIAL;
2968
2969        trace_hns3_gro(skb);
2970
2971        return 0;
2972}
2973
2974static void hns3_checksum_complete(struct hns3_enet_ring *ring,
2975                                   struct sk_buff *skb, u32 l234info)
2976{
2977        u32 lo, hi;
2978
2979        u64_stats_update_begin(&ring->syncp);
2980        ring->stats.csum_complete++;
2981        u64_stats_update_end(&ring->syncp);
2982        skb->ip_summed = CHECKSUM_COMPLETE;
2983        lo = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_L_M,
2984                             HNS3_RXD_L2_CSUM_L_S);
2985        hi = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_H_M,
2986                             HNS3_RXD_L2_CSUM_H_S);
2987        skb->csum = csum_unfold((__force __sum16)(lo | hi << 8));
2988}
2989
2990static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2991                             u32 l234info, u32 bd_base_info, u32 ol_info)
2992{
2993        struct net_device *netdev = ring_to_netdev(ring);
2994        int l3_type, l4_type;
2995        int ol4_type;
2996
2997        skb->ip_summed = CHECKSUM_NONE;
2998
2999        skb_checksum_none_assert(skb);
3000
3001        if (!(netdev->features & NETIF_F_RXCSUM))
3002                return;
3003
3004        if (l234info & BIT(HNS3_RXD_L2_CSUM_B)) {
3005                hns3_checksum_complete(ring, skb, l234info);
3006                return;
3007        }
3008
3009        /* check if hardware has done checksum */
3010        if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B)))
3011                return;
3012
3013        if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) |
3014                                 BIT(HNS3_RXD_OL3E_B) |
3015                                 BIT(HNS3_RXD_OL4E_B)))) {
3016                u64_stats_update_begin(&ring->syncp);
3017                ring->stats.l3l4_csum_err++;
3018                u64_stats_update_end(&ring->syncp);
3019
3020                return;
3021        }
3022
3023        ol4_type = hnae3_get_field(ol_info, HNS3_RXD_OL4ID_M,
3024                                   HNS3_RXD_OL4ID_S);
3025        switch (ol4_type) {
3026        case HNS3_OL4_TYPE_MAC_IN_UDP:
3027        case HNS3_OL4_TYPE_NVGRE:
3028                skb->csum_level = 1;
3029                fallthrough;
3030        case HNS3_OL4_TYPE_NO_TUN:
3031                l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
3032                                          HNS3_RXD_L3ID_S);
3033                l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
3034                                          HNS3_RXD_L4ID_S);
3035                /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
3036                if ((l3_type == HNS3_L3_TYPE_IPV4 ||
3037                     l3_type == HNS3_L3_TYPE_IPV6) &&
3038                    (l4_type == HNS3_L4_TYPE_UDP ||
3039                     l4_type == HNS3_L4_TYPE_TCP ||
3040                     l4_type == HNS3_L4_TYPE_SCTP))
3041                        skb->ip_summed = CHECKSUM_UNNECESSARY;
3042                break;
3043        default:
3044                break;
3045        }
3046}
3047
3048static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
3049{
3050        if (skb_has_frag_list(skb))
3051                napi_gro_flush(&ring->tqp_vector->napi, false);
3052
3053        napi_gro_receive(&ring->tqp_vector->napi, skb);
3054}
3055
3056static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
3057                                struct hns3_desc *desc, u32 l234info,
3058                                u16 *vlan_tag)
3059{
3060        struct hnae3_handle *handle = ring->tqp->handle;
3061        struct pci_dev *pdev = ring->tqp->handle->pdev;
3062        struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
3063
3064        if (unlikely(ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2)) {
3065                *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3066                if (!(*vlan_tag & VLAN_VID_MASK))
3067                        *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3068
3069                return (*vlan_tag != 0);
3070        }
3071
3072#define HNS3_STRP_OUTER_VLAN    0x1
3073#define HNS3_STRP_INNER_VLAN    0x2
3074#define HNS3_STRP_BOTH          0x3
3075
3076        /* Hardware always insert VLAN tag into RX descriptor when
3077         * remove the tag from packet, driver needs to determine
3078         * reporting which tag to stack.
3079         */
3080        switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
3081                                HNS3_RXD_STRP_TAGP_S)) {
3082        case HNS3_STRP_OUTER_VLAN:
3083                if (handle->port_base_vlan_state !=
3084                                HNAE3_PORT_BASE_VLAN_DISABLE)
3085                        return false;
3086
3087                *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3088                return true;
3089        case HNS3_STRP_INNER_VLAN:
3090                if (handle->port_base_vlan_state !=
3091                                HNAE3_PORT_BASE_VLAN_DISABLE)
3092                        return false;
3093
3094                *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3095                return true;
3096        case HNS3_STRP_BOTH:
3097                if (handle->port_base_vlan_state ==
3098                                HNAE3_PORT_BASE_VLAN_DISABLE)
3099                        *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
3100                else
3101                        *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
3102
3103                return true;
3104        default:
3105                return false;
3106        }
3107}
3108
3109static void hns3_rx_ring_move_fw(struct hns3_enet_ring *ring)
3110{
3111        ring->desc[ring->next_to_clean].rx.bd_base_info &=
3112                cpu_to_le32(~BIT(HNS3_RXD_VLD_B));
3113        ring->next_to_clean += 1;
3114
3115        if (unlikely(ring->next_to_clean == ring->desc_num))
3116                ring->next_to_clean = 0;
3117}
3118
3119static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length,
3120                          unsigned char *va)
3121{
3122        struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
3123        struct net_device *netdev = ring_to_netdev(ring);
3124        struct sk_buff *skb;
3125
3126        ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
3127        skb = ring->skb;
3128        if (unlikely(!skb)) {
3129                hns3_rl_err(netdev, "alloc rx skb fail\n");
3130
3131                u64_stats_update_begin(&ring->syncp);
3132                ring->stats.sw_err_cnt++;
3133                u64_stats_update_end(&ring->syncp);
3134
3135                return -ENOMEM;
3136        }
3137
3138        trace_hns3_rx_desc(ring);
3139        prefetchw(skb->data);
3140
3141        ring->pending_buf = 1;
3142        ring->frag_num = 0;
3143        ring->tail_skb = NULL;
3144        if (length <= HNS3_RX_HEAD_SIZE) {
3145                memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
3146
3147                /* We can reuse buffer as-is, just make sure it is reusable */
3148                if (dev_page_is_reusable(desc_cb->priv))
3149                        desc_cb->reuse_flag = 1;
3150                else /* This page cannot be reused so discard it */
3151                        __page_frag_cache_drain(desc_cb->priv,
3152                                                desc_cb->pagecnt_bias);
3153
3154                hns3_rx_ring_move_fw(ring);
3155                return 0;
3156        }
3157        u64_stats_update_begin(&ring->syncp);
3158        ring->stats.seg_pkt_cnt++;
3159        u64_stats_update_end(&ring->syncp);
3160
3161        ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE);
3162        __skb_put(skb, ring->pull_len);
3163        hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,
3164                            desc_cb);
3165        hns3_rx_ring_move_fw(ring);
3166
3167        return 0;
3168}
3169
3170static int hns3_add_frag(struct hns3_enet_ring *ring)
3171{
3172        struct sk_buff *skb = ring->skb;
3173        struct sk_buff *head_skb = skb;
3174        struct sk_buff *new_skb;
3175        struct hns3_desc_cb *desc_cb;
3176        struct hns3_desc *desc;
3177        u32 bd_base_info;
3178
3179        do {
3180                desc = &ring->desc[ring->next_to_clean];
3181                desc_cb = &ring->desc_cb[ring->next_to_clean];
3182                bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3183                /* make sure HW write desc complete */
3184                dma_rmb();
3185                if (!(bd_base_info & BIT(HNS3_RXD_VLD_B)))
3186                        return -ENXIO;
3187
3188                if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {
3189                        new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);
3190                        if (unlikely(!new_skb)) {
3191                                hns3_rl_err(ring_to_netdev(ring),
3192                                            "alloc rx fraglist skb fail\n");
3193                                return -ENXIO;
3194                        }
3195                        ring->frag_num = 0;
3196
3197                        if (ring->tail_skb) {
3198                                ring->tail_skb->next = new_skb;
3199                                ring->tail_skb = new_skb;
3200                        } else {
3201                                skb_shinfo(skb)->frag_list = new_skb;
3202                                ring->tail_skb = new_skb;
3203                        }
3204                }
3205
3206                if (ring->tail_skb) {
3207                        head_skb->truesize += hns3_buf_size(ring);
3208                        head_skb->data_len += le16_to_cpu(desc->rx.size);
3209                        head_skb->len += le16_to_cpu(desc->rx.size);
3210                        skb = ring->tail_skb;
3211                }
3212
3213                dma_sync_single_for_cpu(ring_to_dev(ring),
3214                                desc_cb->dma + desc_cb->page_offset,
3215                                hns3_buf_size(ring),
3216                                DMA_FROM_DEVICE);
3217
3218                hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);
3219                trace_hns3_rx_desc(ring);
3220                hns3_rx_ring_move_fw(ring);
3221                ring->pending_buf++;
3222        } while (!(bd_base_info & BIT(HNS3_RXD_FE_B)));
3223
3224        return 0;
3225}
3226
3227static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring,
3228                                     struct sk_buff *skb, u32 l234info,
3229                                     u32 bd_base_info, u32 ol_info)
3230{
3231        u32 l3_type;
3232
3233        skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,
3234                                                    HNS3_RXD_GRO_SIZE_M,
3235                                                    HNS3_RXD_GRO_SIZE_S);
3236        /* if there is no HW GRO, do not set gro params */
3237        if (!skb_shinfo(skb)->gso_size) {
3238                hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info);
3239                return 0;
3240        }
3241
3242        NAPI_GRO_CB(skb)->count = hnae3_get_field(l234info,
3243                                                  HNS3_RXD_GRO_COUNT_M,
3244                                                  HNS3_RXD_GRO_COUNT_S);
3245
3246        l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S);
3247        if (l3_type == HNS3_L3_TYPE_IPV4)
3248                skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
3249        else if (l3_type == HNS3_L3_TYPE_IPV6)
3250                skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
3251        else
3252                return -EFAULT;
3253
3254        return  hns3_gro_complete(skb, l234info);
3255}
3256
3257static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
3258                                     struct sk_buff *skb, u32 rss_hash)
3259{
3260        struct hnae3_handle *handle = ring->tqp->handle;
3261        enum pkt_hash_types rss_type;
3262
3263        if (rss_hash)
3264                rss_type = handle->kinfo.rss_type;
3265        else
3266                rss_type = PKT_HASH_TYPE_NONE;
3267
3268        skb_set_hash(skb, rss_hash, rss_type);
3269}
3270
3271static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb)
3272{
3273        struct net_device *netdev = ring_to_netdev(ring);
3274        enum hns3_pkt_l2t_type l2_frame_type;
3275        u32 bd_base_info, l234info, ol_info;
3276        struct hns3_desc *desc;
3277        unsigned int len;
3278        int pre_ntc, ret;
3279
3280        /* bdinfo handled below is only valid on the last BD of the
3281         * current packet, and ring->next_to_clean indicates the first
3282         * descriptor of next packet, so need - 1 below.
3283         */
3284        pre_ntc = ring->next_to_clean ? (ring->next_to_clean - 1) :
3285                                        (ring->desc_num - 1);
3286        desc = &ring->desc[pre_ntc];
3287        bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3288        l234info = le32_to_cpu(desc->rx.l234_info);
3289        ol_info = le32_to_cpu(desc->rx.ol_info);
3290
3291        /* Based on hw strategy, the tag offloaded will be stored at
3292         * ot_vlan_tag in two layer tag case, and stored at vlan_tag
3293         * in one layer tag case.
3294         */
3295        if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
3296                u16 vlan_tag;
3297
3298                if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
3299                        __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
3300                                               vlan_tag);
3301        }
3302
3303        if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) |
3304                                  BIT(HNS3_RXD_L2E_B))))) {
3305                u64_stats_update_begin(&ring->syncp);
3306                if (l234info & BIT(HNS3_RXD_L2E_B))
3307                        ring->stats.l2_err++;
3308                else
3309                        ring->stats.err_pkt_len++;
3310                u64_stats_update_end(&ring->syncp);
3311
3312                return -EFAULT;
3313        }
3314
3315        len = skb->len;
3316
3317        /* Do update ip stack process */
3318        skb->protocol = eth_type_trans(skb, netdev);
3319
3320        /* This is needed in order to enable forwarding support */
3321        ret = hns3_set_gro_and_checksum(ring, skb, l234info,
3322                                        bd_base_info, ol_info);
3323        if (unlikely(ret)) {
3324                u64_stats_update_begin(&ring->syncp);
3325                ring->stats.rx_err_cnt++;
3326                u64_stats_update_end(&ring->syncp);
3327                return ret;
3328        }
3329
3330        l2_frame_type = hnae3_get_field(l234info, HNS3_RXD_DMAC_M,
3331                                        HNS3_RXD_DMAC_S);
3332
3333        u64_stats_update_begin(&ring->syncp);
3334        ring->stats.rx_pkts++;
3335        ring->stats.rx_bytes += len;
3336
3337        if (l2_frame_type == HNS3_L2_TYPE_MULTICAST)
3338                ring->stats.rx_multicast++;
3339
3340        u64_stats_update_end(&ring->syncp);
3341
3342        ring->tqp_vector->rx_group.total_bytes += len;
3343
3344        hns3_set_rx_skb_rss_type(ring, skb, le32_to_cpu(desc->rx.rss_hash));
3345        return 0;
3346}
3347
3348static int hns3_handle_rx_bd(struct hns3_enet_ring *ring)
3349{
3350        struct sk_buff *skb = ring->skb;
3351        struct hns3_desc_cb *desc_cb;
3352        struct hns3_desc *desc;
3353        unsigned int length;
3354        u32 bd_base_info;
3355        int ret;
3356
3357        desc = &ring->desc[ring->next_to_clean];
3358        desc_cb = &ring->desc_cb[ring->next_to_clean];
3359
3360        prefetch(desc);
3361
3362        if (!skb) {
3363                bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
3364                /* Check valid BD */
3365                if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))
3366                        return -ENXIO;
3367
3368                dma_rmb();
3369                length = le16_to_cpu(desc->rx.size);
3370
3371                ring->va = desc_cb->buf + desc_cb->page_offset;
3372
3373                dma_sync_single_for_cpu(ring_to_dev(ring),
3374                                desc_cb->dma + desc_cb->page_offset,
3375                                hns3_buf_size(ring),
3376                                DMA_FROM_DEVICE);
3377
3378                /* Prefetch first cache line of first page.
3379                 * Idea is to cache few bytes of the header of the packet.
3380                 * Our L1 Cache line size is 64B so need to prefetch twice to make
3381                 * it 128B. But in actual we can have greater size of caches with
3382                 * 128B Level 1 cache lines. In such a case, single fetch would
3383                 * suffice to cache in the relevant part of the header.
3384                 */
3385                net_prefetch(ring->va);
3386
3387                ret = hns3_alloc_skb(ring, length, ring->va);
3388                skb = ring->skb;
3389
3390                if (ret < 0) /* alloc buffer fail */
3391                        return ret;
3392                if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */
3393                        ret = hns3_add_frag(ring);
3394                        if (ret)
3395                                return ret;
3396                }
3397        } else {
3398                ret = hns3_add_frag(ring);
3399                if (ret)
3400                        return ret;
3401        }
3402
3403        /* As the head data may be changed when GRO enable, copy
3404         * the head data in after other data rx completed
3405         */
3406        if (skb->len > HNS3_RX_HEAD_SIZE)
3407                memcpy(skb->data, ring->va,
3408                       ALIGN(ring->pull_len, sizeof(long)));
3409
3410        ret = hns3_handle_bdinfo(ring, skb);
3411        if (unlikely(ret)) {
3412                dev_kfree_skb_any(skb);
3413                return ret;
3414        }
3415
3416        skb_record_rx_queue(skb, ring->tqp->tqp_index);
3417        return 0;
3418}
3419
3420int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget,
3421                       void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
3422{
3423#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
3424        int unused_count = hns3_desc_unused(ring);
3425        int recv_pkts = 0;
3426        int err;
3427
3428        unused_count -= ring->pending_buf;
3429
3430        while (recv_pkts < budget) {
3431                /* Reuse or realloc buffers */
3432                if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
3433                        hns3_nic_alloc_rx_buffers(ring, unused_count);
3434                        unused_count = hns3_desc_unused(ring) -
3435                                        ring->pending_buf;
3436                }
3437
3438                /* Poll one pkt */
3439                err = hns3_handle_rx_bd(ring);
3440                /* Do not get FE for the packet or failed to alloc skb */
3441                if (unlikely(!ring->skb || err == -ENXIO)) {
3442                        goto out;
3443                } else if (likely(!err)) {
3444                        rx_fn(ring, ring->skb);
3445                        recv_pkts++;
3446                }
3447
3448                unused_count += ring->pending_buf;
3449                ring->skb = NULL;
3450                ring->pending_buf = 0;
3451        }
3452
3453out:
3454        /* Make all data has been write before submit */
3455        if (unused_count > 0)
3456                hns3_nic_alloc_rx_buffers(ring, unused_count);
3457
3458        return recv_pkts;
3459}
3460
3461static bool hns3_get_new_flow_lvl(struct hns3_enet_ring_group *ring_group)
3462{
3463#define HNS3_RX_LOW_BYTE_RATE 10000
3464#define HNS3_RX_MID_BYTE_RATE 20000
3465#define HNS3_RX_ULTRA_PACKET_RATE 40
3466
3467        enum hns3_flow_level_range new_flow_level;
3468        struct hns3_enet_tqp_vector *tqp_vector;
3469        int packets_per_msecs, bytes_per_msecs;
3470        u32 time_passed_ms;
3471
3472        tqp_vector = ring_group->ring->tqp_vector;
3473        time_passed_ms =
3474                jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
3475        if (!time_passed_ms)
3476                return false;
3477
3478        do_div(ring_group->total_packets, time_passed_ms);
3479        packets_per_msecs = ring_group->total_packets;
3480
3481        do_div(ring_group->total_bytes, time_passed_ms);
3482        bytes_per_msecs = ring_group->total_bytes;
3483
3484        new_flow_level = ring_group->coal.flow_level;
3485
3486        /* Simple throttlerate management
3487         * 0-10MB/s   lower     (50000 ints/s)
3488         * 10-20MB/s   middle    (20000 ints/s)
3489         * 20-1249MB/s high      (18000 ints/s)
3490         * > 40000pps  ultra     (8000 ints/s)
3491         */
3492        switch (new_flow_level) {
3493        case HNS3_FLOW_LOW:
3494                if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
3495                        new_flow_level = HNS3_FLOW_MID;
3496                break;
3497        case HNS3_FLOW_MID:
3498                if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
3499                        new_flow_level = HNS3_FLOW_HIGH;
3500                else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
3501                        new_flow_level = HNS3_FLOW_LOW;
3502                break;
3503        case HNS3_FLOW_HIGH:
3504        case HNS3_FLOW_ULTRA:
3505        default:
3506                if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
3507                        new_flow_level = HNS3_FLOW_MID;
3508                break;
3509        }
3510
3511        if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
3512            &tqp_vector->rx_group == ring_group)
3513                new_flow_level = HNS3_FLOW_ULTRA;
3514
3515        ring_group->total_bytes = 0;
3516        ring_group->total_packets = 0;
3517        ring_group->coal.flow_level = new_flow_level;
3518
3519        return true;
3520}
3521
3522static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
3523{
3524        struct hns3_enet_tqp_vector *tqp_vector;
3525        u16 new_int_gl;
3526
3527        if (!ring_group->ring)
3528                return false;
3529
3530        tqp_vector = ring_group->ring->tqp_vector;
3531        if (!tqp_vector->last_jiffies)
3532                return false;
3533
3534        if (ring_group->total_packets == 0) {
3535                ring_group->coal.int_gl = HNS3_INT_GL_50K;
3536                ring_group->coal.flow_level = HNS3_FLOW_LOW;
3537                return true;
3538        }
3539
3540        if (!hns3_get_new_flow_lvl(ring_group))
3541                return false;
3542
3543        new_int_gl = ring_group->coal.int_gl;
3544        switch (ring_group->coal.flow_level) {
3545        case HNS3_FLOW_LOW:
3546                new_int_gl = HNS3_INT_GL_50K;
3547                break;
3548        case HNS3_FLOW_MID:
3549                new_int_gl = HNS3_INT_GL_20K;
3550                break;
3551        case HNS3_FLOW_HIGH:
3552                new_int_gl = HNS3_INT_GL_18K;
3553                break;
3554        case HNS3_FLOW_ULTRA:
3555                new_int_gl = HNS3_INT_GL_8K;
3556                break;
3557        default:
3558                break;
3559        }
3560
3561        if (new_int_gl != ring_group->coal.int_gl) {
3562                ring_group->coal.int_gl = new_int_gl;
3563                return true;
3564        }
3565        return false;
3566}
3567
3568static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
3569{
3570        struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
3571        struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
3572        bool rx_update, tx_update;
3573
3574        /* update param every 1000ms */
3575        if (time_before(jiffies,
3576                        tqp_vector->last_jiffies + msecs_to_jiffies(1000)))
3577                return;
3578
3579        if (rx_group->coal.adapt_enable) {
3580                rx_update = hns3_get_new_int_gl(rx_group);
3581                if (rx_update)
3582                        hns3_set_vector_coalesce_rx_gl(tqp_vector,
3583                                                       rx_group->coal.int_gl);
3584        }
3585
3586        if (tx_group->coal.adapt_enable) {
3587                tx_update = hns3_get_new_int_gl(tx_group);
3588                if (tx_update)
3589                        hns3_set_vector_coalesce_tx_gl(tqp_vector,
3590                                                       tx_group->coal.int_gl);
3591        }
3592
3593        tqp_vector->last_jiffies = jiffies;
3594}
3595
3596static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
3597{
3598        struct hns3_nic_priv *priv = netdev_priv(napi->dev);
3599        struct hns3_enet_ring *ring;
3600        int rx_pkt_total = 0;
3601
3602        struct hns3_enet_tqp_vector *tqp_vector =
3603                container_of(napi, struct hns3_enet_tqp_vector, napi);
3604        bool clean_complete = true;
3605        int rx_budget = budget;
3606
3607        if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
3608                napi_complete(napi);
3609                return 0;
3610        }
3611
3612        /* Since the actual Tx work is minimal, we can give the Tx a larger
3613         * budget and be more aggressive about cleaning up the Tx descriptors.
3614         */
3615        hns3_for_each_ring(ring, tqp_vector->tx_group)
3616                hns3_clean_tx_ring(ring, budget);
3617
3618        /* make sure rx ring budget not smaller than 1 */
3619        if (tqp_vector->num_tqps > 1)
3620                rx_budget = max(budget / tqp_vector->num_tqps, 1);
3621
3622        hns3_for_each_ring(ring, tqp_vector->rx_group) {
3623                int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
3624                                                    hns3_rx_skb);
3625                if (rx_cleaned >= rx_budget)
3626                        clean_complete = false;
3627
3628                rx_pkt_total += rx_cleaned;
3629        }
3630
3631        tqp_vector->rx_group.total_packets += rx_pkt_total;
3632
3633        if (!clean_complete)
3634                return budget;
3635
3636        if (napi_complete(napi) &&
3637            likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
3638                hns3_update_new_int_gl(tqp_vector);
3639                hns3_mask_vector_irq(tqp_vector, 1);
3640        }
3641
3642        return rx_pkt_total;
3643}
3644
3645static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
3646                                      struct hnae3_ring_chain_node *head)
3647{
3648        struct pci_dev *pdev = tqp_vector->handle->pdev;
3649        struct hnae3_ring_chain_node *cur_chain = head;
3650        struct hnae3_ring_chain_node *chain;
3651        struct hns3_enet_ring *tx_ring;
3652        struct hns3_enet_ring *rx_ring;
3653
3654        tx_ring = tqp_vector->tx_group.ring;
3655        if (tx_ring) {
3656                cur_chain->tqp_index = tx_ring->tqp->tqp_index;
3657                hnae3_set_bit(