linux/drivers/net/ethernet/pensando/ionic/ionic_lif.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4#include <linux/ethtool.h>
   5#include <linux/printk.h>
   6#include <linux/dynamic_debug.h>
   7#include <linux/netdevice.h>
   8#include <linux/etherdevice.h>
   9#include <linux/if_vlan.h>
  10#include <linux/rtnetlink.h>
  11#include <linux/interrupt.h>
  12#include <linux/pci.h>
  13#include <linux/cpumask.h>
  14
  15#include "ionic.h"
  16#include "ionic_bus.h"
  17#include "ionic_lif.h"
  18#include "ionic_txrx.h"
  19#include "ionic_ethtool.h"
  20#include "ionic_debugfs.h"
  21
  22/* queuetype support level */
  23static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  24        [IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  25        [IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  26        [IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
  27        [IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
  28                                      * 1 =   ... with Tx SG version 1
  29                                      */
  30};
  31
  32static void ionic_lif_rx_mode(struct ionic_lif *lif);
  33static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
  34static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
  35static void ionic_link_status_check(struct ionic_lif *lif);
  36static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  37static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  38static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  39
  40static void ionic_txrx_deinit(struct ionic_lif *lif);
  41static int ionic_txrx_init(struct ionic_lif *lif);
  42static int ionic_start_queues(struct ionic_lif *lif);
  43static void ionic_stop_queues(struct ionic_lif *lif);
  44static void ionic_lif_queue_identify(struct ionic_lif *lif);
  45
  46static void ionic_dim_work(struct work_struct *work)
  47{
  48        struct dim *dim = container_of(work, struct dim, work);
  49        struct dim_cq_moder cur_moder;
  50        struct ionic_qcq *qcq;
  51        u32 new_coal;
  52
  53        cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  54        qcq = container_of(dim, struct ionic_qcq, dim);
  55        new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
  56        new_coal = new_coal ? new_coal : 1;
  57
  58        if (qcq->intr.dim_coal_hw != new_coal) {
  59                unsigned int qi = qcq->cq.bound_q->index;
  60                struct ionic_lif *lif = qcq->q.lif;
  61
  62                qcq->intr.dim_coal_hw = new_coal;
  63
  64                ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
  65                                     lif->rxqcqs[qi]->intr.index,
  66                                     qcq->intr.dim_coal_hw);
  67        }
  68
  69        dim->state = DIM_START_MEASURE;
  70}
  71
  72static void ionic_lif_deferred_work(struct work_struct *work)
  73{
  74        struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  75        struct ionic_deferred *def = &lif->deferred;
  76        struct ionic_deferred_work *w = NULL;
  77
  78        do {
  79                spin_lock_bh(&def->lock);
  80                if (!list_empty(&def->list)) {
  81                        w = list_first_entry(&def->list,
  82                                             struct ionic_deferred_work, list);
  83                        list_del(&w->list);
  84                }
  85                spin_unlock_bh(&def->lock);
  86
  87                if (!w)
  88                        break;
  89
  90                switch (w->type) {
  91                case IONIC_DW_TYPE_RX_MODE:
  92                        ionic_lif_rx_mode(lif);
  93                        break;
  94                case IONIC_DW_TYPE_RX_ADDR_ADD:
  95                        ionic_lif_addr_add(lif, w->addr);
  96                        break;
  97                case IONIC_DW_TYPE_RX_ADDR_DEL:
  98                        ionic_lif_addr_del(lif, w->addr);
  99                        break;
 100                case IONIC_DW_TYPE_LINK_STATUS:
 101                        ionic_link_status_check(lif);
 102                        break;
 103                case IONIC_DW_TYPE_LIF_RESET:
 104                        if (w->fw_status)
 105                                ionic_lif_handle_fw_up(lif);
 106                        else
 107                                ionic_lif_handle_fw_down(lif);
 108                        break;
 109                default:
 110                        break;
 111                }
 112                kfree(w);
 113                w = NULL;
 114        } while (true);
 115}
 116
 117void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
 118                                struct ionic_deferred_work *work)
 119{
 120        spin_lock_bh(&def->lock);
 121        list_add_tail(&work->list, &def->list);
 122        spin_unlock_bh(&def->lock);
 123        schedule_work(&def->work);
 124}
 125
 126static void ionic_link_status_check(struct ionic_lif *lif)
 127{
 128        struct net_device *netdev = lif->netdev;
 129        u16 link_status;
 130        bool link_up;
 131
 132        if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 133                return;
 134
 135        /* Don't put carrier back up if we're in a broken state */
 136        if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 137                clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 138                return;
 139        }
 140
 141        link_status = le16_to_cpu(lif->info->status.link_status);
 142        link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 143
 144        if (link_up) {
 145                int err = 0;
 146
 147                if (netdev->flags & IFF_UP && netif_running(netdev)) {
 148                        mutex_lock(&lif->queue_lock);
 149                        err = ionic_start_queues(lif);
 150                        if (err && err != -EBUSY) {
 151                                netdev_err(lif->netdev,
 152                                           "Failed to start queues: %d\n", err);
 153                                set_bit(IONIC_LIF_F_BROKEN, lif->state);
 154                                netif_carrier_off(lif->netdev);
 155                        }
 156                        mutex_unlock(&lif->queue_lock);
 157                }
 158
 159                if (!err && !netif_carrier_ok(netdev)) {
 160                        ionic_port_identify(lif->ionic);
 161                        netdev_info(netdev, "Link up - %d Gbps\n",
 162                                    le32_to_cpu(lif->info->status.link_speed) / 1000);
 163                        netif_carrier_on(netdev);
 164                }
 165        } else {
 166                if (netif_carrier_ok(netdev)) {
 167                        netdev_info(netdev, "Link down\n");
 168                        netif_carrier_off(netdev);
 169                }
 170
 171                if (netdev->flags & IFF_UP && netif_running(netdev)) {
 172                        mutex_lock(&lif->queue_lock);
 173                        ionic_stop_queues(lif);
 174                        mutex_unlock(&lif->queue_lock);
 175                }
 176        }
 177
 178        clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 179}
 180
 181void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 182{
 183        struct ionic_deferred_work *work;
 184
 185        /* we only need one request outstanding at a time */
 186        if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 187                return;
 188
 189        if (!can_sleep) {
 190                work = kzalloc(sizeof(*work), GFP_ATOMIC);
 191                if (!work) {
 192                        clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 193                        return;
 194                }
 195
 196                work->type = IONIC_DW_TYPE_LINK_STATUS;
 197                ionic_lif_deferred_enqueue(&lif->deferred, work);
 198        } else {
 199                ionic_link_status_check(lif);
 200        }
 201}
 202
 203static irqreturn_t ionic_isr(int irq, void *data)
 204{
 205        struct napi_struct *napi = data;
 206
 207        napi_schedule_irqoff(napi);
 208
 209        return IRQ_HANDLED;
 210}
 211
 212static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 213{
 214        struct ionic_intr_info *intr = &qcq->intr;
 215        struct device *dev = lif->ionic->dev;
 216        struct ionic_queue *q = &qcq->q;
 217        const char *name;
 218
 219        if (lif->registered)
 220                name = lif->netdev->name;
 221        else
 222                name = dev_name(dev);
 223
 224        snprintf(intr->name, sizeof(intr->name),
 225                 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
 226
 227        return devm_request_irq(dev, intr->vector, ionic_isr,
 228                                0, intr->name, &qcq->napi);
 229}
 230
 231static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 232{
 233        struct ionic *ionic = lif->ionic;
 234        int index;
 235
 236        index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 237        if (index == ionic->nintrs) {
 238                netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 239                            __func__, index, ionic->nintrs);
 240                return -ENOSPC;
 241        }
 242
 243        set_bit(index, ionic->intrs);
 244        ionic_intr_init(&ionic->idev, intr, index);
 245
 246        return 0;
 247}
 248
 249static void ionic_intr_free(struct ionic *ionic, int index)
 250{
 251        if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 252                clear_bit(index, ionic->intrs);
 253}
 254
 255static int ionic_qcq_enable(struct ionic_qcq *qcq)
 256{
 257        struct ionic_queue *q = &qcq->q;
 258        struct ionic_lif *lif = q->lif;
 259        struct ionic_dev *idev;
 260        struct device *dev;
 261
 262        struct ionic_admin_ctx ctx = {
 263                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 264                .cmd.q_control = {
 265                        .opcode = IONIC_CMD_Q_CONTROL,
 266                        .lif_index = cpu_to_le16(lif->index),
 267                        .type = q->type,
 268                        .index = cpu_to_le32(q->index),
 269                        .oper = IONIC_Q_ENABLE,
 270                },
 271        };
 272
 273        idev = &lif->ionic->idev;
 274        dev = lif->ionic->dev;
 275
 276        dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 277                ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 278
 279        if (qcq->flags & IONIC_QCQ_F_INTR) {
 280                irq_set_affinity_hint(qcq->intr.vector,
 281                                      &qcq->intr.affinity_mask);
 282                napi_enable(&qcq->napi);
 283                ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 284                ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 285                                IONIC_INTR_MASK_CLEAR);
 286        }
 287
 288        return ionic_adminq_post_wait(lif, &ctx);
 289}
 290
 291static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
 292{
 293        struct ionic_queue *q;
 294        struct ionic_lif *lif;
 295        int err = 0;
 296
 297        struct ionic_admin_ctx ctx = {
 298                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 299                .cmd.q_control = {
 300                        .opcode = IONIC_CMD_Q_CONTROL,
 301                        .oper = IONIC_Q_DISABLE,
 302                },
 303        };
 304
 305        if (!qcq)
 306                return -ENXIO;
 307
 308        q = &qcq->q;
 309        lif = q->lif;
 310
 311        if (qcq->flags & IONIC_QCQ_F_INTR) {
 312                struct ionic_dev *idev = &lif->ionic->idev;
 313
 314                cancel_work_sync(&qcq->dim.work);
 315                ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 316                                IONIC_INTR_MASK_SET);
 317                synchronize_irq(qcq->intr.vector);
 318                irq_set_affinity_hint(qcq->intr.vector, NULL);
 319                napi_disable(&qcq->napi);
 320        }
 321
 322        if (send_to_hw) {
 323                ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 324                ctx.cmd.q_control.type = q->type;
 325                ctx.cmd.q_control.index = cpu_to_le32(q->index);
 326                dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 327                        ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 328
 329                err = ionic_adminq_post_wait(lif, &ctx);
 330        }
 331
 332        return err;
 333}
 334
 335static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 336{
 337        struct ionic_dev *idev = &lif->ionic->idev;
 338
 339        if (!qcq)
 340                return;
 341
 342        if (!(qcq->flags & IONIC_QCQ_F_INITED))
 343                return;
 344
 345        if (qcq->flags & IONIC_QCQ_F_INTR) {
 346                ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 347                                IONIC_INTR_MASK_SET);
 348                netif_napi_del(&qcq->napi);
 349        }
 350
 351        qcq->flags &= ~IONIC_QCQ_F_INITED;
 352}
 353
 354static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 355{
 356        if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 357                return;
 358
 359        irq_set_affinity_hint(qcq->intr.vector, NULL);
 360        devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 361        qcq->intr.vector = 0;
 362        ionic_intr_free(lif->ionic, qcq->intr.index);
 363        qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 364}
 365
 366static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 367{
 368        struct device *dev = lif->ionic->dev;
 369
 370        if (!qcq)
 371                return;
 372
 373        ionic_debugfs_del_qcq(qcq);
 374
 375        if (qcq->q_base) {
 376                dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 377                qcq->q_base = NULL;
 378                qcq->q_base_pa = 0;
 379        }
 380
 381        if (qcq->cq_base) {
 382                dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 383                qcq->cq_base = NULL;
 384                qcq->cq_base_pa = 0;
 385        }
 386
 387        if (qcq->sg_base) {
 388                dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 389                qcq->sg_base = NULL;
 390                qcq->sg_base_pa = 0;
 391        }
 392
 393        ionic_qcq_intr_free(lif, qcq);
 394
 395        if (qcq->cq.info) {
 396                devm_kfree(dev, qcq->cq.info);
 397                qcq->cq.info = NULL;
 398        }
 399        if (qcq->q.info) {
 400                devm_kfree(dev, qcq->q.info);
 401                qcq->q.info = NULL;
 402        }
 403}
 404
 405static void ionic_qcqs_free(struct ionic_lif *lif)
 406{
 407        struct device *dev = lif->ionic->dev;
 408        struct ionic_qcq *adminqcq;
 409        unsigned long irqflags;
 410
 411        if (lif->notifyqcq) {
 412                ionic_qcq_free(lif, lif->notifyqcq);
 413                devm_kfree(dev, lif->notifyqcq);
 414                lif->notifyqcq = NULL;
 415        }
 416
 417        if (lif->adminqcq) {
 418                spin_lock_irqsave(&lif->adminq_lock, irqflags);
 419                adminqcq = READ_ONCE(lif->adminqcq);
 420                lif->adminqcq = NULL;
 421                spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 422                if (adminqcq) {
 423                        ionic_qcq_free(lif, adminqcq);
 424                        devm_kfree(dev, adminqcq);
 425                }
 426        }
 427
 428        if (lif->rxqcqs) {
 429                devm_kfree(dev, lif->rxqstats);
 430                lif->rxqstats = NULL;
 431                devm_kfree(dev, lif->rxqcqs);
 432                lif->rxqcqs = NULL;
 433        }
 434
 435        if (lif->txqcqs) {
 436                devm_kfree(dev, lif->txqstats);
 437                lif->txqstats = NULL;
 438                devm_kfree(dev, lif->txqcqs);
 439                lif->txqcqs = NULL;
 440        }
 441}
 442
 443static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 444                                      struct ionic_qcq *n_qcq)
 445{
 446        if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
 447                ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
 448                n_qcq->flags &= ~IONIC_QCQ_F_INTR;
 449        }
 450
 451        n_qcq->intr.vector = src_qcq->intr.vector;
 452        n_qcq->intr.index = src_qcq->intr.index;
 453}
 454
 455static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 456{
 457        int err;
 458
 459        if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 460                qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 461                return 0;
 462        }
 463
 464        err = ionic_intr_alloc(lif, &qcq->intr);
 465        if (err) {
 466                netdev_warn(lif->netdev, "no intr for %s: %d\n",
 467                            qcq->q.name, err);
 468                goto err_out;
 469        }
 470
 471        err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 472        if (err < 0) {
 473                netdev_warn(lif->netdev, "no vector for %s: %d\n",
 474                            qcq->q.name, err);
 475                goto err_out_free_intr;
 476        }
 477        qcq->intr.vector = err;
 478        ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 479                               IONIC_INTR_MASK_SET);
 480
 481        err = ionic_request_irq(lif, qcq);
 482        if (err) {
 483                netdev_warn(lif->netdev, "irq request failed %d\n", err);
 484                goto err_out_free_intr;
 485        }
 486
 487        /* try to get the irq on the local numa node first */
 488        qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
 489                                             dev_to_node(lif->ionic->dev));
 490        if (qcq->intr.cpu != -1)
 491                cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
 492
 493        netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 494        return 0;
 495
 496err_out_free_intr:
 497        ionic_intr_free(lif->ionic, qcq->intr.index);
 498err_out:
 499        return err;
 500}
 501
 502static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 503                           unsigned int index,
 504                           const char *name, unsigned int flags,
 505                           unsigned int num_descs, unsigned int desc_size,
 506                           unsigned int cq_desc_size,
 507                           unsigned int sg_desc_size,
 508                           unsigned int pid, struct ionic_qcq **qcq)
 509{
 510        struct ionic_dev *idev = &lif->ionic->idev;
 511        struct device *dev = lif->ionic->dev;
 512        void *q_base, *cq_base, *sg_base;
 513        dma_addr_t cq_base_pa = 0;
 514        dma_addr_t sg_base_pa = 0;
 515        dma_addr_t q_base_pa = 0;
 516        struct ionic_qcq *new;
 517        int err;
 518
 519        *qcq = NULL;
 520
 521        new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 522        if (!new) {
 523                netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 524                err = -ENOMEM;
 525                goto err_out;
 526        }
 527
 528        new->q.dev = dev;
 529        new->flags = flags;
 530
 531        new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
 532                                   GFP_KERNEL);
 533        if (!new->q.info) {
 534                netdev_err(lif->netdev, "Cannot allocate queue info\n");
 535                err = -ENOMEM;
 536                goto err_out_free_qcq;
 537        }
 538
 539        new->q.type = type;
 540        new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 541
 542        err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 543                           desc_size, sg_desc_size, pid);
 544        if (err) {
 545                netdev_err(lif->netdev, "Cannot initialize queue\n");
 546                goto err_out_free_q_info;
 547        }
 548
 549        err = ionic_alloc_qcq_interrupt(lif, new);
 550        if (err)
 551                goto err_out;
 552
 553        new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
 554                                    GFP_KERNEL);
 555        if (!new->cq.info) {
 556                netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
 557                err = -ENOMEM;
 558                goto err_out_free_irq;
 559        }
 560
 561        err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 562        if (err) {
 563                netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 564                goto err_out_free_cq_info;
 565        }
 566
 567        if (flags & IONIC_QCQ_F_NOTIFYQ) {
 568                int q_size, cq_size;
 569
 570                /* q & cq need to be contiguous in case of notifyq */
 571                q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 572                cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 573
 574                new->q_size = PAGE_SIZE + q_size + cq_size;
 575                new->q_base = dma_alloc_coherent(dev, new->q_size,
 576                                                 &new->q_base_pa, GFP_KERNEL);
 577                if (!new->q_base) {
 578                        netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 579                        err = -ENOMEM;
 580                        goto err_out_free_cq_info;
 581                }
 582                q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 583                q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 584                ionic_q_map(&new->q, q_base, q_base_pa);
 585
 586                cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
 587                cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 588                ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 589                ionic_cq_bind(&new->cq, &new->q);
 590        } else {
 591                new->q_size = PAGE_SIZE + (num_descs * desc_size);
 592                new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 593                                                 GFP_KERNEL);
 594                if (!new->q_base) {
 595                        netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 596                        err = -ENOMEM;
 597                        goto err_out_free_cq_info;
 598                }
 599                q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 600                q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 601                ionic_q_map(&new->q, q_base, q_base_pa);
 602
 603                new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 604                new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 605                                                  GFP_KERNEL);
 606                if (!new->cq_base) {
 607                        netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 608                        err = -ENOMEM;
 609                        goto err_out_free_q;
 610                }
 611                cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 612                cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 613                ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 614                ionic_cq_bind(&new->cq, &new->q);
 615        }
 616
 617        if (flags & IONIC_QCQ_F_SG) {
 618                new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 619                new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 620                                                  GFP_KERNEL);
 621                if (!new->sg_base) {
 622                        netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 623                        err = -ENOMEM;
 624                        goto err_out_free_cq;
 625                }
 626                sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 627                sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 628                ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 629        }
 630
 631        INIT_WORK(&new->dim.work, ionic_dim_work);
 632        new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 633
 634        *qcq = new;
 635
 636        return 0;
 637
 638err_out_free_cq:
 639        dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 640err_out_free_q:
 641        dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 642err_out_free_cq_info:
 643        devm_kfree(dev, new->cq.info);
 644err_out_free_irq:
 645        if (flags & IONIC_QCQ_F_INTR) {
 646                devm_free_irq(dev, new->intr.vector, &new->napi);
 647                ionic_intr_free(lif->ionic, new->intr.index);
 648        }
 649err_out_free_q_info:
 650        devm_kfree(dev, new->q.info);
 651err_out_free_qcq:
 652        devm_kfree(dev, new);
 653err_out:
 654        dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 655        return err;
 656}
 657
 658static int ionic_qcqs_alloc(struct ionic_lif *lif)
 659{
 660        struct device *dev = lif->ionic->dev;
 661        unsigned int flags;
 662        int err;
 663
 664        flags = IONIC_QCQ_F_INTR;
 665        err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 666                              IONIC_ADMINQ_LENGTH,
 667                              sizeof(struct ionic_admin_cmd),
 668                              sizeof(struct ionic_admin_comp),
 669                              0, lif->kern_pid, &lif->adminqcq);
 670        if (err)
 671                return err;
 672        ionic_debugfs_add_qcq(lif, lif->adminqcq);
 673
 674        if (lif->ionic->nnqs_per_lif) {
 675                flags = IONIC_QCQ_F_NOTIFYQ;
 676                err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 677                                      flags, IONIC_NOTIFYQ_LENGTH,
 678                                      sizeof(struct ionic_notifyq_cmd),
 679                                      sizeof(union ionic_notifyq_comp),
 680                                      0, lif->kern_pid, &lif->notifyqcq);
 681                if (err)
 682                        goto err_out;
 683                ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 684
 685                /* Let the notifyq ride on the adminq interrupt */
 686                ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 687        }
 688
 689        err = -ENOMEM;
 690        lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 691                                   sizeof(*lif->txqcqs), GFP_KERNEL);
 692        if (!lif->txqcqs)
 693                goto err_out;
 694        lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 695                                   sizeof(*lif->rxqcqs), GFP_KERNEL);
 696        if (!lif->rxqcqs)
 697                goto err_out;
 698
 699        lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 700                                     sizeof(*lif->txqstats), GFP_KERNEL);
 701        if (!lif->txqstats)
 702                goto err_out;
 703        lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 704                                     sizeof(*lif->rxqstats), GFP_KERNEL);
 705        if (!lif->rxqstats)
 706                goto err_out;
 707
 708        return 0;
 709
 710err_out:
 711        ionic_qcqs_free(lif);
 712        return err;
 713}
 714
 715static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 716{
 717        qcq->q.tail_idx = 0;
 718        qcq->q.head_idx = 0;
 719        qcq->cq.tail_idx = 0;
 720        qcq->cq.done_color = 1;
 721        memset(qcq->q_base, 0, qcq->q_size);
 722        memset(qcq->cq_base, 0, qcq->cq_size);
 723        memset(qcq->sg_base, 0, qcq->sg_size);
 724}
 725
 726static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 727{
 728        struct device *dev = lif->ionic->dev;
 729        struct ionic_queue *q = &qcq->q;
 730        struct ionic_cq *cq = &qcq->cq;
 731        struct ionic_admin_ctx ctx = {
 732                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 733                .cmd.q_init = {
 734                        .opcode = IONIC_CMD_Q_INIT,
 735                        .lif_index = cpu_to_le16(lif->index),
 736                        .type = q->type,
 737                        .ver = lif->qtype_info[q->type].version,
 738                        .index = cpu_to_le32(q->index),
 739                        .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 740                                             IONIC_QINIT_F_SG),
 741                        .pid = cpu_to_le16(q->pid),
 742                        .ring_size = ilog2(q->num_descs),
 743                        .ring_base = cpu_to_le64(q->base_pa),
 744                        .cq_ring_base = cpu_to_le64(cq->base_pa),
 745                        .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 746                        .features = cpu_to_le64(q->features),
 747                },
 748        };
 749        unsigned int intr_index;
 750        int err;
 751
 752        intr_index = qcq->intr.index;
 753
 754        ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
 755
 756        dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 757        dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 758        dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 759        dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 760        dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 761        dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 762        dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 763
 764        ionic_qcq_sanitize(qcq);
 765
 766        err = ionic_adminq_post_wait(lif, &ctx);
 767        if (err)
 768                return err;
 769
 770        q->hw_type = ctx.comp.q_init.hw_type;
 771        q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 772        q->dbval = IONIC_DBELL_QID(q->hw_index);
 773
 774        dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 775        dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 776
 777        if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 778                netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
 779                               NAPI_POLL_WEIGHT);
 780
 781        qcq->flags |= IONIC_QCQ_F_INITED;
 782
 783        return 0;
 784}
 785
 786static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 787{
 788        struct device *dev = lif->ionic->dev;
 789        struct ionic_queue *q = &qcq->q;
 790        struct ionic_cq *cq = &qcq->cq;
 791        struct ionic_admin_ctx ctx = {
 792                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 793                .cmd.q_init = {
 794                        .opcode = IONIC_CMD_Q_INIT,
 795                        .lif_index = cpu_to_le16(lif->index),
 796                        .type = q->type,
 797                        .ver = lif->qtype_info[q->type].version,
 798                        .index = cpu_to_le32(q->index),
 799                        .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 800                                             IONIC_QINIT_F_SG),
 801                        .intr_index = cpu_to_le16(cq->bound_intr->index),
 802                        .pid = cpu_to_le16(q->pid),
 803                        .ring_size = ilog2(q->num_descs),
 804                        .ring_base = cpu_to_le64(q->base_pa),
 805                        .cq_ring_base = cpu_to_le64(cq->base_pa),
 806                        .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 807                        .features = cpu_to_le64(q->features),
 808                },
 809        };
 810        int err;
 811
 812        dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 813        dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 814        dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 815        dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 816        dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 817        dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 818        dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 819
 820        ionic_qcq_sanitize(qcq);
 821
 822        err = ionic_adminq_post_wait(lif, &ctx);
 823        if (err)
 824                return err;
 825
 826        q->hw_type = ctx.comp.q_init.hw_type;
 827        q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 828        q->dbval = IONIC_DBELL_QID(q->hw_index);
 829
 830        dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 831        dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 832
 833        if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 834                netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
 835                               NAPI_POLL_WEIGHT);
 836        else
 837                netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
 838                               NAPI_POLL_WEIGHT);
 839
 840        qcq->flags |= IONIC_QCQ_F_INITED;
 841
 842        return 0;
 843}
 844
 845int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 846{
 847        unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 848        unsigned int txq_i, flags;
 849        struct ionic_qcq *txq;
 850        u64 features;
 851        int err;
 852
 853        mutex_lock(&lif->queue_lock);
 854
 855        if (lif->hwstamp_txq)
 856                goto out;
 857
 858        features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 859
 860        num_desc = IONIC_MIN_TXRX_DESC;
 861        desc_sz = sizeof(struct ionic_txq_desc);
 862        comp_sz = 2 * sizeof(struct ionic_txq_comp);
 863
 864        if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 865            lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 866                sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 867        else
 868                sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 869
 870        txq_i = lif->ionic->ntxqs_per_lif;
 871        flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 872
 873        err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 874                              num_desc, desc_sz, comp_sz, sg_desc_sz,
 875                              lif->kern_pid, &txq);
 876        if (err)
 877                goto err_qcq_alloc;
 878
 879        txq->q.features = features;
 880
 881        ionic_link_qcq_interrupts(lif->adminqcq, txq);
 882        ionic_debugfs_add_qcq(lif, txq);
 883
 884        lif->hwstamp_txq = txq;
 885
 886        if (netif_running(lif->netdev)) {
 887                err = ionic_lif_txq_init(lif, txq);
 888                if (err)
 889                        goto err_qcq_init;
 890
 891                if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 892                        err = ionic_qcq_enable(txq);
 893                        if (err)
 894                                goto err_qcq_enable;
 895                }
 896        }
 897
 898out:
 899        mutex_unlock(&lif->queue_lock);
 900
 901        return 0;
 902
 903err_qcq_enable:
 904        ionic_lif_qcq_deinit(lif, txq);
 905err_qcq_init:
 906        lif->hwstamp_txq = NULL;
 907        ionic_debugfs_del_qcq(txq);
 908        ionic_qcq_free(lif, txq);
 909        devm_kfree(lif->ionic->dev, txq);
 910err_qcq_alloc:
 911        mutex_unlock(&lif->queue_lock);
 912        return err;
 913}
 914
 915int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
 916{
 917        unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 918        unsigned int rxq_i, flags;
 919        struct ionic_qcq *rxq;
 920        u64 features;
 921        int err;
 922
 923        mutex_lock(&lif->queue_lock);
 924
 925        if (lif->hwstamp_rxq)
 926                goto out;
 927
 928        features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 929
 930        num_desc = IONIC_MIN_TXRX_DESC;
 931        desc_sz = sizeof(struct ionic_rxq_desc);
 932        comp_sz = 2 * sizeof(struct ionic_rxq_comp);
 933        sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
 934
 935        rxq_i = lif->ionic->nrxqs_per_lif;
 936        flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
 937
 938        err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
 939                              num_desc, desc_sz, comp_sz, sg_desc_sz,
 940                              lif->kern_pid, &rxq);
 941        if (err)
 942                goto err_qcq_alloc;
 943
 944        rxq->q.features = features;
 945
 946        ionic_link_qcq_interrupts(lif->adminqcq, rxq);
 947        ionic_debugfs_add_qcq(lif, rxq);
 948
 949        lif->hwstamp_rxq = rxq;
 950
 951        if (netif_running(lif->netdev)) {
 952                err = ionic_lif_rxq_init(lif, rxq);
 953                if (err)
 954                        goto err_qcq_init;
 955
 956                if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 957                        ionic_rx_fill(&rxq->q);
 958                        err = ionic_qcq_enable(rxq);
 959                        if (err)
 960                                goto err_qcq_enable;
 961                }
 962        }
 963
 964out:
 965        mutex_unlock(&lif->queue_lock);
 966
 967        return 0;
 968
 969err_qcq_enable:
 970        ionic_lif_qcq_deinit(lif, rxq);
 971err_qcq_init:
 972        lif->hwstamp_rxq = NULL;
 973        ionic_debugfs_del_qcq(rxq);
 974        ionic_qcq_free(lif, rxq);
 975        devm_kfree(lif->ionic->dev, rxq);
 976err_qcq_alloc:
 977        mutex_unlock(&lif->queue_lock);
 978        return err;
 979}
 980
 981int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
 982{
 983        struct ionic_queue_params qparam;
 984
 985        ionic_init_queue_params(lif, &qparam);
 986
 987        if (rx_all)
 988                qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 989        else
 990                qparam.rxq_features = 0;
 991
 992        /* if we're not running, just set the values and return */
 993        if (!netif_running(lif->netdev)) {
 994                lif->rxq_features = qparam.rxq_features;
 995                return 0;
 996        }
 997
 998        return ionic_reconfigure_queues(lif, &qparam);
 999}
1000
1001int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1002{
1003        struct ionic_admin_ctx ctx = {
1004                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1005                .cmd.lif_setattr = {
1006                        .opcode = IONIC_CMD_LIF_SETATTR,
1007                        .index = cpu_to_le16(lif->index),
1008                        .attr = IONIC_LIF_ATTR_TXSTAMP,
1009                        .txstamp_mode = cpu_to_le16(txstamp_mode),
1010                },
1011        };
1012
1013        return ionic_adminq_post_wait(lif, &ctx);
1014}
1015
1016static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1017{
1018        struct ionic_admin_ctx ctx = {
1019                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1020                .cmd.rx_filter_del = {
1021                        .opcode = IONIC_CMD_RX_FILTER_DEL,
1022                        .lif_index = cpu_to_le16(lif->index),
1023                },
1024        };
1025        struct ionic_rx_filter *f;
1026        u32 filter_id;
1027        int err;
1028
1029        spin_lock_bh(&lif->rx_filters.lock);
1030
1031        f = ionic_rx_filter_rxsteer(lif);
1032        if (!f) {
1033                spin_unlock_bh(&lif->rx_filters.lock);
1034                return;
1035        }
1036
1037        filter_id = f->filter_id;
1038        ionic_rx_filter_free(lif, f);
1039
1040        spin_unlock_bh(&lif->rx_filters.lock);
1041
1042        netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1043
1044        ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1045
1046        err = ionic_adminq_post_wait(lif, &ctx);
1047        if (err && err != -EEXIST)
1048                netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1049}
1050
1051static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1052{
1053        struct ionic_admin_ctx ctx = {
1054                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1055                .cmd.rx_filter_add = {
1056                        .opcode = IONIC_CMD_RX_FILTER_ADD,
1057                        .lif_index = cpu_to_le16(lif->index),
1058                        .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1059                        .pkt_class = cpu_to_le64(pkt_class),
1060                },
1061        };
1062        u8 qtype;
1063        u32 qid;
1064        int err;
1065
1066        if (!lif->hwstamp_rxq)
1067                return -EINVAL;
1068
1069        qtype = lif->hwstamp_rxq->q.type;
1070        ctx.cmd.rx_filter_add.qtype = qtype;
1071
1072        qid = lif->hwstamp_rxq->q.index;
1073        ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1074
1075        netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1076        err = ionic_adminq_post_wait(lif, &ctx);
1077        if (err && err != -EEXIST)
1078                return err;
1079
1080        return ionic_rx_filter_save(lif, 0, qid, 0, &ctx);
1081}
1082
1083int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1084{
1085        ionic_lif_del_hwstamp_rxfilt(lif);
1086
1087        if (!pkt_class)
1088                return 0;
1089
1090        return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1091}
1092
1093static bool ionic_notifyq_service(struct ionic_cq *cq,
1094                                  struct ionic_cq_info *cq_info)
1095{
1096        union ionic_notifyq_comp *comp = cq_info->cq_desc;
1097        struct ionic_deferred_work *work;
1098        struct net_device *netdev;
1099        struct ionic_queue *q;
1100        struct ionic_lif *lif;
1101        u64 eid;
1102
1103        q = cq->bound_q;
1104        lif = q->info[0].cb_arg;
1105        netdev = lif->netdev;
1106        eid = le64_to_cpu(comp->event.eid);
1107
1108        /* Have we run out of new completions to process? */
1109        if ((s64)(eid - lif->last_eid) <= 0)
1110                return false;
1111
1112        lif->last_eid = eid;
1113
1114        dev_dbg(lif->ionic->dev, "notifyq event:\n");
1115        dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1116                         comp, sizeof(*comp), true);
1117
1118        switch (le16_to_cpu(comp->event.ecode)) {
1119        case IONIC_EVENT_LINK_CHANGE:
1120                ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1121                break;
1122        case IONIC_EVENT_RESET:
1123                work = kzalloc(sizeof(*work), GFP_ATOMIC);
1124                if (!work) {
1125                        netdev_err(lif->netdev, "Reset event dropped\n");
1126                } else {
1127                        work->type = IONIC_DW_TYPE_LIF_RESET;
1128                        ionic_lif_deferred_enqueue(&lif->deferred, work);
1129                }
1130                break;
1131        default:
1132                netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1133                            comp->event.ecode, eid);
1134                break;
1135        }
1136
1137        return true;
1138}
1139
1140static bool ionic_adminq_service(struct ionic_cq *cq,
1141                                 struct ionic_cq_info *cq_info)
1142{
1143        struct ionic_admin_comp *comp = cq_info->cq_desc;
1144
1145        if (!color_match(comp->color, cq->done_color))
1146                return false;
1147
1148        ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1149
1150        return true;
1151}
1152
1153static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1154{
1155        struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1156        struct ionic_lif *lif = napi_to_cq(napi)->lif;
1157        struct ionic_dev *idev = &lif->ionic->idev;
1158        unsigned long irqflags;
1159        unsigned int flags = 0;
1160        int rx_work = 0;
1161        int tx_work = 0;
1162        int n_work = 0;
1163        int a_work = 0;
1164        int work_done;
1165        int credits;
1166
1167        if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1168                n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1169                                          ionic_notifyq_service, NULL, NULL);
1170
1171        spin_lock_irqsave(&lif->adminq_lock, irqflags);
1172        if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1173                a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1174                                          ionic_adminq_service, NULL, NULL);
1175        spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1176
1177        if (lif->hwstamp_rxq)
1178                rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1179                                           ionic_rx_service, NULL, NULL);
1180
1181        if (lif->hwstamp_txq)
1182                tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1183                                           ionic_tx_service, NULL, NULL);
1184
1185        work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1186        if (work_done < budget && napi_complete_done(napi, work_done)) {
1187                flags |= IONIC_INTR_CRED_UNMASK;
1188                intr->rearm_count++;
1189        }
1190
1191        if (work_done || flags) {
1192                flags |= IONIC_INTR_CRED_RESET_COALESCE;
1193                credits = n_work + a_work + rx_work + tx_work;
1194                ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1195        }
1196
1197        return work_done;
1198}
1199
1200void ionic_get_stats64(struct net_device *netdev,
1201                       struct rtnl_link_stats64 *ns)
1202{
1203        struct ionic_lif *lif = netdev_priv(netdev);
1204        struct ionic_lif_stats *ls;
1205
1206        memset(ns, 0, sizeof(*ns));
1207        ls = &lif->info->stats;
1208
1209        ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1210                         le64_to_cpu(ls->rx_mcast_packets) +
1211                         le64_to_cpu(ls->rx_bcast_packets);
1212
1213        ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1214                         le64_to_cpu(ls->tx_mcast_packets) +
1215                         le64_to_cpu(ls->tx_bcast_packets);
1216
1217        ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1218                       le64_to_cpu(ls->rx_mcast_bytes) +
1219                       le64_to_cpu(ls->rx_bcast_bytes);
1220
1221        ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1222                       le64_to_cpu(ls->tx_mcast_bytes) +
1223                       le64_to_cpu(ls->tx_bcast_bytes);
1224
1225        ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1226                         le64_to_cpu(ls->rx_mcast_drop_packets) +
1227                         le64_to_cpu(ls->rx_bcast_drop_packets);
1228
1229        ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1230                         le64_to_cpu(ls->tx_mcast_drop_packets) +
1231                         le64_to_cpu(ls->tx_bcast_drop_packets);
1232
1233        ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1234
1235        ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1236
1237        ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1238                               le64_to_cpu(ls->rx_queue_disabled) +
1239                               le64_to_cpu(ls->rx_desc_fetch_error) +
1240                               le64_to_cpu(ls->rx_desc_data_error);
1241
1242        ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1243                                le64_to_cpu(ls->tx_queue_disabled) +
1244                                le64_to_cpu(ls->tx_desc_fetch_error) +
1245                                le64_to_cpu(ls->tx_desc_data_error);
1246
1247        ns->rx_errors = ns->rx_over_errors +
1248                        ns->rx_missed_errors;
1249
1250        ns->tx_errors = ns->tx_aborted_errors;
1251}
1252
1253static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1254{
1255        struct ionic_admin_ctx ctx = {
1256                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1257                .cmd.rx_filter_add = {
1258                        .opcode = IONIC_CMD_RX_FILTER_ADD,
1259                        .lif_index = cpu_to_le16(lif->index),
1260                        .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1261                },
1262        };
1263        struct ionic_rx_filter *f;
1264        int err;
1265
1266        /* don't bother if we already have it */
1267        spin_lock_bh(&lif->rx_filters.lock);
1268        f = ionic_rx_filter_by_addr(lif, addr);
1269        spin_unlock_bh(&lif->rx_filters.lock);
1270        if (f)
1271                return 0;
1272
1273        netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
1274
1275        memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1276        err = ionic_adminq_post_wait(lif, &ctx);
1277        if (err && err != -EEXIST)
1278                return err;
1279
1280        return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1281}
1282
1283static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1284{
1285        struct ionic_admin_ctx ctx = {
1286                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1287                .cmd.rx_filter_del = {
1288                        .opcode = IONIC_CMD_RX_FILTER_DEL,
1289                        .lif_index = cpu_to_le16(lif->index),
1290                },
1291        };
1292        struct ionic_rx_filter *f;
1293        int err;
1294
1295        spin_lock_bh(&lif->rx_filters.lock);
1296        f = ionic_rx_filter_by_addr(lif, addr);
1297        if (!f) {
1298                spin_unlock_bh(&lif->rx_filters.lock);
1299                return -ENOENT;
1300        }
1301
1302        netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1303                   addr, f->filter_id);
1304
1305        ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1306        ionic_rx_filter_free(lif, f);
1307        spin_unlock_bh(&lif->rx_filters.lock);
1308
1309        err = ionic_adminq_post_wait(lif, &ctx);
1310        if (err && err != -EEXIST)
1311                return err;
1312
1313        return 0;
1314}
1315
1316static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add)
1317{
1318        unsigned int nmfilters;
1319        unsigned int nufilters;
1320
1321        if (add) {
1322                /* Do we have space for this filter?  We test the counters
1323                 * here before checking the need for deferral so that we
1324                 * can return an overflow error to the stack.
1325                 */
1326                nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1327                nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1328
1329                if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1330                        lif->nmcast++;
1331                else if (!is_multicast_ether_addr(addr) &&
1332                         lif->nucast < nufilters)
1333                        lif->nucast++;
1334                else
1335                        return -ENOSPC;
1336        } else {
1337                if (is_multicast_ether_addr(addr) && lif->nmcast)
1338                        lif->nmcast--;
1339                else if (!is_multicast_ether_addr(addr) && lif->nucast)
1340                        lif->nucast--;
1341        }
1342
1343        netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1344                   add ? "add" : "del", addr);
1345        if (add)
1346                return ionic_lif_addr_add(lif, addr);
1347        else
1348                return ionic_lif_addr_del(lif, addr);
1349
1350        return 0;
1351}
1352
1353static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1354{
1355        return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR);
1356}
1357
1358static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1359{
1360        return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR);
1361}
1362
1363static void ionic_lif_rx_mode(struct ionic_lif *lif)
1364{
1365        struct net_device *netdev = lif->netdev;
1366        unsigned int nfilters;
1367        unsigned int nd_flags;
1368        char buf[128];
1369        u16 rx_mode;
1370        int i;
1371#define REMAIN(__x) (sizeof(buf) - (__x))
1372
1373        mutex_lock(&lif->config_lock);
1374
1375        /* grab the flags once for local use */
1376        nd_flags = netdev->flags;
1377
1378        rx_mode = IONIC_RX_MODE_F_UNICAST;
1379        rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1380        rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1381        rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1382        rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1383
1384        /* sync unicast addresses
1385         * next check to see if we're in an overflow state
1386         *    if so, we track that we overflowed and enable NIC PROMISC
1387         *    else if the overflow is set and not needed
1388         *       we remove our overflow flag and check the netdev flags
1389         *       to see if we can disable NIC PROMISC
1390         */
1391        __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1392        nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1393        if (netdev_uc_count(netdev) + 1 > nfilters) {
1394                rx_mode |= IONIC_RX_MODE_F_PROMISC;
1395                lif->uc_overflow = true;
1396        } else if (lif->uc_overflow) {
1397                lif->uc_overflow = false;
1398                if (!(nd_flags & IFF_PROMISC))
1399                        rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1400        }
1401
1402        /* same for multicast */
1403        __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1404        nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1405        if (netdev_mc_count(netdev) > nfilters) {
1406                rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1407                lif->mc_overflow = true;
1408        } else if (lif->mc_overflow) {
1409                lif->mc_overflow = false;
1410                if (!(nd_flags & IFF_ALLMULTI))
1411                        rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1412        }
1413
1414        i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1415                      lif->rx_mode, rx_mode);
1416        if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1417                i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1418        if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1419                i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1420        if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1421                i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1422        if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1423                i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1424        if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1425                i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1426        if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1427                i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1428        netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1429
1430        if (lif->rx_mode != rx_mode) {
1431                struct ionic_admin_ctx ctx = {
1432                        .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1433                        .cmd.rx_mode_set = {
1434                                .opcode = IONIC_CMD_RX_MODE_SET,
1435                                .lif_index = cpu_to_le16(lif->index),
1436                        },
1437                };
1438                int err;
1439
1440                ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1441                err = ionic_adminq_post_wait(lif, &ctx);
1442                if (err)
1443                        netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1444                                    rx_mode, err);
1445                else
1446                        lif->rx_mode = rx_mode;
1447        }
1448
1449        mutex_unlock(&lif->config_lock);
1450}
1451
1452static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
1453{
1454        struct ionic_lif *lif = netdev_priv(netdev);
1455        struct ionic_deferred_work *work;
1456
1457        if (!can_sleep) {
1458                work = kzalloc(sizeof(*work), GFP_ATOMIC);
1459                if (!work) {
1460                        netdev_err(lif->netdev, "rxmode change dropped\n");
1461                        return;
1462                }
1463                work->type = IONIC_DW_TYPE_RX_MODE;
1464                netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1465                ionic_lif_deferred_enqueue(&lif->deferred, work);
1466        } else {
1467                ionic_lif_rx_mode(lif);
1468        }
1469}
1470
1471static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1472{
1473        ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
1474}
1475
1476static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1477{
1478        u64 wanted = 0;
1479
1480        if (features & NETIF_F_HW_VLAN_CTAG_TX)
1481                wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1482        if (features & NETIF_F_HW_VLAN_CTAG_RX)
1483                wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1484        if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1485                wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1486        if (features & NETIF_F_RXHASH)
1487                wanted |= IONIC_ETH_HW_RX_HASH;
1488        if (features & NETIF_F_RXCSUM)
1489                wanted |= IONIC_ETH_HW_RX_CSUM;
1490        if (features & NETIF_F_SG)
1491                wanted |= IONIC_ETH_HW_TX_SG;
1492        if (features & NETIF_F_HW_CSUM)
1493                wanted |= IONIC_ETH_HW_TX_CSUM;
1494        if (features & NETIF_F_TSO)
1495                wanted |= IONIC_ETH_HW_TSO;
1496        if (features & NETIF_F_TSO6)
1497                wanted |= IONIC_ETH_HW_TSO_IPV6;
1498        if (features & NETIF_F_TSO_ECN)
1499                wanted |= IONIC_ETH_HW_TSO_ECN;
1500        if (features & NETIF_F_GSO_GRE)
1501                wanted |= IONIC_ETH_HW_TSO_GRE;
1502        if (features & NETIF_F_GSO_GRE_CSUM)
1503                wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1504        if (features & NETIF_F_GSO_IPXIP4)
1505                wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1506        if (features & NETIF_F_GSO_IPXIP6)
1507                wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1508        if (features & NETIF_F_GSO_UDP_TUNNEL)
1509                wanted |= IONIC_ETH_HW_TSO_UDP;
1510        if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1511                wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1512
1513        return cpu_to_le64(wanted);
1514}
1515
1516static int ionic_set_nic_features(struct ionic_lif *lif,
1517                                  netdev_features_t features)
1518{
1519        struct device *dev = lif->ionic->dev;
1520        struct ionic_admin_ctx ctx = {
1521                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1522                .cmd.lif_setattr = {
1523                        .opcode = IONIC_CMD_LIF_SETATTR,
1524                        .index = cpu_to_le16(lif->index),
1525                        .attr = IONIC_LIF_ATTR_FEATURES,
1526                },
1527        };
1528        u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1529                         IONIC_ETH_HW_VLAN_RX_STRIP |
1530                         IONIC_ETH_HW_VLAN_RX_FILTER;
1531        u64 old_hw_features;
1532        int err;
1533
1534        ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1535
1536        if (lif->phc)
1537                ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1538
1539        err = ionic_adminq_post_wait(lif, &ctx);
1540        if (err)
1541                return err;
1542
1543        old_hw_features = lif->hw_features;
1544        lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1545                                       ctx.comp.lif_setattr.features);
1546
1547        if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1548                ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1549
1550        if ((vlan_flags & features) &&
1551            !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1552                dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1553
1554        if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1555                dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1556        if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1557                dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1558        if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1559                dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1560        if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1561                dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1562        if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1563                dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1564        if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1565                dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1566        if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1567                dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1568        if (lif->hw_features & IONIC_ETH_HW_TSO)
1569                dev_dbg(dev, "feature ETH_HW_TSO\n");
1570        if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1571                dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1572        if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1573                dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1574        if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1575                dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1576        if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1577                dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1578        if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1579                dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1580        if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1581                dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1582        if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1583                dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1584        if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1585                dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1586        if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1587                dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1588
1589        return 0;
1590}
1591
1592static int ionic_init_nic_features(struct ionic_lif *lif)
1593{
1594        struct net_device *netdev = lif->netdev;
1595        netdev_features_t features;
1596        int err;
1597
1598        /* set up what we expect to support by default */
1599        features = NETIF_F_HW_VLAN_CTAG_TX |
1600                   NETIF_F_HW_VLAN_CTAG_RX |
1601                   NETIF_F_HW_VLAN_CTAG_FILTER |
1602                   NETIF_F_RXHASH |
1603                   NETIF_F_SG |
1604                   NETIF_F_HW_CSUM |
1605                   NETIF_F_RXCSUM |
1606                   NETIF_F_TSO |
1607                   NETIF_F_TSO6 |
1608                   NETIF_F_TSO_ECN;
1609
1610        err = ionic_set_nic_features(lif, features);
1611        if (err)
1612                return err;
1613
1614        /* tell the netdev what we actually can support */
1615        netdev->features |= NETIF_F_HIGHDMA;
1616
1617        if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1618                netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1619        if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1620                netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1621        if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1622                netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1623        if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1624                netdev->hw_features |= NETIF_F_RXHASH;
1625        if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1626                netdev->hw_features |= NETIF_F_SG;
1627
1628        if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1629                netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1630        if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1631                netdev->hw_enc_features |= NETIF_F_RXCSUM;
1632        if (lif->hw_features & IONIC_ETH_HW_TSO)
1633                netdev->hw_enc_features |= NETIF_F_TSO;
1634        if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1635                netdev->hw_enc_features |= NETIF_F_TSO6;
1636        if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1637                netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1638        if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1639                netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1640        if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1641                netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1642        if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1643                netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1644        if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1645                netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1646        if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1647                netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1648        if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1649                netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1650
1651        netdev->hw_features |= netdev->hw_enc_features;
1652        netdev->features |= netdev->hw_features;
1653        netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1654
1655        netdev->priv_flags |= IFF_UNICAST_FLT |
1656                              IFF_LIVE_ADDR_CHANGE;
1657
1658        return 0;
1659}
1660
1661static int ionic_set_features(struct net_device *netdev,
1662                              netdev_features_t features)
1663{
1664        struct ionic_lif *lif = netdev_priv(netdev);
1665        int err;
1666
1667        netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1668                   __func__, (u64)lif->netdev->features, (u64)features);
1669
1670        err = ionic_set_nic_features(lif, features);
1671
1672        return err;
1673}
1674
1675static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1676{
1677        struct sockaddr *addr = sa;
1678        u8 *mac;
1679        int err;
1680
1681        mac = (u8 *)addr->sa_data;
1682        if (ether_addr_equal(netdev->dev_addr, mac))
1683                return 0;
1684
1685        err = eth_prepare_mac_addr_change(netdev, addr);
1686        if (err)
1687                return err;
1688
1689        if (!is_zero_ether_addr(netdev->dev_addr)) {
1690                netdev_info(netdev, "deleting mac addr %pM\n",
1691                            netdev->dev_addr);
1692                ionic_addr_del(netdev, netdev->dev_addr);
1693        }
1694
1695        eth_commit_mac_addr_change(netdev, addr);
1696        netdev_info(netdev, "updating mac addr %pM\n", mac);
1697
1698        return ionic_addr_add(netdev, mac);
1699}
1700
1701static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1702{
1703        /* Stop and clean the queues before reconfiguration */
1704        mutex_lock(&lif->queue_lock);
1705        netif_device_detach(lif->netdev);
1706        ionic_stop_queues(lif);
1707        ionic_txrx_deinit(lif);
1708}
1709
1710static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1711{
1712        int err;
1713
1714        /* Re-init the queues after reconfiguration */
1715
1716        /* The only way txrx_init can fail here is if communication
1717         * with FW is suddenly broken.  There's not much we can do
1718         * at this point - error messages have already been printed,
1719         * so we can continue on and the user can eventually do a
1720         * DOWN and UP to try to reset and clear the issue.
1721         */
1722        err = ionic_txrx_init(lif);
1723        mutex_unlock(&lif->queue_lock);
1724        ionic_link_status_check_request(lif, CAN_SLEEP);
1725        netif_device_attach(lif->netdev);
1726
1727        return err;
1728}
1729
1730static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1731{
1732        struct ionic_lif *lif = netdev_priv(netdev);
1733        struct ionic_admin_ctx ctx = {
1734                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1735                .cmd.lif_setattr = {
1736                        .opcode = IONIC_CMD_LIF_SETATTR,
1737                        .index = cpu_to_le16(lif->index),
1738                        .attr = IONIC_LIF_ATTR_MTU,
1739                        .mtu = cpu_to_le32(new_mtu),
1740                },
1741        };
1742        int err;
1743
1744        err = ionic_adminq_post_wait(lif, &ctx);
1745        if (err)
1746                return err;
1747
1748        /* if we're not running, nothing more to do */
1749        if (!netif_running(netdev)) {
1750                netdev->mtu = new_mtu;
1751                return 0;
1752        }
1753
1754        ionic_stop_queues_reconfig(lif);
1755        netdev->mtu = new_mtu;
1756        return ionic_start_queues_reconfig(lif);
1757}
1758
1759static void ionic_tx_timeout_work(struct work_struct *ws)
1760{
1761        struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1762
1763        if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1764                return;
1765
1766        /* if we were stopped before this scheduled job was launched,
1767         * don't bother the queues as they are already stopped.
1768         */
1769        if (!netif_running(lif->netdev))
1770                return;
1771
1772        ionic_stop_queues_reconfig(lif);
1773        ionic_start_queues_reconfig(lif);
1774}
1775
1776static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1777{
1778        struct ionic_lif *lif = netdev_priv(netdev);
1779
1780        netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1781        schedule_work(&lif->tx_timeout_work);
1782}
1783
1784static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1785                                 u16 vid)
1786{
1787        struct ionic_lif *lif = netdev_priv(netdev);
1788        struct ionic_admin_ctx ctx = {
1789                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1790                .cmd.rx_filter_add = {
1791                        .opcode = IONIC_CMD_RX_FILTER_ADD,
1792                        .lif_index = cpu_to_le16(lif->index),
1793                        .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1794                        .vlan.vlan = cpu_to_le16(vid),
1795                },
1796        };
1797        int err;
1798
1799        netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1800        err = ionic_adminq_post_wait(lif, &ctx);
1801        if (err)
1802                return err;
1803
1804        return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1805}
1806
1807static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1808                                  u16 vid)
1809{
1810        struct ionic_lif *lif = netdev_priv(netdev);
1811        struct ionic_admin_ctx ctx = {
1812                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1813                .cmd.rx_filter_del = {
1814                        .opcode = IONIC_CMD_RX_FILTER_DEL,
1815                        .lif_index = cpu_to_le16(lif->index),
1816                },
1817        };
1818        struct ionic_rx_filter *f;
1819
1820        spin_lock_bh(&lif->rx_filters.lock);
1821
1822        f = ionic_rx_filter_by_vlan(lif, vid);
1823        if (!f) {
1824                spin_unlock_bh(&lif->rx_filters.lock);
1825                return -ENOENT;
1826        }
1827
1828        netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1829                   vid, f->filter_id);
1830
1831        ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1832        ionic_rx_filter_free(lif, f);
1833        spin_unlock_bh(&lif->rx_filters.lock);
1834
1835        return ionic_adminq_post_wait(lif, &ctx);
1836}
1837
1838int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1839                         const u8 *key, const u32 *indir)
1840{
1841        struct ionic_admin_ctx ctx = {
1842                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1843                .cmd.lif_setattr = {
1844                        .opcode = IONIC_CMD_LIF_SETATTR,
1845                        .attr = IONIC_LIF_ATTR_RSS,
1846                        .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1847                },
1848        };
1849        unsigned int i, tbl_sz;
1850
1851        if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1852                lif->rss_types = types;
1853                ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1854        }
1855
1856        if (key)
1857                memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1858
1859        if (indir) {
1860                tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1861                for (i = 0; i < tbl_sz; i++)
1862                        lif->rss_ind_tbl[i] = indir[i];
1863        }
1864
1865        memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1866               IONIC_RSS_HASH_KEY_SIZE);
1867
1868        return ionic_adminq_post_wait(lif, &ctx);
1869}
1870
1871static int ionic_lif_rss_init(struct ionic_lif *lif)
1872{
1873        unsigned int tbl_sz;
1874        unsigned int i;
1875
1876        lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1877                         IONIC_RSS_TYPE_IPV4_TCP |
1878                         IONIC_RSS_TYPE_IPV4_UDP |
1879                         IONIC_RSS_TYPE_IPV6     |
1880                         IONIC_RSS_TYPE_IPV6_TCP |
1881                         IONIC_RSS_TYPE_IPV6_UDP;
1882
1883        /* Fill indirection table with 'default' values */
1884        tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1885        for (i = 0; i < tbl_sz; i++)
1886                lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1887
1888        return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1889}
1890
1891static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1892{
1893        int tbl_sz;
1894
1895        tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1896        memset(lif->rss_ind_tbl, 0, tbl_sz);
1897        memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1898
1899        ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1900}
1901
1902static void ionic_lif_quiesce(struct ionic_lif *lif)
1903{
1904        struct ionic_admin_ctx ctx = {
1905                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1906                .cmd.lif_setattr = {
1907                        .opcode = IONIC_CMD_LIF_SETATTR,
1908                        .index = cpu_to_le16(lif->index),
1909                        .attr = IONIC_LIF_ATTR_STATE,
1910                        .state = IONIC_LIF_QUIESCE,
1911                },
1912        };
1913        int err;
1914
1915        err = ionic_adminq_post_wait(lif, &ctx);
1916        if (err)
1917                netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1918}
1919
1920static void ionic_txrx_disable(struct ionic_lif *lif)
1921{
1922        unsigned int i;
1923        int err = 0;
1924
1925        if (lif->txqcqs) {
1926                for (i = 0; i < lif->nxqs; i++)
1927                        err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1928        }
1929
1930        if (lif->hwstamp_txq)
1931                err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1932
1933        if (lif->rxqcqs) {
1934                for (i = 0; i < lif->nxqs; i++)
1935                        err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1936        }
1937
1938        if (lif->hwstamp_rxq)
1939                err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1940
1941        ionic_lif_quiesce(lif);
1942}
1943
1944static void ionic_txrx_deinit(struct ionic_lif *lif)
1945{
1946        unsigned int i;
1947
1948        if (lif->txqcqs) {
1949                for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1950                        ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1951                        ionic_tx_flush(&lif->txqcqs[i]->cq);
1952                        ionic_tx_empty(&lif->txqcqs[i]->q);
1953                }
1954        }
1955
1956        if (lif->rxqcqs) {
1957                for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1958                        ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1959                        ionic_rx_empty(&lif->rxqcqs[i]->q);
1960                }
1961        }
1962        lif->rx_mode = 0;
1963
1964        if (lif->hwstamp_txq) {
1965                ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1966                ionic_tx_flush(&lif->hwstamp_txq->cq);
1967                ionic_tx_empty(&lif->hwstamp_txq->q);
1968        }
1969
1970        if (lif->hwstamp_rxq) {
1971                ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1972                ionic_rx_empty(&lif->hwstamp_rxq->q);
1973        }
1974}
1975
1976static void ionic_txrx_free(struct ionic_lif *lif)
1977{
1978        unsigned int i;
1979
1980        if (lif->txqcqs) {
1981                for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1982                        ionic_qcq_free(lif, lif->txqcqs[i]);
1983                        devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1984                        lif->txqcqs[i] = NULL;
1985                }
1986        }
1987
1988        if (lif->rxqcqs) {
1989                for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1990                        ionic_qcq_free(lif, lif->rxqcqs[i]);
1991                        devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1992                        lif->rxqcqs[i] = NULL;
1993                }
1994        }
1995
1996        if (lif->hwstamp_txq) {
1997                ionic_qcq_free(lif, lif->hwstamp_txq);
1998                devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
1999                lif->hwstamp_txq = NULL;
2000        }
2001
2002        if (lif->hwstamp_rxq) {
2003                ionic_qcq_free(lif, lif->hwstamp_rxq);
2004                devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2005                lif->hwstamp_rxq = NULL;
2006        }
2007}
2008
2009static int ionic_txrx_alloc(struct ionic_lif *lif)
2010{
2011        unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2012        unsigned int flags, i;
2013        int err = 0;
2014
2015        num_desc = lif->ntxq_descs;
2016        desc_sz = sizeof(struct ionic_txq_desc);
2017        comp_sz = sizeof(struct ionic_txq_comp);
2018
2019        if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2020            lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2021                                          sizeof(struct ionic_txq_sg_desc_v1))
2022                sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2023        else
2024                sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2025
2026        flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2027        if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2028                flags |= IONIC_QCQ_F_INTR;
2029        for (i = 0; i < lif->nxqs; i++) {
2030                err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2031                                      num_desc, desc_sz, comp_sz, sg_desc_sz,
2032                                      lif->kern_pid, &lif->txqcqs[i]);
2033                if (err)
2034                        goto err_out;
2035
2036                if (flags & IONIC_QCQ_F_INTR) {
2037                        ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2038                                             lif->txqcqs[i]->intr.index,
2039                                             lif->tx_coalesce_hw);
2040                        if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2041                                lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2042                }
2043
2044                ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2045        }
2046
2047        flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2048
2049        num_desc = lif->nrxq_descs;
2050        desc_sz = sizeof(struct ionic_rxq_desc);
2051        comp_sz = sizeof(struct ionic_rxq_comp);
2052        sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2053
2054        if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2055                comp_sz *= 2;
2056
2057        for (i = 0; i < lif->nxqs; i++) {
2058                err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2059                                      num_desc, desc_sz, comp_sz, sg_desc_sz,
2060                                      lif->kern_pid, &lif->rxqcqs[i]);
2061                if (err)
2062                        goto err_out;
2063
2064                lif->rxqcqs[i]->q.features = lif->rxq_features;
2065
2066                ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2067                                     lif->rxqcqs[i]->intr.index,
2068                                     lif->rx_coalesce_hw);
2069                if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2070                        lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2071
2072                if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2073                        ionic_link_qcq_interrupts(lif->rxqcqs[i],
2074                                                  lif->txqcqs[i]);
2075
2076                ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2077        }
2078
2079        return 0;
2080
2081err_out:
2082        ionic_txrx_free(lif);
2083
2084        return err;
2085}
2086
2087static int ionic_txrx_init(struct ionic_lif *lif)
2088{
2089        unsigned int i;
2090        int err;
2091
2092        for (i = 0; i < lif->nxqs; i++) {
2093                err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2094                if (err)
2095                        goto err_out;
2096
2097                err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2098                if (err) {
2099                        ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2100                        goto err_out;
2101                }
2102        }
2103
2104        if (lif->netdev->features & NETIF_F_RXHASH)
2105                ionic_lif_rss_init(lif);
2106
2107        ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
2108
2109        return 0;
2110
2111err_out:
2112        while (i--) {
2113                ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2114                ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2115        }
2116
2117        return err;
2118}
2119
2120static int ionic_txrx_enable(struct ionic_lif *lif)
2121{
2122        int derr = 0;
2123        int i, err;
2124
2125        for (i = 0; i < lif->nxqs; i++) {
2126                if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2127                        dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2128                        err = -ENXIO;
2129                        goto err_out;
2130                }
2131
2132                ionic_rx_fill(&lif->rxqcqs[i]->q);
2133                err = ionic_qcq_enable(lif->rxqcqs[i]);
2134                if (err)
2135                        goto err_out;
2136
2137                err = ionic_qcq_enable(lif->txqcqs[i]);
2138                if (err) {
2139                        derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
2140                        goto err_out;
2141                }
2142        }
2143
2144        if (lif->hwstamp_rxq) {
2145                ionic_rx_fill(&lif->hwstamp_rxq->q);
2146                err = ionic_qcq_enable(lif->hwstamp_rxq);
2147                if (err)
2148                        goto err_out_hwstamp_rx;
2149        }
2150
2151        if (lif->hwstamp_txq) {
2152                err = ionic_qcq_enable(lif->hwstamp_txq);
2153                if (err)
2154                        goto err_out_hwstamp_tx;
2155        }
2156
2157        return 0;
2158
2159err_out_hwstamp_tx:
2160        if (lif->hwstamp_rxq)
2161                derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2162err_out_hwstamp_rx:
2163        i = lif->nxqs;
2164err_out:
2165        while (i--) {
2166                derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2167                derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
2168        }
2169
2170        return err;
2171}
2172
2173static int ionic_start_queues(struct ionic_lif *lif)
2174{
2175        int err;
2176
2177        if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2178                return -EIO;
2179
2180        if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2181                return -EBUSY;
2182
2183        if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2184                return 0;
2185
2186        err = ionic_txrx_enable(lif);
2187        if (err) {
2188                clear_bit(IONIC_LIF_F_UP, lif->state);
2189                return err;
2190        }
2191        netif_tx_wake_all_queues(lif->netdev);
2192
2193        return 0;
2194}
2195
2196static int ionic_open(struct net_device *netdev)
2197{
2198        struct ionic_lif *lif = netdev_priv(netdev);
2199        int err;
2200
2201        /* If recovering from a broken state, clear the bit and we'll try again */
2202        if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2203                netdev_info(netdev, "clearing broken state\n");
2204
2205        err = ionic_txrx_alloc(lif);
2206        if (err)
2207                return err;
2208
2209        err = ionic_txrx_init(lif);
2210        if (err)
2211                goto err_txrx_free;
2212
2213        err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2214        if (err)
2215                goto err_txrx_deinit;
2216
2217        err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2218        if (err)
2219                goto err_txrx_deinit;
2220
2221        /* don't start the queues until we have link */
2222        if (netif_carrier_ok(netdev)) {
2223                err = ionic_start_queues(lif);
2224                if (err)
2225                        goto err_txrx_deinit;
2226        }
2227
2228        return 0;
2229
2230err_txrx_deinit:
2231        ionic_txrx_deinit(lif);
2232err_txrx_free:
2233        ionic_txrx_free(lif);
2234        return err;
2235}
2236
2237static void ionic_stop_queues(struct ionic_lif *lif)
2238{
2239        if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2240                return;
2241
2242        netif_tx_disable(lif->netdev);
2243        ionic_txrx_disable(lif);
2244}
2245
2246static int ionic_stop(struct net_device *netdev)
2247{
2248        struct ionic_lif *lif = netdev_priv(netdev);
2249
2250        if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2251                return 0;
2252
2253        ionic_stop_queues(lif);
2254        ionic_txrx_deinit(lif);
2255        ionic_txrx_free(lif);
2256
2257        return 0;
2258}
2259
2260static int ionic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2261{
2262        struct ionic_lif *lif = netdev_priv(netdev);
2263
2264        switch (cmd) {
2265        case SIOCSHWTSTAMP:
2266                return ionic_lif_hwstamp_set(lif, ifr);
2267        case SIOCGHWTSTAMP:
2268                return ionic_lif_hwstamp_get(lif, ifr);
2269        default:
2270                return -EOPNOTSUPP;
2271        }
2272}
2273
2274static int ionic_get_vf_config(struct net_device *netdev,
2275                               int vf, struct ifla_vf_info *ivf)
2276{
2277        struct ionic_lif *lif = netdev_priv(netdev);
2278        struct ionic *ionic = lif->ionic;
2279        int ret = 0;
2280
2281        if (!netif_device_present(netdev))
2282                return -EBUSY;
2283
2284        down_read(&ionic->vf_op_lock);
2285
2286        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2287                ret = -EINVAL;
2288        } else {
2289                ivf->vf           = vf;
2290                ivf->vlan         = le16_to_cpu(ionic->vfs[vf].vlanid);
2291                ivf->qos          = 0;
2292                ivf->spoofchk     = ionic->vfs[vf].spoofchk;
2293                ivf->linkstate    = ionic->vfs[vf].linkstate;
2294                ivf->max_tx_rate  = le32_to_cpu(ionic->vfs[vf].maxrate);
2295                ivf->trusted      = ionic->vfs[vf].trusted;
2296                ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2297        }
2298
2299        up_read(&ionic->vf_op_lock);
2300        return ret;
2301}
2302
2303static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2304                              struct ifla_vf_stats *vf_stats)
2305{
2306        struct ionic_lif *lif = netdev_priv(netdev);
2307        struct ionic *ionic = lif->ionic;
2308        struct ionic_lif_stats *vs;
2309        int ret = 0;
2310
2311        if (!netif_device_present(netdev))
2312                return -EBUSY;
2313
2314        down_read(&ionic->vf_op_lock);
2315
2316        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2317                ret = -EINVAL;
2318        } else {
2319                memset(vf_stats, 0, sizeof(*vf_stats));
2320                vs = &ionic->vfs[vf].stats;
2321
2322                vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2323                vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2324                vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2325                vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2326                vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2327                vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2328                vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2329                                       le64_to_cpu(vs->rx_mcast_drop_packets) +
2330                                       le64_to_cpu(vs->rx_bcast_drop_packets);
2331                vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2332                                       le64_to_cpu(vs->tx_mcast_drop_packets) +
2333                                       le64_to_cpu(vs->tx_bcast_drop_packets);
2334        }
2335
2336        up_read(&ionic->vf_op_lock);
2337        return ret;
2338}
2339
2340static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2341{
2342        struct ionic_lif *lif = netdev_priv(netdev);
2343        struct ionic *ionic = lif->ionic;
2344        int ret;
2345
2346        if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2347                return -EINVAL;
2348
2349        if (!netif_device_present(netdev))
2350                return -EBUSY;
2351
2352        down_write(&ionic->vf_op_lock);
2353
2354        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2355                ret = -EINVAL;
2356        } else {
2357                ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2358                if (!ret)
2359                        ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2360        }
2361
2362        up_write(&ionic->vf_op_lock);
2363        return ret;
2364}
2365
2366static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2367                             u8 qos, __be16 proto)
2368{
2369        struct ionic_lif *lif = netdev_priv(netdev);
2370        struct ionic *ionic = lif->ionic;
2371        int ret;
2372
2373        /* until someday when we support qos */
2374        if (qos)
2375                return -EINVAL;
2376
2377        if (vlan > 4095)
2378                return -EINVAL;
2379
2380        if (proto != htons(ETH_P_8021Q))
2381                return -EPROTONOSUPPORT;
2382
2383        if (!netif_device_present(netdev))
2384                return -EBUSY;
2385
2386        down_write(&ionic->vf_op_lock);
2387
2388        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2389                ret = -EINVAL;
2390        } else {
2391                ret = ionic_set_vf_config(ionic, vf,
2392                                          IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2393                if (!ret)
2394                        ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2395        }
2396
2397        up_write(&ionic->vf_op_lock);
2398        return ret;
2399}
2400
2401static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2402                             int tx_min, int tx_max)
2403{
2404        struct ionic_lif *lif = netdev_priv(netdev);
2405        struct ionic *ionic = lif->ionic;
2406        int ret;
2407
2408        /* setting the min just seems silly */
2409        if (tx_min)
2410                return -EINVAL;
2411
2412        if (!netif_device_present(netdev))
2413                return -EBUSY;
2414
2415        down_write(&ionic->vf_op_lock);
2416
2417        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2418                ret = -EINVAL;
2419        } else {
2420                ret = ionic_set_vf_config(ionic, vf,
2421                                          IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2422                if (!ret)
2423                        lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2424        }
2425
2426        up_write(&ionic->vf_op_lock);
2427        return ret;
2428}
2429
2430static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2431{
2432        struct ionic_lif *lif = netdev_priv(netdev);
2433        struct ionic *ionic = lif->ionic;
2434        u8 data = set;  /* convert to u8 for config */
2435        int ret;
2436
2437        if (!netif_device_present(netdev))
2438                return -EBUSY;
2439
2440        down_write(&ionic->vf_op_lock);
2441
2442        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2443                ret = -EINVAL;
2444        } else {
2445                ret = ionic_set_vf_config(ionic, vf,
2446                                          IONIC_VF_ATTR_SPOOFCHK, &data);
2447                if (!ret)
2448                        ionic->vfs[vf].spoofchk = data;
2449        }
2450
2451        up_write(&ionic->vf_op_lock);
2452        return ret;
2453}
2454
2455static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2456{
2457        struct ionic_lif *lif = netdev_priv(netdev);
2458        struct ionic *ionic = lif->ionic;
2459        u8 data = set;  /* convert to u8 for config */
2460        int ret;
2461
2462        if (!netif_device_present(netdev))
2463                return -EBUSY;
2464
2465        down_write(&ionic->vf_op_lock);
2466
2467        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2468                ret = -EINVAL;
2469        } else {
2470                ret = ionic_set_vf_config(ionic, vf,
2471                                          IONIC_VF_ATTR_TRUST, &data);
2472                if (!ret)
2473                        ionic->vfs[vf].trusted = data;
2474        }
2475
2476        up_write(&ionic->vf_op_lock);
2477        return ret;
2478}
2479
2480static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2481{
2482        struct ionic_lif *lif = netdev_priv(netdev);
2483        struct ionic *ionic = lif->ionic;
2484        u8 data;
2485        int ret;
2486
2487        switch (set) {
2488        case IFLA_VF_LINK_STATE_ENABLE:
2489                data = IONIC_VF_LINK_STATUS_UP;
2490                break;
2491        case IFLA_VF_LINK_STATE_DISABLE:
2492                data = IONIC_VF_LINK_STATUS_DOWN;
2493                break;
2494        case IFLA_VF_LINK_STATE_AUTO:
2495                data = IONIC_VF_LINK_STATUS_AUTO;
2496                break;
2497        default:
2498                return -EINVAL;
2499        }
2500
2501        if (!netif_device_present(netdev))
2502                return -EBUSY;
2503
2504        down_write(&ionic->vf_op_lock);
2505
2506        if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2507                ret = -EINVAL;
2508        } else {
2509                ret = ionic_set_vf_config(ionic, vf,
2510                                          IONIC_VF_ATTR_LINKSTATE, &data);
2511                if (!ret)
2512                        ionic->vfs[vf].linkstate = set;
2513        }
2514
2515        up_write(&ionic->vf_op_lock);
2516        return ret;
2517}
2518
2519static const struct net_device_ops ionic_netdev_ops = {
2520        .ndo_open               = ionic_open,
2521        .ndo_stop               = ionic_stop,
2522        .ndo_do_ioctl           = ionic_do_ioctl,
2523        .ndo_start_xmit         = ionic_start_xmit,
2524        .ndo_get_stats64        = ionic_get_stats64,
2525        .ndo_set_rx_mode        = ionic_ndo_set_rx_mode,
2526        .ndo_set_features       = ionic_set_features,
2527        .ndo_set_mac_address    = ionic_set_mac_address,
2528        .ndo_validate_addr      = eth_validate_addr,
2529        .ndo_tx_timeout         = ionic_tx_timeout,
2530        .ndo_change_mtu         = ionic_change_mtu,
2531        .ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2532        .ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2533        .ndo_set_vf_vlan        = ionic_set_vf_vlan,
2534        .ndo_set_vf_trust       = ionic_set_vf_trust,
2535        .ndo_set_vf_mac         = ionic_set_vf_mac,
2536        .ndo_set_vf_rate        = ionic_set_vf_rate,
2537        .ndo_set_vf_spoofchk    = ionic_set_vf_spoofchk,
2538        .ndo_get_vf_config      = ionic_get_vf_config,
2539        .ndo_set_vf_link_state  = ionic_set_vf_link_state,
2540        .ndo_get_vf_stats       = ionic_get_vf_stats,
2541};
2542
2543static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2544{
2545        /* only swapping the queues, not the napi, flags, or other stuff */
2546        swap(a->q.features,   b->q.features);
2547        swap(a->q.num_descs,  b->q.num_descs);
2548        swap(a->q.desc_size,  b->q.desc_size);
2549        swap(a->q.base,       b->q.base);
2550        swap(a->q.base_pa,    b->q.base_pa);
2551        swap(a->q.info,       b->q.info);
2552        swap(a->q_base,       b->q_base);
2553        swap(a->q_base_pa,    b->q_base_pa);
2554        swap(a->q_size,       b->q_size);
2555
2556        swap(a->q.sg_desc_size, b->q.sg_desc_size);
2557        swap(a->q.sg_base,    b->q.sg_base);
2558        swap(a->q.sg_base_pa, b->q.sg_base_pa);
2559        swap(a->sg_base,      b->sg_base);
2560        swap(a->sg_base_pa,   b->sg_base_pa);
2561        swap(a->sg_size,      b->sg_size);
2562
2563        swap(a->cq.num_descs, b->cq.num_descs);
2564        swap(a->cq.desc_size, b->cq.desc_size);
2565        swap(a->cq.base,      b->cq.base);
2566        swap(a->cq.base_pa,   b->cq.base_pa);
2567        swap(a->cq.info,      b->cq.info);
2568        swap(a->cq_base,      b->cq_base);
2569        swap(a->cq_base_pa,   b->cq_base_pa);
2570        swap(a->cq_size,      b->cq_size);
2571
2572        ionic_debugfs_del_qcq(a);
2573        ionic_debugfs_add_qcq(a->q.lif, a);
2574}
2575
2576int ionic_reconfigure_queues(struct ionic_lif *lif,
2577                             struct ionic_queue_params *qparam)
2578{
2579        unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2580        struct ionic_qcq **tx_qcqs = NULL;
2581        struct ionic_qcq **rx_qcqs = NULL;
2582        unsigned int flags, i;
2583        int err = -ENOMEM;
2584
2585        /* allocate temporary qcq arrays to hold new queue structs */
2586        if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2587                tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2588                                       sizeof(struct ionic_qcq *), GFP_KERNEL);
2589                if (!tx_qcqs)
2590                        goto err_out;
2591        }
2592        if (qparam->nxqs != lif->nxqs ||
2593            qparam->nrxq_descs != lif->nrxq_descs ||
2594            qparam->rxq_features != lif->rxq_features) {
2595                rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2596                                       sizeof(struct ionic_qcq *), GFP_KERNEL);
2597                if (!rx_qcqs)
2598                        goto err_out;
2599        }
2600
2601        /* allocate new desc_info and rings, but leave the interrupt setup
2602         * until later so as to not mess with the still-running queues
2603         */
2604        if (tx_qcqs) {
2605                num_desc = qparam->ntxq_descs;
2606                desc_sz = sizeof(struct ionic_txq_desc);
2607                comp_sz = sizeof(struct ionic_txq_comp);
2608
2609                if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2610                    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2611                    sizeof(struct ionic_txq_sg_desc_v1))
2612                        sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2613                else
2614                        sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2615
2616                for (i = 0; i < qparam->nxqs; i++) {
2617                        flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2618                        err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2619                                              num_desc, desc_sz, comp_sz, sg_desc_sz,
2620                                              lif->kern_pid, &tx_qcqs[i]);
2621                        if (err)
2622                                goto err_out;
2623                }
2624        }
2625
2626        if (rx_qcqs) {
2627                num_desc = qparam->nrxq_descs;
2628                desc_sz = sizeof(struct ionic_rxq_desc);
2629                comp_sz = sizeof(struct ionic_rxq_comp);
2630                sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2631
2632                if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2633                        comp_sz *= 2;
2634
2635                for (i = 0; i < qparam->nxqs; i++) {
2636                        flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2637                        err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2638                                              num_desc, desc_sz, comp_sz, sg_desc_sz,
2639                                              lif->kern_pid, &rx_qcqs[i]);
2640                        if (err)
2641                                goto err_out;
2642
2643                        rx_qcqs[i]->q.features = qparam->rxq_features;
2644                }
2645        }
2646
2647        /* stop and clean the queues */
2648        ionic_stop_queues_reconfig(lif);
2649
2650        if (qparam->nxqs != lif->nxqs) {
2651                err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2652                if (err)
2653                        goto err_out_reinit_unlock;
2654                err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2655                if (err) {
2656                        netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2657                        goto err_out_reinit_unlock;
2658                }
2659        }
2660
2661        /* swap new desc_info and rings, keeping existing interrupt config */
2662        if (tx_qcqs) {
2663                lif->ntxq_descs = qparam->ntxq_descs;
2664                for (i = 0; i < qparam->nxqs; i++)
2665                        ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2666        }
2667
2668        if (rx_qcqs) {
2669                lif->nrxq_descs = qparam->nrxq_descs;
2670                for (i = 0; i < qparam->nxqs; i++)
2671                        ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2672        }
2673
2674        /* if we need to change the interrupt layout, this is the time */
2675        if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2676            qparam->nxqs != lif->nxqs) {
2677                if (qparam->intr_split) {
2678                        set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2679                } else {
2680                        clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2681                        lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2682                        lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2683                }
2684
2685                /* clear existing interrupt assignments */
2686                for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2687                        ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2688                        ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2689                }
2690
2691                /* re-assign the interrupts */
2692                for (i = 0; i < qparam->nxqs; i++) {
2693                        lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2694                        err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2695                        ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2696                                             lif->rxqcqs[i]->intr.index,
2697                                             lif->rx_coalesce_hw);
2698
2699                        if (qparam->intr_split) {
2700                                lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2701                                err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2702                                ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2703                                                     lif->txqcqs[i]->intr.index,
2704                                                     lif->tx_coalesce_hw);
2705                                if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2706                                        lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2707                        } else {
2708                                lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2709                                ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2710                        }
2711                }
2712        }
2713
2714        /* now we can rework the debugfs mappings */
2715        if (tx_qcqs) {
2716                for (i = 0; i < qparam->nxqs; i++) {
2717                        ionic_debugfs_del_qcq(lif->txqcqs[i]);
2718                        ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2719                }
2720        }
2721
2722        if (rx_qcqs) {
2723                for (i = 0; i < qparam->nxqs; i++) {
2724                        ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2725                        ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2726                }
2727        }
2728
2729        swap(lif->nxqs, qparam->nxqs);
2730        swap(lif->rxq_features, qparam->rxq_features);
2731
2732err_out_reinit_unlock:
2733        /* re-init the queues, but don't lose an error code */
2734        if (err)
2735                ionic_start_queues_reconfig(lif);
2736        else
2737                err = ionic_start_queues_reconfig(lif);
2738
2739err_out:
2740        /* free old allocs without cleaning intr */
2741        for (i = 0; i < qparam->nxqs; i++) {
2742                if (tx_qcqs && tx_qcqs[i]) {
2743                        tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2744                        ionic_qcq_free(lif, tx_qcqs[i]);
2745                        devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2746                        tx_qcqs[i] = NULL;
2747                }
2748                if (rx_qcqs && rx_qcqs[i]) {
2749                        rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2750                        ionic_qcq_free(lif, rx_qcqs[i]);
2751                        devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2752                        rx_qcqs[i] = NULL;
2753                }
2754        }
2755
2756        /* free q array */
2757        if (rx_qcqs) {
2758                devm_kfree(lif->ionic->dev, rx_qcqs);
2759                rx_qcqs = NULL;
2760        }
2761        if (tx_qcqs) {
2762                devm_kfree(lif->ionic->dev, tx_qcqs);
2763                tx_qcqs = NULL;
2764        }
2765
2766        /* clean the unused dma and info allocations when new set is smaller
2767         * than the full array, but leave the qcq shells in place
2768         */
2769        for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2770                lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2771                ionic_qcq_free(lif, lif->txqcqs[i]);
2772
2773                lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2774                ionic_qcq_free(lif, lif->rxqcqs[i]);
2775        }
2776
2777        return err;
2778}
2779
2780int ionic_lif_alloc(struct ionic *ionic)
2781{
2782        struct device *dev = ionic->dev;
2783        union ionic_lif_identity *lid;
2784        struct net_device *netdev;
2785        struct ionic_lif *lif;
2786        int tbl_sz;
2787        int err;
2788
2789        lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2790        if (!lid)
2791                return -ENOMEM;
2792
2793        netdev = alloc_etherdev_mqs(sizeof(*lif),
2794                                    ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2795        if (!netdev) {
2796                dev_err(dev, "Cannot allocate netdev, aborting\n");
2797                err = -ENOMEM;
2798                goto err_out_free_lid;
2799        }
2800
2801        SET_NETDEV_DEV(netdev, dev);
2802
2803        lif = netdev_priv(netdev);
2804        lif->netdev = netdev;
2805        ionic->lif = lif;
2806        netdev->netdev_ops = &ionic_netdev_ops;
2807        ionic_ethtool_set_ops(netdev);
2808
2809        netdev->watchdog_timeo = 2 * HZ;
2810        netif_carrier_off(netdev);
2811
2812        lif->identity = lid;
2813        lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2814        err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2815        if (err) {
2816                dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2817                        lif->lif_type, err);
2818                goto err_out_free_netdev;
2819        }
2820        lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2821                                     le32_to_cpu(lif->identity->eth.min_frame_size));
2822        lif->netdev->max_mtu =
2823                le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2824
2825        lif->neqs = ionic->neqs_per_lif;
2826        lif->nxqs = ionic->ntxqs_per_lif;
2827
2828        lif->ionic = ionic;
2829        lif->index = 0;
2830        lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2831        lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2832
2833        /* Convert the default coalesce value to actual hw resolution */
2834        lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2835        lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2836                                                    lif->rx_coalesce_usecs);
2837        lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2838        lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2839        set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2840        set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2841
2842        snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2843
2844        spin_lock_init(&lif->adminq_lock);
2845
2846        spin_lock_init(&lif->deferred.lock);
2847        INIT_LIST_HEAD(&lif->deferred.list);
2848        INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2849
2850        /* allocate lif info */
2851        lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2852        lif->info = dma_alloc_coherent(dev, lif->info_sz,
2853                                       &lif->info_pa, GFP_KERNEL);
2854        if (!lif->info) {
2855                dev_err(dev, "Failed to allocate lif info, aborting\n");
2856                err = -ENOMEM;
2857                goto err_out_free_netdev;
2858        }
2859
2860        ionic_debugfs_add_lif(lif);
2861
2862        /* allocate control queues and txrx queue arrays */
2863        ionic_lif_queue_identify(lif);
2864        err = ionic_qcqs_alloc(lif);
2865        if (err)
2866                goto err_out_free_lif_info;
2867
2868        /* allocate rss indirection table */
2869        tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2870        lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2871        lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2872                                              &lif->rss_ind_tbl_pa,
2873                                              GFP_KERNEL);
2874
2875        if (!lif->rss_ind_tbl) {
2876                err = -ENOMEM;
2877                dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2878                goto err_out_free_qcqs;
2879        }
2880        netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2881
2882        ionic_lif_alloc_phc(lif);
2883
2884        return 0;
2885
2886err_out_free_qcqs:
2887        ionic_qcqs_free(lif);
2888err_out_free_lif_info:
2889        dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2890        lif->info = NULL;
2891        lif->info_pa = 0;
2892err_out_free_netdev:
2893        free_netdev(lif->netdev);
2894        lif = NULL;
2895err_out_free_lid:
2896        kfree(lid);
2897
2898        return err;
2899}
2900
2901static void ionic_lif_reset(struct ionic_lif *lif)
2902{
2903        struct ionic_dev *idev = &lif->ionic->idev;
2904
2905        mutex_lock(&lif->ionic->dev_cmd_lock);
2906        ionic_dev_cmd_lif_reset(idev, lif->index);
2907        ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2908        mutex_unlock(&lif->ionic->dev_cmd_lock);
2909}
2910
2911static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2912{
2913        struct ionic *ionic = lif->ionic;
2914
2915        if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2916                return;
2917
2918        dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2919
2920        netif_device_detach(lif->netdev);
2921
2922        if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2923                dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2924                mutex_lock(&lif->queue_lock);
2925                ionic_stop_queues(lif);
2926                mutex_unlock(&lif->queue_lock);
2927        }
2928
2929        if (netif_running(lif->netdev)) {
2930                ionic_txrx_deinit(lif);
2931                ionic_txrx_free(lif);
2932        }
2933        ionic_lif_deinit(lif);
2934        ionic_reset(ionic);
2935        ionic_qcqs_free(lif);
2936
2937        dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2938}
2939
2940static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2941{
2942        struct ionic *ionic = lif->ionic;
2943        int err;
2944
2945        if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2946                return;
2947
2948        dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2949
2950        ionic_init_devinfo(ionic);
2951        err = ionic_identify(ionic);
2952        if (err)
2953                goto err_out;
2954        err = ionic_port_identify(ionic);
2955        if (err)
2956                goto err_out;
2957        err = ionic_port_init(ionic);
2958        if (err)
2959                goto err_out;
2960        err = ionic_qcqs_alloc(lif);
2961        if (err)
2962                goto err_out;
2963
2964        err = ionic_lif_init(lif);
2965        if (err)
2966                goto err_qcqs_free;
2967
2968        if (lif->registered)
2969                ionic_lif_set_netdev_info(lif);
2970
2971        ionic_rx_filter_replay(lif);
2972
2973        if (netif_running(lif->netdev)) {
2974                err = ionic_txrx_alloc(lif);
2975                if (err)
2976                        goto err_lifs_deinit;
2977
2978                err = ionic_txrx_init(lif);
2979                if (err)
2980                        goto err_txrx_free;
2981        }
2982
2983        clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
2984        ionic_link_status_check_request(lif, CAN_SLEEP);
2985        netif_device_attach(lif->netdev);
2986        dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2987
2988        /* restore the hardware timestamping queues */
2989        ionic_lif_hwstamp_replay(lif);
2990
2991        return;
2992
2993err_txrx_free:
2994        ionic_txrx_free(lif);
2995err_lifs_deinit:
2996        ionic_lif_deinit(lif);
2997err_qcqs_free:
2998        ionic_qcqs_free(lif);
2999err_out:
3000        dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3001}
3002
3003void ionic_lif_free(struct ionic_lif *lif)
3004{
3005        struct device *dev = lif->ionic->dev;
3006
3007        ionic_lif_free_phc(lif);
3008
3009        /* free rss indirection table */
3010        dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3011                          lif->rss_ind_tbl_pa);
3012        lif->rss_ind_tbl = NULL;
3013        lif->rss_ind_tbl_pa = 0;
3014
3015        /* free queues */
3016        ionic_qcqs_free(lif);
3017        if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3018                ionic_lif_reset(lif);
3019
3020        /* free lif info */
3021        kfree(lif->identity);
3022        dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3023        lif->info = NULL;
3024        lif->info_pa = 0;
3025
3026        /* unmap doorbell page */
3027        ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3028        lif->kern_dbpage = NULL;
3029        kfree(lif->dbid_inuse);
3030        lif->dbid_inuse = NULL;
3031
3032        /* free netdev & lif */
3033        ionic_debugfs_del_lif(lif);
3034        free_netdev(lif->netdev);
3035}
3036
3037void ionic_lif_deinit(struct ionic_lif *lif)
3038{
3039        if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3040                return;
3041
3042        if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3043                cancel_work_sync(&lif->deferred.work);
3044                cancel_work_sync(&lif->tx_timeout_work);
3045                ionic_rx_filters_deinit(lif);
3046                if (lif->netdev->features & NETIF_F_RXHASH)
3047                        ionic_lif_rss_deinit(lif);
3048        }
3049
3050        napi_disable(&lif->adminqcq->napi);
3051        ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3052        ionic_lif_qcq_deinit(lif, lif->adminqcq);
3053
3054        mutex_destroy(&lif->config_lock);
3055        mutex_destroy(&lif->queue_lock);
3056        ionic_lif_reset(lif);
3057}
3058
3059static int ionic_lif_adminq_init(struct ionic_lif *lif)
3060{
3061        struct device *dev = lif->ionic->dev;
3062        struct ionic_q_init_comp comp;
3063        struct ionic_dev *idev;
3064        struct ionic_qcq *qcq;
3065        struct ionic_queue *q;
3066        int err;
3067
3068        idev = &lif->ionic->idev;
3069        qcq = lif->adminqcq;
3070        q = &qcq->q;
3071
3072        mutex_lock(&lif->ionic->dev_cmd_lock);
3073        ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3074        err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3075        ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3076        mutex_unlock(&lif->ionic->dev_cmd_lock);
3077        if (err) {
3078                netdev_err(lif->netdev, "adminq init failed %d\n", err);
3079                return err;
3080        }
3081
3082        q->hw_type = comp.hw_type;
3083        q->hw_index = le32_to_cpu(comp.hw_index);
3084        q->dbval = IONIC_DBELL_QID(q->hw_index);
3085
3086        dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3087        dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3088
3089        netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3090                       NAPI_POLL_WEIGHT);
3091
3092        napi_enable(&qcq->napi);
3093
3094        if (qcq->flags & IONIC_QCQ_F_INTR)
3095                ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3096                                IONIC_INTR_MASK_CLEAR);
3097
3098        qcq->flags |= IONIC_QCQ_F_INITED;
3099
3100        return 0;
3101}
3102
3103static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3104{
3105        struct ionic_qcq *qcq = lif->notifyqcq;
3106        struct device *dev = lif->ionic->dev;
3107        struct ionic_queue *q = &qcq->q;
3108        int err;
3109
3110        struct ionic_admin_ctx ctx = {
3111                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3112                .cmd.q_init = {
3113                        .opcode = IONIC_CMD_Q_INIT,
3114                        .lif_index = cpu_to_le16(lif->index),
3115                        .type = q->type,
3116                        .ver = lif->qtype_info[q->type].version,
3117                        .index = cpu_to_le32(q->index),
3118                        .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3119                                             IONIC_QINIT_F_ENA),
3120                        .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3121                        .pid = cpu_to_le16(q->pid),
3122                        .ring_size = ilog2(q->num_descs),
3123                        .ring_base = cpu_to_le64(q->base_pa),
3124                }
3125        };
3126
3127        dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3128        dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3129        dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3130        dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3131
3132        err = ionic_adminq_post_wait(lif, &ctx);
3133        if (err)
3134                return err;
3135
3136        lif->last_eid = 0;
3137        q->hw_type = ctx.comp.q_init.hw_type;
3138        q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3139        q->dbval = IONIC_DBELL_QID(q->hw_index);
3140
3141        dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3142        dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3143
3144        /* preset the callback info */
3145        q->info[0].cb_arg = lif;
3146
3147        qcq->flags |= IONIC_QCQ_F_INITED;
3148
3149        return 0;
3150}
3151
3152static int ionic_station_set(struct ionic_lif *lif)
3153{
3154        struct net_device *netdev = lif->netdev;
3155        struct ionic_admin_ctx ctx = {
3156                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3157                .cmd.lif_getattr = {
3158                        .opcode = IONIC_CMD_LIF_GETATTR,
3159                        .index = cpu_to_le16(lif->index),
3160                        .attr = IONIC_LIF_ATTR_MAC,
3161                },
3162        };
3163        struct sockaddr addr;
3164        int err;
3165
3166        err = ionic_adminq_post_wait(lif, &ctx);
3167        if (err)
3168                return err;
3169        netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3170                   ctx.comp.lif_getattr.mac);
3171        if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3172                return 0;
3173
3174        if (!is_zero_ether_addr(netdev->dev_addr)) {
3175                /* If the netdev mac is non-zero and doesn't match the default
3176                 * device address, it was set by something earlier and we're
3177                 * likely here again after a fw-upgrade reset.  We need to be
3178                 * sure the netdev mac is in our filter list.
3179                 */
3180                if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3181                                      netdev->dev_addr))
3182                        ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR);
3183        } else {
3184                /* Update the netdev mac with the device's mac */
3185                memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3186                addr.sa_family = AF_INET;
3187                err = eth_prepare_mac_addr_change(netdev, &addr);
3188                if (err) {
3189                        netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3190                                    addr.sa_data, err);
3191                        return 0;
3192                }
3193
3194                eth_commit_mac_addr_change(netdev, &addr);
3195        }
3196
3197        netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3198                   netdev->dev_addr);
3199        ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR);
3200
3201        return 0;
3202}
3203
3204int ionic_lif_init(struct ionic_lif *lif)
3205{
3206        struct ionic_dev *idev = &lif->ionic->idev;
3207        struct device *dev = lif->ionic->dev;
3208        struct ionic_lif_init_comp comp;
3209        int dbpage_num;
3210        int err;
3211
3212        mutex_lock(&lif->ionic->dev_cmd_lock);
3213        ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3214        err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3215        ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3216        mutex_unlock(&lif->ionic->dev_cmd_lock);
3217        if (err)
3218                return err;
3219
3220        lif->hw_index = le16_to_cpu(comp.hw_index);
3221        mutex_init(&lif->queue_lock);
3222        mutex_init(&lif->config_lock);
3223
3224        /* now that we have the hw_index we can figure out our doorbell page */
3225        lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3226        if (!lif->dbid_count) {
3227                dev_err(dev, "No doorbell pages, aborting\n");
3228                return -EINVAL;
3229        }
3230
3231        lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3232        if (!lif->dbid_inuse) {
3233                dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3234                return -ENOMEM;
3235        }
3236
3237        /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3238        set_bit(0, lif->dbid_inuse);
3239        lif->kern_pid = 0;
3240
3241        dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3242        lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3243        if (!lif->kern_dbpage) {
3244                dev_err(dev, "Cannot map dbpage, aborting\n");
3245                err = -ENOMEM;
3246                goto err_out_free_dbid;
3247        }
3248
3249        err = ionic_lif_adminq_init(lif);
3250        if (err)
3251                goto err_out_adminq_deinit;
3252
3253        if (lif->ionic->nnqs_per_lif) {
3254                err = ionic_lif_notifyq_init(lif);
3255                if (err)
3256                        goto err_out_notifyq_deinit;
3257        }
3258
3259        err = ionic_init_nic_features(lif);
3260        if (err)
3261                goto err_out_notifyq_deinit;
3262
3263        if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3264                err = ionic_rx_filters_init(lif);
3265                if (err)
3266                        goto err_out_notifyq_deinit;
3267        }
3268
3269        err = ionic_station_set(lif);
3270        if (err)
3271                goto err_out_notifyq_deinit;
3272
3273        lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3274
3275        set_bit(IONIC_LIF_F_INITED, lif->state);
3276
3277        INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3278
3279        return 0;
3280
3281err_out_notifyq_deinit:
3282        ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3283err_out_adminq_deinit:
3284        ionic_lif_qcq_deinit(lif, lif->adminqcq);
3285        ionic_lif_reset(lif);
3286        ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3287        lif->kern_dbpage = NULL;
3288err_out_free_dbid:
3289        kfree(lif->dbid_inuse);
3290        lif->dbid_inuse = NULL;
3291
3292        return err;
3293}
3294
3295static void ionic_lif_notify_work(struct work_struct *ws)
3296{
3297}
3298
3299static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3300{
3301        struct ionic_admin_ctx ctx = {
3302                .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3303                .cmd.lif_setattr = {
3304                        .opcode = IONIC_CMD_LIF_SETATTR,
3305                        .index = cpu_to_le16(lif->index),
3306                        .attr = IONIC_LIF_ATTR_NAME,
3307                },
3308        };
3309
3310        strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3311                sizeof(ctx.cmd.lif_setattr.name));
3312
3313        ionic_adminq_post_wait(lif, &ctx);
3314}
3315
3316static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3317{
3318        if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3319                return NULL;
3320
3321        return netdev_priv(netdev);
3322}
3323
3324static int ionic_lif_notify(struct notifier_block *nb,
3325                            unsigned long event, void *info)
3326{
3327        struct net_device *ndev = netdev_notifier_info_to_dev(info);
3328        struct ionic *ionic = container_of(nb, struct ionic, nb);
3329        struct ionic_lif *lif = ionic_netdev_lif(ndev);
3330
3331        if (!lif || lif->ionic != ionic)
3332                return NOTIFY_DONE;
3333
3334        switch (event) {
3335        case NETDEV_CHANGENAME:
3336                ionic_lif_set_netdev_info(lif);
3337                break;
3338        }
3339
3340        return NOTIFY_DONE;
3341}
3342
3343int ionic_lif_register(struct ionic_lif *lif)
3344{
3345        int err;
3346
3347        ionic_lif_register_phc(lif);
3348
3349        INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3350
3351        lif->ionic->nb.notifier_call = ionic_lif_notify;
3352
3353        err = register_netdevice_notifier(&lif->ionic->nb);
3354        if (err)
3355                lif->ionic->nb.notifier_call = NULL;
3356
3357        /* only register LIF0 for now */
3358        err = register_netdev(lif->netdev);
3359        if (err) {
3360                dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3361                ionic_lif_unregister_phc(lif);
3362                return err;
3363        }
3364
3365        ionic_link_status_check_request(lif, CAN_SLEEP);
3366        lif->registered = true;
3367        ionic_lif_set_netdev_info(lif);
3368
3369        return 0;
3370}
3371
3372void ionic_lif_unregister(struct ionic_lif *lif)
3373{
3374        if (lif->ionic->nb.notifier_call) {
3375                unregister_netdevice_notifier(&lif->ionic->nb);
3376                cancel_work_sync(&lif->ionic->nb_work);
3377                lif->ionic->nb.notifier_call = NULL;
3378        }
3379
3380        if (lif->netdev->reg_state == NETREG_REGISTERED)
3381                unregister_netdev(lif->netdev);
3382
3383        ionic_lif_unregister_phc(lif);
3384
3385        lif->registered = false;
3386}
3387
3388static void ionic_lif_queue_identify(struct ionic_lif *lif)
3389{
3390        union ionic_q_identity __iomem *q_ident;
3391        struct ionic *ionic = lif->ionic;
3392        struct ionic_dev *idev;
3393        int qtype;
3394        int err;
3395
3396        idev = &lif->ionic->idev;
3397        q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3398
3399        for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3400                struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3401
3402                /* filter out the ones we know about */
3403                switch (qtype) {
3404                case IONIC_QTYPE_ADMINQ:
3405                case IONIC_QTYPE_NOTIFYQ:
3406                case IONIC_QTYPE_RXQ:
3407                case IONIC_QTYPE_TXQ:
3408                        break;
3409                default:
3410                        continue;
3411                }
3412
3413                memset(qti, 0, sizeof(*qti));
3414
3415                mutex_lock(&ionic->dev_cmd_lock);
3416                ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3417                                             ionic_qtype_versions[qtype]);
3418                err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3419                if (!err) {
3420                        qti->version   = readb(&q_ident->version);
3421                        qti->supported = readb(&q_ident->supported);
3422                        qti->features  = readq(&q_ident->features);
3423                        qti->desc_sz   = readw(&q_ident->desc_sz);
3424                        qti->comp_sz   = readw(&q_ident->comp_sz);
3425                        qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3426                        qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3427                        qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3428                }
3429                mutex_unlock(&ionic->dev_cmd_lock);
3430
3431                if (err == -EINVAL) {
3432                        dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3433                        continue;
3434                } else if (err == -EIO) {
3435                        dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3436                        return;
3437                } else if (err) {
3438                        dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3439                                qtype, err);
3440                        return;
3441                }
3442
3443                dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3444                        qtype, qti->version);
3445                dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3446                        qtype, qti->supported);
3447                dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3448                        qtype, qti->features);
3449                dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3450                        qtype, qti->desc_sz);
3451                dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3452                        qtype, qti->comp_sz);
3453                dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3454                        qtype, qti->sg_desc_sz);
3455                dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3456                        qtype, qti->max_sg_elems);
3457                dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3458                        qtype, qti->sg_desc_stride);
3459        }
3460}
3461
3462int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3463                       union ionic_lif_identity *lid)
3464{
3465        struct ionic_dev *idev = &ionic->idev;
3466        size_t sz;
3467        int err;
3468
3469        sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3470
3471        mutex_lock(&ionic->dev_cmd_lock);
3472        ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3473        err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3474        memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3475        mutex_unlock(&ionic->dev_cmd_lock);
3476        if (err)
3477                return (err);
3478
3479        dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3480                le64_to_cpu(lid->capabilities));
3481
3482        dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3483                le32_to_cpu(lid->eth.max_ucast_filters));
3484        dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3485                le32_to_cpu(lid->eth.max_mcast_filters));
3486        dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3487                le64_to_cpu(lid->eth.config.features));
3488        dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3489                le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3490        dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3491                le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3492        dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3493                le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3494        dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3495                le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3496        dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3497        dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3498        dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3499                le32_to_cpu(lid->eth.config.mtu));
3500
3501        return 0;
3502}
3503
3504int ionic_lif_size(struct ionic *ionic)
3505{
3506        struct ionic_identity *ident = &ionic->ident;
3507        unsigned int nintrs, dev_nintrs;
3508        union ionic_lif_config *lc;
3509        unsigned int ntxqs_per_lif;
3510        unsigned int nrxqs_per_lif;
3511        unsigned int neqs_per_lif;
3512        unsigned int nnqs_per_lif;
3513        unsigned int nxqs, neqs;
3514        unsigned int min_intrs;
3515        int err;
3516
3517        lc = &ident->lif.eth.config;
3518        dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3519        neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3520        nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3521        ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3522        nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3523
3524        /* reserve last queue id for hardware timestamping */
3525        if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3526                if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3527                        lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3528                } else {
3529                        ntxqs_per_lif -= 1;
3530                        nrxqs_per_lif -= 1;
3531                }
3532        }
3533
3534        nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3535        nxqs = min(nxqs, num_online_cpus());
3536        neqs = min(neqs_per_lif, num_online_cpus());
3537
3538try_again:
3539        /* interrupt usage:
3540         *    1 for master lif adminq/notifyq
3541         *    1 for each CPU for master lif TxRx queue pairs
3542         *    whatever's left is for RDMA queues
3543         */
3544        nintrs = 1 + nxqs + neqs;
3545        min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3546
3547        if (nintrs > dev_nintrs)
3548                goto try_fewer;
3549
3550        err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3551        if (err < 0 && err != -ENOSPC) {
3552                dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3553                return err;
3554        }
3555        if (err == -ENOSPC)
3556                goto try_fewer;
3557
3558        if (err != nintrs) {
3559                ionic_bus_free_irq_vectors(ionic);
3560                goto try_fewer;
3561        }
3562
3563        ionic->nnqs_per_lif = nnqs_per_lif;
3564        ionic->neqs_per_lif = neqs;
3565        ionic->ntxqs_per_lif = nxqs;
3566        ionic->nrxqs_per_lif = nxqs;
3567        ionic->nintrs = nintrs;
3568
3569        ionic_debugfs_add_sizes(ionic);
3570
3571        return 0;
3572
3573try_fewer:
3574        if (nnqs_per_lif > 1) {
3575                nnqs_per_lif >>= 1;
3576                goto try_again;
3577        }
3578        if (neqs > 1) {
3579                neqs >>= 1;
3580                goto try_again;
3581        }
3582        if (nxqs > 1) {
3583                nxqs >>= 1;
3584                goto try_again;
3585        }
3586        dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3587        return -ENOSPC;
3588}
3589