linux/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <net/tc_act/tc_gact.h>
  34#include <net/pkt_cls.h>
  35#include <linux/mlx5/fs.h>
  36#include <net/vxlan.h>
  37#include <net/geneve.h>
  38#include <linux/bpf.h>
  39#include <linux/if_bridge.h>
  40#include <net/page_pool.h>
  41#include <net/xdp_sock_drv.h>
  42#include "eswitch.h"
  43#include "en.h"
  44#include "en/txrx.h"
  45#include "en_tc.h"
  46#include "en_rep.h"
  47#include "en_accel/ipsec.h"
  48#include "en_accel/en_accel.h"
  49#include "en_accel/tls.h"
  50#include "accel/ipsec.h"
  51#include "accel/tls.h"
  52#include "lib/vxlan.h"
  53#include "lib/clock.h"
  54#include "en/port.h"
  55#include "en/xdp.h"
  56#include "lib/eq.h"
  57#include "en/monitor_stats.h"
  58#include "en/health.h"
  59#include "en/params.h"
  60#include "en/xsk/pool.h"
  61#include "en/xsk/setup.h"
  62#include "en/xsk/rx.h"
  63#include "en/xsk/tx.h"
  64#include "en/hv_vhca_stats.h"
  65#include "en/devlink.h"
  66#include "lib/mlx5.h"
  67#include "en/ptp.h"
  68#include "qos.h"
  69#include "en/trap.h"
  70#include "fpga/ipsec.h"
  71
  72bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
  73{
  74        bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) &&
  75                MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
  76                MLX5_CAP_ETH(mdev, reg_umr_sq);
  77        u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq);
  78        bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap;
  79
  80        if (!striding_rq_umr)
  81                return false;
  82        if (!inline_umr) {
  83                mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
  84                               (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
  85                return false;
  86        }
  87        return true;
  88}
  89
  90void mlx5e_update_carrier(struct mlx5e_priv *priv)
  91{
  92        struct mlx5_core_dev *mdev = priv->mdev;
  93        u8 port_state;
  94        bool up;
  95
  96        port_state = mlx5_query_vport_state(mdev,
  97                                            MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
  98                                            0);
  99
 100        up = port_state == VPORT_STATE_UP;
 101        if (up == netif_carrier_ok(priv->netdev))
 102                netif_carrier_event(priv->netdev);
 103        if (up) {
 104                netdev_info(priv->netdev, "Link up\n");
 105                netif_carrier_on(priv->netdev);
 106        } else {
 107                netdev_info(priv->netdev, "Link down\n");
 108                netif_carrier_off(priv->netdev);
 109        }
 110}
 111
 112static void mlx5e_update_carrier_work(struct work_struct *work)
 113{
 114        struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 115                                               update_carrier_work);
 116
 117        mutex_lock(&priv->state_lock);
 118        if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 119                if (priv->profile->update_carrier)
 120                        priv->profile->update_carrier(priv);
 121        mutex_unlock(&priv->state_lock);
 122}
 123
 124static void mlx5e_update_stats_work(struct work_struct *work)
 125{
 126        struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 127                                               update_stats_work);
 128
 129        mutex_lock(&priv->state_lock);
 130        priv->profile->update_stats(priv);
 131        mutex_unlock(&priv->state_lock);
 132}
 133
 134void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
 135{
 136        if (!priv->profile->update_stats)
 137                return;
 138
 139        if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state)))
 140                return;
 141
 142        queue_work(priv->wq, &priv->update_stats_work);
 143}
 144
 145static int async_event(struct notifier_block *nb, unsigned long event, void *data)
 146{
 147        struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
 148        struct mlx5_eqe   *eqe = data;
 149
 150        if (event != MLX5_EVENT_TYPE_PORT_CHANGE)
 151                return NOTIFY_DONE;
 152
 153        switch (eqe->sub_type) {
 154        case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
 155        case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
 156                queue_work(priv->wq, &priv->update_carrier_work);
 157                break;
 158        default:
 159                return NOTIFY_DONE;
 160        }
 161
 162        return NOTIFY_OK;
 163}
 164
 165static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 166{
 167        priv->events_nb.notifier_call = async_event;
 168        mlx5_notifier_register(priv->mdev, &priv->events_nb);
 169}
 170
 171static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 172{
 173        mlx5_notifier_unregister(priv->mdev, &priv->events_nb);
 174}
 175
 176static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
 177{
 178        struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb);
 179        int err;
 180
 181        switch (event) {
 182        case MLX5_DRIVER_EVENT_TYPE_TRAP:
 183                err = mlx5e_handle_trap_event(priv, data);
 184                break;
 185        default:
 186                netdev_warn(priv->netdev, "Sync event: Unknown event %ld\n", event);
 187                err = -EINVAL;
 188        }
 189        return err;
 190}
 191
 192static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
 193{
 194        priv->blocking_events_nb.notifier_call = blocking_event;
 195        mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb);
 196}
 197
 198static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
 199{
 200        mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
 201}
 202
 203static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
 204                                       struct mlx5e_icosq *sq,
 205                                       struct mlx5e_umr_wqe *wqe)
 206{
 207        struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 208        struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
 209        u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
 210
 211        cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 212                                      ds_cnt);
 213        cseg->umr_mkey  = rq->mkey_be;
 214
 215        ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
 216        ucseg->xlt_octowords =
 217                cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
 218        ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 219}
 220
 221static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
 222{
 223        int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 224
 225        rq->mpwqe.info = kvzalloc_node(array_size(wq_sz,
 226                                                  sizeof(*rq->mpwqe.info)),
 227                                       GFP_KERNEL, node);
 228        if (!rq->mpwqe.info)
 229                return -ENOMEM;
 230
 231        mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe);
 232
 233        return 0;
 234}
 235
 236static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
 237                                 u64 npages, u8 page_shift,
 238                                 struct mlx5_core_mkey *umr_mkey,
 239                                 dma_addr_t filler_addr)
 240{
 241        struct mlx5_mtt *mtt;
 242        int inlen;
 243        void *mkc;
 244        u32 *in;
 245        int err;
 246        int i;
 247
 248        inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages;
 249
 250        in = kvzalloc(inlen, GFP_KERNEL);
 251        if (!in)
 252                return -ENOMEM;
 253
 254        mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 255
 256        MLX5_SET(mkc, mkc, free, 1);
 257        MLX5_SET(mkc, mkc, umr_en, 1);
 258        MLX5_SET(mkc, mkc, lw, 1);
 259        MLX5_SET(mkc, mkc, lr, 1);
 260        MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 261        mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 262        MLX5_SET(mkc, mkc, qpn, 0xffffff);
 263        MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
 264        MLX5_SET64(mkc, mkc, len, npages << page_shift);
 265        MLX5_SET(mkc, mkc, translations_octword_size,
 266                 MLX5_MTT_OCTW(npages));
 267        MLX5_SET(mkc, mkc, log_page_size, page_shift);
 268        MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 269                 MLX5_MTT_OCTW(npages));
 270
 271        /* Initialize the mkey with all MTTs pointing to a default
 272         * page (filler_addr). When the channels are activated, UMR
 273         * WQEs will redirect the RX WQEs to the actual memory from
 274         * the RQ's pool, while the gaps (wqe_overflow) remain mapped
 275         * to the default page.
 276         */
 277        mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 278        for (i = 0 ; i < npages ; i++)
 279                mtt[i].ptag = cpu_to_be64(filler_addr);
 280
 281        err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 282
 283        kvfree(in);
 284        return err;
 285}
 286
 287static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
 288{
 289        u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
 290
 291        return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey,
 292                                     rq->wqe_overflow.addr);
 293}
 294
 295static u64 mlx5e_get_mpwqe_offset(u16 wqe_ix)
 296{
 297        return MLX5E_REQUIRED_MTTS(wqe_ix) << PAGE_SHIFT;
 298}
 299
 300static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 301{
 302        struct mlx5e_wqe_frag_info next_frag = {};
 303        struct mlx5e_wqe_frag_info *prev = NULL;
 304        int i;
 305
 306        next_frag.di = &rq->wqe.di[0];
 307
 308        for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 309                struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
 310                struct mlx5e_wqe_frag_info *frag =
 311                        &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
 312                int f;
 313
 314                for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
 315                        if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
 316                                next_frag.di++;
 317                                next_frag.offset = 0;
 318                                if (prev)
 319                                        prev->last_in_page = true;
 320                        }
 321                        *frag = next_frag;
 322
 323                        /* prepare next */
 324                        next_frag.offset += frag_info[f].frag_stride;
 325                        prev = frag;
 326                }
 327        }
 328
 329        if (prev)
 330                prev->last_in_page = true;
 331}
 332
 333int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
 334{
 335        int len = wq_sz << rq->wqe.info.log_num_frags;
 336
 337        rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node);
 338        if (!rq->wqe.di)
 339                return -ENOMEM;
 340
 341        mlx5e_init_frags_partition(rq);
 342
 343        return 0;
 344}
 345
 346void mlx5e_free_di_list(struct mlx5e_rq *rq)
 347{
 348        kvfree(rq->wqe.di);
 349}
 350
 351static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
 352{
 353        struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work);
 354
 355        mlx5e_reporter_rq_cqe_err(rq);
 356}
 357
 358static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 359{
 360        rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
 361        if (!rq->wqe_overflow.page)
 362                return -ENOMEM;
 363
 364        rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
 365                                             PAGE_SIZE, rq->buff.map_dir);
 366        if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
 367                __free_page(rq->wqe_overflow.page);
 368                return -ENOMEM;
 369        }
 370        return 0;
 371}
 372
 373static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
 374{
 375         dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
 376                        rq->buff.map_dir);
 377         __free_page(rq->wqe_overflow.page);
 378}
 379
 380static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
 381                             struct mlx5e_rq *rq)
 382{
 383        struct mlx5_core_dev *mdev = c->mdev;
 384        int err;
 385
 386        rq->wq_type      = params->rq_wq_type;
 387        rq->pdev         = c->pdev;
 388        rq->netdev       = c->netdev;
 389        rq->priv         = c->priv;
 390        rq->tstamp       = c->tstamp;
 391        rq->clock        = &mdev->clock;
 392        rq->icosq        = &c->icosq;
 393        rq->ix           = c->ix;
 394        rq->mdev         = mdev;
 395        rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 396        rq->xdpsq        = &c->rq_xdpsq;
 397        rq->stats        = &c->priv->channel_stats[c->ix].rq;
 398        rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
 399        err = mlx5e_rq_set_handlers(rq, params, NULL);
 400        if (err)
 401                return err;
 402
 403        return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, 0);
 404}
 405
 406static int mlx5e_alloc_rq(struct mlx5e_params *params,
 407                          struct mlx5e_xsk_param *xsk,
 408                          struct mlx5e_rq_param *rqp,
 409                          int node, struct mlx5e_rq *rq)
 410{
 411        struct page_pool_params pp_params = { 0 };
 412        struct mlx5_core_dev *mdev = rq->mdev;
 413        void *rqc = rqp->rqc;
 414        void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 415        u32 pool_size;
 416        int wq_sz;
 417        int err;
 418        int i;
 419
 420        rqp->wq.db_numa_node = node;
 421        INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
 422
 423        if (params->xdp_prog)
 424                bpf_prog_inc(params->xdp_prog);
 425        RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
 426
 427        rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 428        rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
 429        pool_size = 1 << params->log_rq_mtu_frames;
 430
 431        switch (rq->wq_type) {
 432        case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 433                err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
 434                                        &rq->wq_ctrl);
 435                if (err)
 436                        goto err_rq_xdp_prog;
 437
 438                err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
 439                if (err)
 440                        goto err_rq_wq_destroy;
 441
 442                rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
 443
 444                wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 445
 446                pool_size = MLX5_MPWRQ_PAGES_PER_WQE <<
 447                        mlx5e_mpwqe_get_log_rq_size(params, xsk);
 448
 449                rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 450                rq->mpwqe.num_strides =
 451                        BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
 452
 453                rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
 454
 455                err = mlx5e_create_rq_umr_mkey(mdev, rq);
 456                if (err)
 457                        goto err_rq_drop_page;
 458                rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
 459
 460                err = mlx5e_rq_alloc_mpwqe_info(rq, node);
 461                if (err)
 462                        goto err_rq_mkey;
 463                break;
 464        default: /* MLX5_WQ_TYPE_CYCLIC */
 465                err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
 466                                         &rq->wq_ctrl);
 467                if (err)
 468                        goto err_rq_xdp_prog;
 469
 470                rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
 471
 472                wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
 473
 474                rq->wqe.info = rqp->frags_info;
 475                rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
 476
 477                rq->wqe.frags =
 478                        kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
 479                                        (wq_sz << rq->wqe.info.log_num_frags)),
 480                                      GFP_KERNEL, node);
 481                if (!rq->wqe.frags) {
 482                        err = -ENOMEM;
 483                        goto err_rq_wq_destroy;
 484                }
 485
 486                err = mlx5e_init_di_list(rq, wq_sz, node);
 487                if (err)
 488                        goto err_rq_frags;
 489
 490                rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey.key);
 491        }
 492
 493        if (xsk) {
 494                err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 495                                                 MEM_TYPE_XSK_BUFF_POOL, NULL);
 496                xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
 497        } else {
 498                /* Create a page_pool and register it with rxq */
 499                pp_params.order     = 0;
 500                pp_params.flags     = 0; /* No-internal DMA mapping in page_pool */
 501                pp_params.pool_size = pool_size;
 502                pp_params.nid       = node;
 503                pp_params.dev       = rq->pdev;
 504                pp_params.dma_dir   = rq->buff.map_dir;
 505
 506                /* page_pool can be used even when there is no rq->xdp_prog,
 507                 * given page_pool does not handle DMA mapping there is no
 508                 * required state to clear. And page_pool gracefully handle
 509                 * elevated refcnt.
 510                 */
 511                rq->page_pool = page_pool_create(&pp_params);
 512                if (IS_ERR(rq->page_pool)) {
 513                        err = PTR_ERR(rq->page_pool);
 514                        rq->page_pool = NULL;
 515                        goto err_free_by_rq_type;
 516                }
 517                if (xdp_rxq_info_is_reg(&rq->xdp_rxq))
 518                        err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 519                                                         MEM_TYPE_PAGE_POOL, rq->page_pool);
 520        }
 521        if (err)
 522                goto err_free_by_rq_type;
 523
 524        for (i = 0; i < wq_sz; i++) {
 525                if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 526                        struct mlx5e_rx_wqe_ll *wqe =
 527                                mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
 528                        u32 byte_count =
 529                                rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
 530                        u64 dma_offset = mlx5e_get_mpwqe_offset(i);
 531
 532                        wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom);
 533                        wqe->data[0].byte_count = cpu_to_be32(byte_count);
 534                        wqe->data[0].lkey = rq->mkey_be;
 535                } else {
 536                        struct mlx5e_rx_wqe_cyc *wqe =
 537                                mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
 538                        int f;
 539
 540                        for (f = 0; f < rq->wqe.info.num_frags; f++) {
 541                                u32 frag_size = rq->wqe.info.arr[f].frag_size |
 542                                        MLX5_HW_START_PADDING;
 543
 544                                wqe->data[f].byte_count = cpu_to_be32(frag_size);
 545                                wqe->data[f].lkey = rq->mkey_be;
 546                        }
 547                        /* check if num_frags is not a pow of two */
 548                        if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
 549                                wqe->data[f].byte_count = 0;
 550                                wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
 551                                wqe->data[f].addr = 0;
 552                        }
 553                }
 554        }
 555
 556        INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
 557
 558        switch (params->rx_cq_moderation.cq_period_mode) {
 559        case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
 560                rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
 561                break;
 562        case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
 563        default:
 564                rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 565        }
 566
 567        rq->page_cache.head = 0;
 568        rq->page_cache.tail = 0;
 569
 570        return 0;
 571
 572err_free_by_rq_type:
 573        switch (rq->wq_type) {
 574        case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 575                kvfree(rq->mpwqe.info);
 576err_rq_mkey:
 577                mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
 578err_rq_drop_page:
 579                mlx5e_free_mpwqe_rq_drop_page(rq);
 580                break;
 581        default: /* MLX5_WQ_TYPE_CYCLIC */
 582                mlx5e_free_di_list(rq);
 583err_rq_frags:
 584                kvfree(rq->wqe.frags);
 585        }
 586err_rq_wq_destroy:
 587        mlx5_wq_destroy(&rq->wq_ctrl);
 588err_rq_xdp_prog:
 589        if (params->xdp_prog)
 590                bpf_prog_put(params->xdp_prog);
 591
 592        return err;
 593}
 594
 595static void mlx5e_free_rq(struct mlx5e_rq *rq)
 596{
 597        struct bpf_prog *old_prog;
 598        int i;
 599
 600        if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) {
 601                old_prog = rcu_dereference_protected(rq->xdp_prog,
 602                                                     lockdep_is_held(&rq->priv->state_lock));
 603                if (old_prog)
 604                        bpf_prog_put(old_prog);
 605        }
 606
 607        switch (rq->wq_type) {
 608        case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 609                kvfree(rq->mpwqe.info);
 610                mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
 611                mlx5e_free_mpwqe_rq_drop_page(rq);
 612                break;
 613        default: /* MLX5_WQ_TYPE_CYCLIC */
 614                kvfree(rq->wqe.frags);
 615                mlx5e_free_di_list(rq);
 616        }
 617
 618        for (i = rq->page_cache.head; i != rq->page_cache.tail;
 619             i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
 620                struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
 621
 622                /* With AF_XDP, page_cache is not used, so this loop is not
 623                 * entered, and it's safe to call mlx5e_page_release_dynamic
 624                 * directly.
 625                 */
 626                mlx5e_page_release_dynamic(rq, dma_info, false);
 627        }
 628
 629        xdp_rxq_info_unreg(&rq->xdp_rxq);
 630        page_pool_destroy(rq->page_pool);
 631        mlx5_wq_destroy(&rq->wq_ctrl);
 632}
 633
 634int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 635{
 636        struct mlx5_core_dev *mdev = rq->mdev;
 637        u8 ts_format;
 638        void *in;
 639        void *rqc;
 640        void *wq;
 641        int inlen;
 642        int err;
 643
 644        inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
 645                sizeof(u64) * rq->wq_ctrl.buf.npages;
 646        in = kvzalloc(inlen, GFP_KERNEL);
 647        if (!in)
 648                return -ENOMEM;
 649
 650        ts_format = mlx5_is_real_time_rq(mdev) ?
 651                            MLX5_TIMESTAMP_FORMAT_REAL_TIME :
 652                            MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
 653        rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 654        wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 655
 656        memcpy(rqc, param->rqc, sizeof(param->rqc));
 657
 658        MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
 659        MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
 660        MLX5_SET(rqc,  rqc, ts_format,          ts_format);
 661        MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
 662                                                MLX5_ADAPTER_PAGE_SHIFT);
 663        MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
 664
 665        mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
 666                                  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
 667
 668        err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
 669
 670        kvfree(in);
 671
 672        return err;
 673}
 674
 675int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
 676{
 677        struct mlx5_core_dev *mdev = rq->mdev;
 678
 679        void *in;
 680        void *rqc;
 681        int inlen;
 682        int err;
 683
 684        inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 685        in = kvzalloc(inlen, GFP_KERNEL);
 686        if (!in)
 687                return -ENOMEM;
 688
 689        if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY)
 690                mlx5e_rqwq_reset(rq);
 691
 692        rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 693
 694        MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 695        MLX5_SET(rqc, rqc, state, next_state);
 696
 697        err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 698
 699        kvfree(in);
 700
 701        return err;
 702}
 703
 704static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
 705{
 706        struct mlx5_core_dev *mdev = rq->mdev;
 707
 708        void *in;
 709        void *rqc;
 710        int inlen;
 711        int err;
 712
 713        inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 714        in = kvzalloc(inlen, GFP_KERNEL);
 715        if (!in)
 716                return -ENOMEM;
 717
 718        rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 719
 720        MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 721        MLX5_SET64(modify_rq_in, in, modify_bitmask,
 722                   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
 723        MLX5_SET(rqc, rqc, scatter_fcs, enable);
 724        MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 725
 726        err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 727
 728        kvfree(in);
 729
 730        return err;
 731}
 732
 733static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 734{
 735        struct mlx5_core_dev *mdev = rq->mdev;
 736        void *in;
 737        void *rqc;
 738        int inlen;
 739        int err;
 740
 741        inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
 742        in = kvzalloc(inlen, GFP_KERNEL);
 743        if (!in)
 744                return -ENOMEM;
 745
 746        rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 747
 748        MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
 749        MLX5_SET64(modify_rq_in, in, modify_bitmask,
 750                   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 751        MLX5_SET(rqc, rqc, vsd, vsd);
 752        MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 753
 754        err = mlx5_core_modify_rq(mdev, rq->rqn, in);
 755
 756        kvfree(in);
 757
 758        return err;
 759}
 760
 761void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 762{
 763        mlx5_core_destroy_rq(rq->mdev, rq->rqn);
 764}
 765
 766int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
 767{
 768        unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
 769
 770        u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
 771
 772        do {
 773                if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
 774                        return 0;
 775
 776                msleep(20);
 777        } while (time_before(jiffies, exp_time));
 778
 779        netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
 780                    rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
 781
 782        mlx5e_reporter_rx_timeout(rq);
 783        return -ETIMEDOUT;
 784}
 785
 786void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq)
 787{
 788        struct mlx5_wq_ll *wq;
 789        u16 head;
 790        int i;
 791
 792        if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
 793                return;
 794
 795        wq = &rq->mpwqe.wq;
 796        head = wq->head;
 797
 798        /* Outstanding UMR WQEs (in progress) start at wq->head */
 799        for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
 800                rq->dealloc_wqe(rq, head);
 801                head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
 802        }
 803
 804        rq->mpwqe.actual_wq_head = wq->head;
 805        rq->mpwqe.umr_in_progress = 0;
 806        rq->mpwqe.umr_completed = 0;
 807}
 808
 809void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 810{
 811        __be16 wqe_ix_be;
 812        u16 wqe_ix;
 813
 814        if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 815                struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
 816
 817                mlx5e_free_rx_in_progress_descs(rq);
 818
 819                while (!mlx5_wq_ll_is_empty(wq)) {
 820                        struct mlx5e_rx_wqe_ll *wqe;
 821
 822                        wqe_ix_be = *wq->tail_next;
 823                        wqe_ix    = be16_to_cpu(wqe_ix_be);
 824                        wqe       = mlx5_wq_ll_get_wqe(wq, wqe_ix);
 825                        rq->dealloc_wqe(rq, wqe_ix);
 826                        mlx5_wq_ll_pop(wq, wqe_ix_be,
 827                                       &wqe->next.next_wqe_index);
 828                }
 829        } else {
 830                struct mlx5_wq_cyc *wq = &rq->wqe.wq;
 831
 832                while (!mlx5_wq_cyc_is_empty(wq)) {
 833                        wqe_ix = mlx5_wq_cyc_get_tail(wq);
 834                        rq->dealloc_wqe(rq, wqe_ix);
 835                        mlx5_wq_cyc_pop(wq);
 836                }
 837        }
 838
 839}
 840
 841int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
 842                  struct mlx5e_xsk_param *xsk, int node,
 843                  struct mlx5e_rq *rq)
 844{
 845        struct mlx5_core_dev *mdev = rq->mdev;
 846        int err;
 847
 848        err = mlx5e_alloc_rq(params, xsk, param, node, rq);
 849        if (err)
 850                return err;
 851
 852        err = mlx5e_create_rq(rq, param);
 853        if (err)
 854                goto err_free_rq;
 855
 856        err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
 857        if (err)
 858                goto err_destroy_rq;
 859
 860        if (mlx5e_is_tls_on(rq->priv) && !mlx5e_accel_is_ktls_device(mdev))
 861                __set_bit(MLX5E_RQ_STATE_FPGA_TLS, &rq->state); /* must be FPGA */
 862
 863        if (MLX5_CAP_ETH(mdev, cqe_checksum_full))
 864                __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state);
 865
 866        if (params->rx_dim_enabled)
 867                __set_bit(MLX5E_RQ_STATE_AM, &rq->state);
 868
 869        /* We disable csum_complete when XDP is enabled since
 870         * XDP programs might manipulate packets which will render
 871         * skb->checksum incorrect.
 872         */
 873        if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog)
 874                __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state);
 875
 876        /* For CQE compression on striding RQ, use stride index provided by
 877         * HW if capability is supported.
 878         */
 879        if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) &&
 880            MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index))
 881                __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state);
 882
 883        return 0;
 884
 885err_destroy_rq:
 886        mlx5e_destroy_rq(rq);
 887err_free_rq:
 888        mlx5e_free_rq(rq);
 889
 890        return err;
 891}
 892
 893void mlx5e_activate_rq(struct mlx5e_rq *rq)
 894{
 895        set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
 896        if (rq->icosq) {
 897                mlx5e_trigger_irq(rq->icosq);
 898        } else {
 899                local_bh_disable();
 900                napi_schedule(rq->cq.napi);
 901                local_bh_enable();
 902        }
 903}
 904
 905void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
 906{
 907        clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
 908        synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
 909}
 910
 911void mlx5e_close_rq(struct mlx5e_rq *rq)
 912{
 913        cancel_work_sync(&rq->dim.work);
 914        if (rq->icosq)
 915                cancel_work_sync(&rq->icosq->recover_work);
 916        cancel_work_sync(&rq->recover_work);
 917        mlx5e_destroy_rq(rq);
 918        mlx5e_free_rx_descs(rq);
 919        mlx5e_free_rq(rq);
 920}
 921
 922static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
 923{
 924        kvfree(sq->db.xdpi_fifo.xi);
 925        kvfree(sq->db.wqe_info);
 926}
 927
 928static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
 929{
 930        struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
 931        int wq_sz        = mlx5_wq_cyc_get_size(&sq->wq);
 932        int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 933
 934        xdpi_fifo->xi = kvzalloc_node(sizeof(*xdpi_fifo->xi) * dsegs_per_wq,
 935                                      GFP_KERNEL, numa);
 936        if (!xdpi_fifo->xi)
 937                return -ENOMEM;
 938
 939        xdpi_fifo->pc   = &sq->xdpi_fifo_pc;
 940        xdpi_fifo->cc   = &sq->xdpi_fifo_cc;
 941        xdpi_fifo->mask = dsegs_per_wq - 1;
 942
 943        return 0;
 944}
 945
 946static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
 947{
 948        int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
 949        int err;
 950
 951        sq->db.wqe_info = kvzalloc_node(sizeof(*sq->db.wqe_info) * wq_sz,
 952                                        GFP_KERNEL, numa);
 953        if (!sq->db.wqe_info)
 954                return -ENOMEM;
 955
 956        err = mlx5e_alloc_xdpsq_fifo(sq, numa);
 957        if (err) {
 958                mlx5e_free_xdpsq_db(sq);
 959                return err;
 960        }
 961
 962        return 0;
 963}
 964
 965static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
 966                             struct mlx5e_params *params,
 967                             struct xsk_buff_pool *xsk_pool,
 968                             struct mlx5e_sq_param *param,
 969                             struct mlx5e_xdpsq *sq,
 970                             bool is_redirect)
 971{
 972        void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
 973        struct mlx5_core_dev *mdev = c->mdev;
 974        struct mlx5_wq_cyc *wq = &sq->wq;
 975        int err;
 976
 977        sq->pdev      = c->pdev;
 978        sq->mkey_be   = c->mkey_be;
 979        sq->channel   = c;
 980        sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
 981        sq->min_inline_mode = params->tx_min_inline_mode;
 982        sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 983        sq->xsk_pool  = xsk_pool;
 984
 985        sq->stats = sq->xsk_pool ?
 986                &c->priv->channel_stats[c->ix].xsksq :
 987                is_redirect ?
 988                        &c->priv->channel_stats[c->ix].xdpsq :
 989                        &c->priv->channel_stats[c->ix].rq_xdpsq;
 990
 991        param->wq.db_numa_node = cpu_to_node(c->cpu);
 992        err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
 993        if (err)
 994                return err;
 995        wq->db = &wq->db[MLX5_SND_DBR];
 996
 997        err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
 998        if (err)
 999                goto err_sq_wq_destroy;
1000
1001        return 0;
1002
1003err_sq_wq_destroy:
1004        mlx5_wq_destroy(&sq->wq_ctrl);
1005
1006        return err;
1007}
1008
1009static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1010{
1011        mlx5e_free_xdpsq_db(sq);
1012        mlx5_wq_destroy(&sq->wq_ctrl);
1013}
1014
1015static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
1016{
1017        kvfree(sq->db.wqe_info);
1018}
1019
1020static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1021{
1022        int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1023        size_t size;
1024
1025        size = array_size(wq_sz, sizeof(*sq->db.wqe_info));
1026        sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1027        if (!sq->db.wqe_info)
1028                return -ENOMEM;
1029
1030        return 0;
1031}
1032
1033static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
1034{
1035        struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1036                                              recover_work);
1037
1038        mlx5e_reporter_icosq_cqe_err(sq);
1039}
1040
1041static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
1042                             struct mlx5e_sq_param *param,
1043                             struct mlx5e_icosq *sq)
1044{
1045        void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1046        struct mlx5_core_dev *mdev = c->mdev;
1047        struct mlx5_wq_cyc *wq = &sq->wq;
1048        int err;
1049
1050        sq->channel   = c;
1051        sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1052        sq->reserved_room = param->stop_room;
1053
1054        param->wq.db_numa_node = cpu_to_node(c->cpu);
1055        err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1056        if (err)
1057                return err;
1058        wq->db = &wq->db[MLX5_SND_DBR];
1059
1060        err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1061        if (err)
1062                goto err_sq_wq_destroy;
1063
1064        INIT_WORK(&sq->recover_work, mlx5e_icosq_err_cqe_work);
1065
1066        return 0;
1067
1068err_sq_wq_destroy:
1069        mlx5_wq_destroy(&sq->wq_ctrl);
1070
1071        return err;
1072}
1073
1074static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
1075{
1076        mlx5e_free_icosq_db(sq);
1077        mlx5_wq_destroy(&sq->wq_ctrl);
1078}
1079
1080void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
1081{
1082        kvfree(sq->db.wqe_info);
1083        kvfree(sq->db.skb_fifo.fifo);
1084        kvfree(sq->db.dma_fifo);
1085}
1086
1087int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
1088{
1089        int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1090        int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1091
1092        sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1093                                                   sizeof(*sq->db.dma_fifo)),
1094                                        GFP_KERNEL, numa);
1095        sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz,
1096                                                        sizeof(*sq->db.skb_fifo.fifo)),
1097                                        GFP_KERNEL, numa);
1098        sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1099                                                   sizeof(*sq->db.wqe_info)),
1100                                        GFP_KERNEL, numa);
1101        if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) {
1102                mlx5e_free_txqsq_db(sq);
1103                return -ENOMEM;
1104        }
1105
1106        sq->dma_fifo_mask = df_sz - 1;
1107
1108        sq->db.skb_fifo.pc   = &sq->skb_fifo_pc;
1109        sq->db.skb_fifo.cc   = &sq->skb_fifo_cc;
1110        sq->db.skb_fifo.mask = df_sz - 1;
1111
1112        return 0;
1113}
1114
1115static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
1116                             int txq_ix,
1117                             struct mlx5e_params *params,
1118                             struct mlx5e_sq_param *param,
1119                             struct mlx5e_txqsq *sq,
1120                             int tc)
1121{
1122        void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1123        struct mlx5_core_dev *mdev = c->mdev;
1124        struct mlx5_wq_cyc *wq = &sq->wq;
1125        int err;
1126
1127        sq->pdev      = c->pdev;
1128        sq->tstamp    = c->tstamp;
1129        sq->clock     = &mdev->clock;
1130        sq->mkey_be   = c->mkey_be;
1131        sq->netdev    = c->netdev;
1132        sq->mdev      = c->mdev;
1133        sq->priv      = c->priv;
1134        sq->ch_ix     = c->ix;
1135        sq->txq_ix    = txq_ix;
1136        sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
1137        sq->min_inline_mode = params->tx_min_inline_mode;
1138        sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1139        INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
1140        if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
1141                set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
1142        if (MLX5_IPSEC_DEV(c->priv->mdev))
1143                set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
1144        if (param->is_mpw)
1145                set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state);
1146        sq->stop_room = param->stop_room;
1147        sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev);
1148
1149        param->wq.db_numa_node = cpu_to_node(c->cpu);
1150        err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1151        if (err)
1152                return err;
1153        wq->db    = &wq->db[MLX5_SND_DBR];
1154
1155        err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
1156        if (err)
1157                goto err_sq_wq_destroy;
1158
1159        INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1160        sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1161
1162        return 0;
1163
1164err_sq_wq_destroy:
1165        mlx5_wq_destroy(&sq->wq_ctrl);
1166
1167        return err;
1168}
1169
1170void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
1171{
1172        mlx5e_free_txqsq_db(sq);
1173        mlx5_wq_destroy(&sq->wq_ctrl);
1174}
1175
1176static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
1177                           struct mlx5e_sq_param *param,
1178                           struct mlx5e_create_sq_param *csp,
1179                           u32 *sqn)
1180{
1181        u8 ts_format;
1182        void *in;
1183        void *sqc;
1184        void *wq;
1185        int inlen;
1186        int err;
1187
1188        inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1189                sizeof(u64) * csp->wq_ctrl->buf.npages;
1190        in = kvzalloc(inlen, GFP_KERNEL);
1191        if (!in)
1192                return -ENOMEM;
1193
1194        ts_format = mlx5_is_real_time_sq(mdev) ?
1195                            MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1196                            MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1197        sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1198        wq = MLX5_ADDR_OF(sqc, sqc, wq);
1199
1200        memcpy(sqc, param->sqc, sizeof(param->sqc));
1201        MLX5_SET(sqc,  sqc, tis_lst_sz, csp->tis_lst_sz);
1202        MLX5_SET(sqc,  sqc, tis_num_0, csp->tisn);
1203        MLX5_SET(sqc,  sqc, cqn, csp->cqn);
1204        MLX5_SET(sqc,  sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn);
1205        MLX5_SET(sqc,  sqc, ts_format, ts_format);
1206
1207
1208        if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
1209                MLX5_SET(sqc,  sqc, min_wqe_inline_mode, csp->min_inline_mode);
1210
1211        MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
1212        MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
1213
1214        MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
1215        MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.hw_objs.bfreg.index);
1216        MLX5_SET(wq,   wq, log_wq_pg_sz,  csp->wq_ctrl->buf.page_shift -
1217                                          MLX5_ADAPTER_PAGE_SHIFT);
1218        MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
1219
1220        mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1221                                  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1222
1223        err = mlx5_core_create_sq(mdev, in, inlen, sqn);
1224
1225        kvfree(in);
1226
1227        return err;
1228}
1229
1230int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1231                    struct mlx5e_modify_sq_param *p)
1232{
1233        u64 bitmask = 0;
1234        void *in;
1235        void *sqc;
1236        int inlen;
1237        int err;
1238
1239        inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1240        in = kvzalloc(inlen, GFP_KERNEL);
1241        if (!in)
1242                return -ENOMEM;
1243
1244        sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1245
1246        MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1247        MLX5_SET(sqc, sqc, state, p->next_state);
1248        if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
1249                bitmask |= 1;
1250                MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
1251        }
1252        if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) {
1253                bitmask |= 1 << 2;
1254                MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id);
1255        }
1256        MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask);
1257
1258        err = mlx5_core_modify_sq(mdev, sqn, in);
1259
1260        kvfree(in);
1261
1262        return err;
1263}
1264
1265static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
1266{
1267        mlx5_core_destroy_sq(mdev, sqn);
1268}
1269
1270int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1271                        struct mlx5e_sq_param *param,
1272                        struct mlx5e_create_sq_param *csp,
1273                        u16 qos_queue_group_id,
1274                        u32 *sqn)
1275{
1276        struct mlx5e_modify_sq_param msp = {0};
1277        int err;
1278
1279        err = mlx5e_create_sq(mdev, param, csp, sqn);
1280        if (err)
1281                return err;
1282
1283        msp.curr_state = MLX5_SQC_STATE_RST;
1284        msp.next_state = MLX5_SQC_STATE_RDY;
1285        if (qos_queue_group_id) {
1286                msp.qos_update = true;
1287                msp.qos_queue_group_id = qos_queue_group_id;
1288        }
1289        err = mlx5e_modify_sq(mdev, *sqn, &msp);
1290        if (err)
1291                mlx5e_destroy_sq(mdev, *sqn);
1292
1293        return err;
1294}
1295
1296static int mlx5e_set_sq_maxrate(struct net_device *dev,
1297                                struct mlx5e_txqsq *sq, u32 rate);
1298
1299int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1300                     struct mlx5e_params *params, struct mlx5e_sq_param *param,
1301                     struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, u16 qos_qid)
1302{
1303        struct mlx5e_create_sq_param csp = {};
1304        u32 tx_rate;
1305        int err;
1306
1307        err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
1308        if (err)
1309                return err;
1310
1311        if (qos_queue_group_id)
1312                sq->stats = c->priv->htb.qos_sq_stats[qos_qid];
1313        else
1314                sq->stats = &c->priv->channel_stats[c->ix].sq[tc];
1315
1316        csp.tisn            = tisn;
1317        csp.tis_lst_sz      = 1;
1318        csp.cqn             = sq->cq.mcq.cqn;
1319        csp.wq_ctrl         = &sq->wq_ctrl;
1320        csp.min_inline_mode = sq->min_inline_mode;
1321        err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn);
1322        if (err)
1323                goto err_free_txqsq;
1324
1325        tx_rate = c->priv->tx_rates[sq->txq_ix];
1326        if (tx_rate)
1327                mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
1328
1329        if (params->tx_dim_enabled)
1330                sq->state |= BIT(MLX5E_SQ_STATE_AM);
1331
1332        return 0;
1333
1334err_free_txqsq:
1335        mlx5e_free_txqsq(sq);
1336
1337        return err;
1338}
1339
1340void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1341{
1342        sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix);
1343        set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1344        netdev_tx_reset_queue(sq->txq);
1345        netif_tx_start_queue(sq->txq);
1346}
1347
1348void mlx5e_tx_disable_queue(struct netdev_queue *txq)
1349{
1350        __netif_tx_lock_bh(txq);
1351        netif_tx_stop_queue(txq);
1352        __netif_tx_unlock_bh(txq);
1353}
1354
1355void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
1356{
1357        struct mlx5_wq_cyc *wq = &sq->wq;
1358
1359        clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1360        synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
1361
1362        mlx5e_tx_disable_queue(sq->txq);
1363
1364        /* last doorbell out, godspeed .. */
1365        if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1366                u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
1367                struct mlx5e_tx_wqe *nop;
1368
1369                sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) {
1370                        .num_wqebbs = 1,
1371                };
1372
1373                nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1374                mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
1375        }
1376}
1377
1378void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1379{
1380        struct mlx5_core_dev *mdev = sq->mdev;
1381        struct mlx5_rate_limit rl = {0};
1382
1383        cancel_work_sync(&sq->dim.work);
1384        cancel_work_sync(&sq->recover_work);
1385        mlx5e_destroy_sq(mdev, sq->sqn);
1386        if (sq->rate_limit) {
1387                rl.rate = sq->rate_limit;
1388                mlx5_rl_remove_rate(mdev, &rl);
1389        }
1390        mlx5e_free_txqsq_descs(sq);
1391        mlx5e_free_txqsq(sq);
1392}
1393
1394void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
1395{
1396        struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq,
1397                                              recover_work);
1398
1399        mlx5e_reporter_tx_err_cqe(sq);
1400}
1401
1402int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
1403                     struct mlx5e_sq_param *param, struct mlx5e_icosq *sq)
1404{
1405        struct mlx5e_create_sq_param csp = {};
1406        int err;
1407
1408        err = mlx5e_alloc_icosq(c, param, sq);
1409        if (err)
1410                return err;
1411
1412        csp.cqn             = sq->cq.mcq.cqn;
1413        csp.wq_ctrl         = &sq->wq_ctrl;
1414        csp.min_inline_mode = params->tx_min_inline_mode;
1415        err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1416        if (err)
1417                goto err_free_icosq;
1418
1419        if (param->is_tls) {
1420                sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list();
1421                if (IS_ERR(sq->ktls_resync)) {
1422                        err = PTR_ERR(sq->ktls_resync);
1423                        goto err_destroy_icosq;
1424                }
1425        }
1426        return 0;
1427
1428err_destroy_icosq:
1429        mlx5e_destroy_sq(c->mdev, sq->sqn);
1430err_free_icosq:
1431        mlx5e_free_icosq(sq);
1432
1433        return err;
1434}
1435
1436void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
1437{
1438        set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1439}
1440
1441void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
1442{
1443        clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1444        synchronize_net(); /* Sync with NAPI. */
1445}
1446
1447void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1448{
1449        struct mlx5e_channel *c = sq->channel;
1450
1451        if (sq->ktls_resync)
1452                mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync);
1453        mlx5e_destroy_sq(c->mdev, sq->sqn);
1454        mlx5e_free_icosq_descs(sq);
1455        mlx5e_free_icosq(sq);
1456}
1457
1458int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1459                     struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
1460                     struct mlx5e_xdpsq *sq, bool is_redirect)
1461{
1462        struct mlx5e_create_sq_param csp = {};
1463        int err;
1464
1465        err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect);
1466        if (err)
1467                return err;
1468
1469        csp.tis_lst_sz      = 1;
1470        csp.tisn            = c->priv->tisn[c->lag_port][0]; /* tc = 0 */
1471        csp.cqn             = sq->cq.mcq.cqn;
1472        csp.wq_ctrl         = &sq->wq_ctrl;
1473        csp.min_inline_mode = sq->min_inline_mode;
1474        set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1475        err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
1476        if (err)
1477                goto err_free_xdpsq;
1478
1479        mlx5e_set_xmit_fp(sq, param->is_mpw);
1480
1481        if (!param->is_mpw) {
1482                unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1483                unsigned int inline_hdr_sz = 0;
1484                int i;
1485
1486                if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1487                        inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1488                        ds_cnt++;
1489                }
1490
1491                /* Pre initialize fixed WQE fields */
1492                for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1493                        struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1494                        struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1495                        struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
1496                        struct mlx5_wqe_data_seg *dseg;
1497
1498                        sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
1499                                .num_wqebbs = 1,
1500                                .num_pkts   = 1,
1501                        };
1502
1503                        cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1504                        eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1505
1506                        dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1507                        dseg->lkey = sq->mkey_be;
1508                }
1509        }
1510
1511        return 0;
1512
1513err_free_xdpsq:
1514        clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1515        mlx5e_free_xdpsq(sq);
1516
1517        return err;
1518}
1519
1520void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1521{
1522        struct mlx5e_channel *c = sq->channel;
1523
1524        clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1525        synchronize_net(); /* Sync with NAPI. */
1526
1527        mlx5e_destroy_sq(c->mdev, sq->sqn);
1528        mlx5e_free_xdpsq_descs(sq);
1529        mlx5e_free_xdpsq(sq);
1530}
1531
1532static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv,
1533                                 struct mlx5e_cq_param *param,
1534                                 struct mlx5e_cq *cq)
1535{
1536        struct mlx5_core_dev *mdev = priv->mdev;
1537        struct mlx5_core_cq *mcq = &cq->mcq;
1538        int err;
1539        u32 i;
1540
1541        err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1542                               &cq->wq_ctrl);
1543        if (err)
1544                return err;
1545
1546        mcq->cqe_sz     = 64;
1547        mcq->set_ci_db  = cq->wq_ctrl.db.db;
1548        mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1549        *mcq->set_ci_db = 0;
1550        *mcq->arm_db    = 0;
1551        mcq->vector     = param->eq_ix;
1552        mcq->comp       = mlx5e_completion_event;
1553        mcq->event      = mlx5e_cq_error_event;
1554
1555        for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1556                struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1557
1558                cqe->op_own = 0xf1;
1559        }
1560
1561        cq->mdev = mdev;
1562        cq->netdev = priv->netdev;
1563        cq->priv = priv;
1564
1565        return 0;
1566}
1567
1568static int mlx5e_alloc_cq(struct mlx5e_priv *priv,
1569                          struct mlx5e_cq_param *param,
1570                          struct mlx5e_create_cq_param *ccp,
1571                          struct mlx5e_cq *cq)
1572{
1573        int err;
1574
1575        param->wq.buf_numa_node = ccp->node;
1576        param->wq.db_numa_node  = ccp->node;
1577        param->eq_ix            = ccp->ix;
1578
1579        err = mlx5e_alloc_cq_common(priv, param, cq);
1580
1581        cq->napi     = ccp->napi;
1582        cq->ch_stats = ccp->ch_stats;
1583
1584        return err;
1585}
1586
1587static void mlx5e_free_cq(struct mlx5e_cq *cq)
1588{
1589        mlx5_wq_destroy(&cq->wq_ctrl);
1590}
1591
1592static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1593{
1594        u32 out[MLX5_ST_SZ_DW(create_cq_out)];
1595        struct mlx5_core_dev *mdev = cq->mdev;
1596        struct mlx5_core_cq *mcq = &cq->mcq;
1597
1598        void *in;
1599        void *cqc;
1600        int inlen;
1601        int eqn;
1602        int err;
1603
1604        err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn);
1605        if (err)
1606                return err;
1607
1608        inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1609                sizeof(u64) * cq->wq_ctrl.buf.npages;
1610        in = kvzalloc(inlen, GFP_KERNEL);
1611        if (!in)
1612                return -ENOMEM;
1613
1614        cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1615
1616        memcpy(cqc, param->cqc, sizeof(param->cqc));
1617
1618        mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1619                                  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1620
1621        MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1622        MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1623        MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
1624        MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1625                                            MLX5_ADAPTER_PAGE_SHIFT);
1626        MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1627
1628        err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
1629
1630        kvfree(in);
1631
1632        if (err)
1633                return err;
1634
1635        mlx5e_cq_arm(cq);
1636
1637        return 0;
1638}
1639
1640static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1641{
1642        mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
1643}
1644
1645int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder,
1646                  struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1647                  struct mlx5e_cq *cq)
1648{
1649        struct mlx5_core_dev *mdev = priv->mdev;
1650        int err;
1651
1652        err = mlx5e_alloc_cq(priv, param, ccp, cq);
1653        if (err)
1654                return err;
1655
1656        err = mlx5e_create_cq(cq, param);
1657        if (err)
1658                goto err_free_cq;
1659
1660        if (MLX5_CAP_GEN(mdev, cq_moderation))
1661                mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
1662        return 0;
1663
1664err_free_cq:
1665        mlx5e_free_cq(cq);
1666
1667        return err;
1668}
1669
1670void mlx5e_close_cq(struct mlx5e_cq *cq)
1671{
1672        mlx5e_destroy_cq(cq);
1673        mlx5e_free_cq(cq);
1674}
1675
1676static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1677                             struct mlx5e_params *params,
1678                             struct mlx5e_create_cq_param *ccp,
1679                             struct mlx5e_channel_param *cparam)
1680{
1681        int err;
1682        int tc;
1683
1684        for (tc = 0; tc < c->num_tc; tc++) {
1685                err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->txq_sq.cqp,
1686                                    ccp, &c->sq[tc].cq);
1687                if (err)
1688                        goto err_close_tx_cqs;
1689        }
1690
1691        return 0;
1692
1693err_close_tx_cqs:
1694        for (tc--; tc >= 0; tc--)
1695                mlx5e_close_cq(&c->sq[tc].cq);
1696
1697        return err;
1698}
1699
1700static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1701{
1702        int tc;
1703
1704        for (tc = 0; tc < c->num_tc; tc++)
1705                mlx5e_close_cq(&c->sq[tc].cq);
1706}
1707
1708static int mlx5e_open_sqs(struct mlx5e_channel *c,
1709                          struct mlx5e_params *params,
1710                          struct mlx5e_channel_param *cparam)
1711{
1712        int err, tc;
1713
1714        for (tc = 0; tc < params->num_tc; tc++) {
1715                int txq_ix = c->ix + tc * params->num_channels;
1716
1717                err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix,
1718                                       params, &cparam->txq_sq, &c->sq[tc], tc, 0, 0);
1719                if (err)
1720                        goto err_close_sqs;
1721        }
1722
1723        return 0;
1724
1725err_close_sqs:
1726        for (tc--; tc >= 0; tc--)
1727                mlx5e_close_txqsq(&c->sq[tc]);
1728
1729        return err;
1730}
1731
1732static void mlx5e_close_sqs(struct mlx5e_channel *c)
1733{
1734        int tc;
1735
1736        for (tc = 0; tc < c->num_tc; tc++)
1737                mlx5e_close_txqsq(&c->sq[tc]);
1738}
1739
1740static int mlx5e_set_sq_maxrate(struct net_device *dev,
1741                                struct mlx5e_txqsq *sq, u32 rate)
1742{
1743        struct mlx5e_priv *priv = netdev_priv(dev);
1744        struct mlx5_core_dev *mdev = priv->mdev;
1745        struct mlx5e_modify_sq_param msp = {0};
1746        struct mlx5_rate_limit rl = {0};
1747        u16 rl_index = 0;
1748        int err;
1749
1750        if (rate == sq->rate_limit)
1751                /* nothing to do */
1752                return 0;
1753
1754        if (sq->rate_limit) {
1755                rl.rate = sq->rate_limit;
1756                /* remove current rl index to free space to next ones */
1757                mlx5_rl_remove_rate(mdev, &rl);
1758        }
1759
1760        sq->rate_limit = 0;
1761
1762        if (rate) {
1763                rl.rate = rate;
1764                err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
1765                if (err) {
1766                        netdev_err(dev, "Failed configuring rate %u: %d\n",
1767                                   rate, err);
1768                        return err;
1769                }
1770        }
1771
1772        msp.curr_state = MLX5_SQC_STATE_RDY;
1773        msp.next_state = MLX5_SQC_STATE_RDY;
1774        msp.rl_index   = rl_index;
1775        msp.rl_update  = true;
1776        err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1777        if (err) {
1778                netdev_err(dev, "Failed configuring rate %u: %d\n",
1779                           rate, err);
1780                /* remove the rate from the table */
1781                if (rate)
1782                        mlx5_rl_remove_rate(mdev, &rl);
1783                return err;
1784        }
1785
1786        sq->rate_limit = rate;
1787        return 0;
1788}
1789
1790static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1791{
1792        struct mlx5e_priv *priv = netdev_priv(dev);
1793        struct mlx5_core_dev *mdev = priv->mdev;
1794        struct mlx5e_txqsq *sq = priv->txq2sq[index];
1795        int err = 0;
1796
1797        if (!mlx5_rl_is_supported(mdev)) {
1798                netdev_err(dev, "Rate limiting is not supported on this device\n");
1799                return -EINVAL;
1800        }
1801
1802        /* rate is given in Mb/sec, HW config is in Kb/sec */
1803        rate = rate << 10;
1804
1805        /* Check whether rate in valid range, 0 is always valid */
1806        if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1807                netdev_err(dev, "TX rate %u, is not in range\n", rate);
1808                return -ERANGE;
1809        }
1810
1811        mutex_lock(&priv->state_lock);
1812        if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1813                err = mlx5e_set_sq_maxrate(dev, sq, rate);
1814        if (!err)
1815                priv->tx_rates[index] = rate;
1816        mutex_unlock(&priv->state_lock);
1817
1818        return err;
1819}
1820
1821static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
1822                             struct mlx5e_rq_param *rq_params)
1823{
1824        int err;
1825
1826        err = mlx5e_init_rxq_rq(c, params, &c->rq);
1827        if (err)
1828                return err;
1829
1830        return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq);
1831}
1832
1833static int mlx5e_open_queues(struct mlx5e_channel *c,
1834                             struct mlx5e_params *params,
1835                             struct mlx5e_channel_param *cparam)
1836{
1837        struct dim_cq_moder icocq_moder = {0, 0};
1838        struct mlx5e_create_cq_param ccp;
1839        int err;
1840
1841        mlx5e_build_create_cq_param(&ccp, c);
1842
1843        err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->async_icosq.cqp, &ccp,
1844                            &c->async_icosq.cq);
1845        if (err)
1846                return err;
1847
1848        err = mlx5e_open_cq(c->priv, icocq_moder, &cparam->icosq.cqp, &ccp,
1849                            &c->icosq.cq);
1850        if (err)
1851                goto err_close_async_icosq_cq;
1852
1853        err = mlx5e_open_tx_cqs(c, params, &ccp, cparam);
1854        if (err)
1855                goto err_close_icosq_cq;
1856
1857        err = mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
1858                            &c->xdpsq.cq);
1859        if (err)
1860                goto err_close_tx_cqs;
1861
1862        err = mlx5e_open_cq(c->priv, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
1863                            &c->rq.cq);
1864        if (err)
1865                goto err_close_xdp_tx_cqs;
1866
1867        err = c->xdp ? mlx5e_open_cq(c->priv, params->tx_cq_moderation, &cparam->xdp_sq.cqp,
1868                                     &ccp, &c->rq_xdpsq.cq) : 0;
1869        if (err)
1870                goto err_close_rx_cq;
1871
1872        spin_lock_init(&c->async_icosq_lock);
1873
1874        err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq);
1875        if (err)
1876                goto err_close_xdpsq_cq;
1877
1878        err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
1879        if (err)
1880                goto err_close_async_icosq;
1881
1882        err = mlx5e_open_sqs(c, params, cparam);
1883        if (err)
1884                goto err_close_icosq;
1885
1886        err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
1887        if (err)
1888                goto err_close_sqs;
1889
1890        if (c->xdp) {
1891                err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
1892                                       &c->rq_xdpsq, false);
1893                if (err)
1894                        goto err_close_rq;
1895        }
1896
1897        err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
1898        if (err)
1899                goto err_close_xdp_sq;
1900
1901        return 0;
1902
1903err_close_xdp_sq:
1904        if (c->xdp)
1905                mlx5e_close_xdpsq(&c->rq_xdpsq);
1906
1907err_close_rq:
1908        mlx5e_close_rq(&c->rq);
1909
1910err_close_sqs:
1911        mlx5e_close_sqs(c);
1912
1913err_close_icosq:
1914        mlx5e_close_icosq(&c->icosq);
1915
1916err_close_async_icosq:
1917        mlx5e_close_icosq(&c->async_icosq);
1918
1919err_close_xdpsq_cq:
1920        if (c->xdp)
1921                mlx5e_close_cq(&c->rq_xdpsq.cq);
1922
1923err_close_rx_cq:
1924        mlx5e_close_cq(&c->rq.cq);
1925
1926err_close_xdp_tx_cqs:
1927        mlx5e_close_cq(&c->xdpsq.cq);
1928
1929err_close_tx_cqs:
1930        mlx5e_close_tx_cqs(c);
1931
1932err_close_icosq_cq:
1933        mlx5e_close_cq(&c->icosq.cq);
1934
1935err_close_async_icosq_cq:
1936        mlx5e_close_cq(&c->async_icosq.cq);
1937
1938        return err;
1939}
1940
1941static void mlx5e_close_queues(struct mlx5e_channel *c)
1942{
1943        mlx5e_close_xdpsq(&c->xdpsq);
1944        if (c->xdp)
1945                mlx5e_close_xdpsq(&c->rq_xdpsq);
1946        mlx5e_close_rq(&c->rq);
1947        mlx5e_close_sqs(c);
1948        mlx5e_close_icosq(&c->icosq);
1949        mlx5e_close_icosq(&c->async_icosq);
1950        if (c->xdp)
1951                mlx5e_close_cq(&c->rq_xdpsq.cq);
1952        mlx5e_close_cq(&c->rq.cq);
1953        mlx5e_close_cq(&c->xdpsq.cq);
1954        mlx5e_close_tx_cqs(c);
1955        mlx5e_close_cq(&c->icosq.cq);
1956        mlx5e_close_cq(&c->async_icosq.cq);
1957}
1958
1959static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
1960{
1961        u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id);
1962
1963        return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev);
1964}
1965
1966static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1967                              struct mlx5e_params *params,
1968                              struct mlx5e_channel_param *cparam,
1969                              struct xsk_buff_pool *xsk_pool,
1970                              struct mlx5e_channel **cp)
1971{
1972        int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix));
1973        struct net_device *netdev = priv->netdev;
1974        struct mlx5e_xsk_param xsk;
1975        struct mlx5e_channel *c;
1976        unsigned int irq;
1977        int err;
1978
1979        err = mlx5_vector2irqn(priv->mdev, ix, &irq);
1980        if (err)
1981                return err;
1982
1983        c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1984        if (!c)
1985                return -ENOMEM;
1986
1987        c->priv     = priv;
1988        c->mdev     = priv->mdev;
1989        c->tstamp   = &priv->tstamp;
1990        c->ix       = ix;
1991        c->cpu      = cpu;
1992        c->pdev     = mlx5_core_dma_dev(priv->mdev);
1993        c->netdev   = priv->netdev;
1994        c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey.key);
1995        c->num_tc   = params->num_tc;
1996        c->xdp      = !!params->xdp_prog;
1997        c->stats    = &priv->channel_stats[ix].ch;
1998        c->aff_mask = irq_get_effective_affinity_mask(irq);
1999        c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix);
2000
2001        netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
2002
2003        err = mlx5e_open_queues(c, params, cparam);
2004        if (unlikely(err))
2005                goto err_napi_del;
2006
2007        if (xsk_pool) {
2008                mlx5e_build_xsk_param(xsk_pool, &xsk);
2009                err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
2010                if (unlikely(err))
2011                        goto err_close_queues;
2012        }
2013
2014        *cp = c;
2015
2016        return 0;
2017
2018err_close_queues:
2019        mlx5e_close_queues(c);
2020
2021err_napi_del:
2022        netif_napi_del(&c->napi);
2023
2024        kvfree(c);
2025
2026        return err;
2027}
2028
2029static void mlx5e_activate_channel(struct mlx5e_channel *c)
2030{
2031        int tc;
2032
2033        napi_enable(&c->napi);
2034
2035        for (tc = 0; tc < c->num_tc; tc++)
2036                mlx5e_activate_txqsq(&c->sq[tc]);
2037        mlx5e_activate_icosq(&c->icosq);
2038        mlx5e_activate_icosq(&c->async_icosq);
2039        mlx5e_activate_rq(&c->rq);
2040
2041        if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2042                mlx5e_activate_xsk(c);
2043}
2044
2045static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2046{
2047        int tc;
2048
2049        if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2050                mlx5e_deactivate_xsk(c);
2051
2052        mlx5e_deactivate_rq(&c->rq);
2053        mlx5e_deactivate_icosq(&c->async_icosq);
2054        mlx5e_deactivate_icosq(&c->icosq);
2055        for (tc = 0; tc < c->num_tc; tc++)
2056                mlx5e_deactivate_txqsq(&c->sq[tc]);
2057        mlx5e_qos_deactivate_queues(c);
2058
2059        napi_disable(&c->napi);
2060}
2061
2062static void mlx5e_close_channel(struct mlx5e_channel *c)
2063{
2064        if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2065                mlx5e_close_xsk(c);
2066        mlx5e_close_queues(c);
2067        mlx5e_qos_close_queues(c);
2068        netif_napi_del(&c->napi);
2069
2070        kvfree(c);
2071}
2072
2073int mlx5e_open_channels(struct mlx5e_priv *priv,
2074                        struct mlx5e_channels *chs)
2075{
2076        struct mlx5e_channel_param *cparam;
2077        int err = -ENOMEM;
2078        int i;
2079
2080        chs->num = chs->params.num_channels;
2081
2082        chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
2083        cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
2084        if (!chs->c || !cparam)
2085                goto err_free;
2086
2087        err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam);
2088        if (err)
2089                goto err_free;
2090
2091        for (i = 0; i < chs->num; i++) {
2092                struct xsk_buff_pool *xsk_pool = NULL;
2093
2094                if (chs->params.xdp_prog)
2095                        xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
2096
2097                err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]);
2098                if (err)
2099                        goto err_close_channels;
2100        }
2101
2102        if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) {
2103                err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp);
2104                if (err)
2105                        goto err_close_channels;
2106        }
2107
2108        err = mlx5e_qos_open_queues(priv, chs);
2109        if (err)
2110                goto err_close_ptp;
2111
2112        mlx5e_health_channels_update(priv);
2113        kvfree(cparam);
2114        return 0;
2115
2116err_close_ptp:
2117        if (chs->ptp)
2118                mlx5e_ptp_close(chs->ptp);
2119
2120err_close_channels:
2121        for (i--; i >= 0; i--)
2122                mlx5e_close_channel(chs->c[i]);
2123
2124err_free:
2125        kfree(chs->c);
2126        kvfree(cparam);
2127        chs->num = 0;
2128        return err;
2129}
2130
2131static void mlx5e_activate_channels(struct mlx5e_channels *chs)
2132{
2133        int i;
2134
2135        for (i = 0; i < chs->num; i++)
2136                mlx5e_activate_channel(chs->c[i]);
2137
2138        if (chs->ptp)
2139                mlx5e_ptp_activate_channel(chs->ptp);
2140}
2141
2142#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
2143
2144static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2145{
2146        int err = 0;
2147        int i;
2148
2149        for (i = 0; i < chs->num; i++) {
2150                int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2151
2152                err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
2153
2154                /* Don't wait on the XSK RQ, because the newer xdpsock sample
2155                 * doesn't provide any Fill Ring entries at the setup stage.
2156                 */
2157        }
2158
2159        return err ? -ETIMEDOUT : 0;
2160}
2161
2162static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2163{
2164        int i;
2165
2166        if (chs->ptp)
2167                mlx5e_ptp_deactivate_channel(chs->ptp);
2168
2169        for (i = 0; i < chs->num; i++)
2170                mlx5e_deactivate_channel(chs->c[i]);
2171}
2172
2173void mlx5e_close_channels(struct mlx5e_channels *chs)
2174{
2175        int i;
2176
2177        if (chs->ptp) {
2178                mlx5e_ptp_close(chs->ptp);
2179                chs->ptp = NULL;
2180        }
2181        for (i = 0; i < chs->num; i++)
2182                mlx5e_close_channel(chs->c[i]);
2183
2184        kfree(chs->c);
2185        chs->num = 0;
2186}
2187
2188static int
2189mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt)
2190{
2191        struct mlx5_core_dev *mdev = priv->mdev;
2192        void *rqtc;
2193        int inlen;
2194        int err;
2195        u32 *in;
2196        int i;
2197
2198        inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2199        in = kvzalloc(inlen, GFP_KERNEL);
2200        if (!in)
2201                return -ENOMEM;
2202
2203        rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2204
2205        MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2206        MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2207
2208        for (i = 0; i < sz; i++)
2209                MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2210
2211        err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
2212        if (!err)
2213                rqt->enabled = true;
2214
2215        kvfree(in);
2216        return err;
2217}
2218
2219void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
2220{
2221        rqt->enabled = false;
2222        mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
2223}
2224
2225int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv)
2226{
2227        struct mlx5e_rqt *rqt = &priv->indir_rqt;
2228        int err;
2229
2230        err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt);
2231        if (err)
2232                mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err);
2233        return err;
2234}
2235
2236int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n)
2237{
2238        int err;
2239        int ix;
2240
2241        for (ix = 0; ix < n; ix++) {
2242                err = mlx5e_create_rqt(priv, 1 /*size */, &tirs[ix].rqt);
2243                if (unlikely(err))
2244                        goto err_destroy_rqts;
2245        }
2246
2247        return 0;
2248
2249err_destroy_rqts:
2250        mlx5_core_warn(priv->mdev, "create rqts failed, %d\n", err);
2251        for (ix--; ix >= 0; ix--)
2252                mlx5e_destroy_rqt(priv, &tirs[ix].rqt);
2253
2254        return err;
2255}
2256
2257void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n)
2258{
2259        int i;
2260
2261        for (i = 0; i < n; i++)
2262                mlx5e_destroy_rqt(priv, &tirs[i].rqt);
2263}
2264
2265static int mlx5e_rx_hash_fn(int hfunc)
2266{
2267        return (hfunc == ETH_RSS_HASH_TOP) ?
2268               MLX5_RX_HASH_FN_TOEPLITZ :
2269               MLX5_RX_HASH_FN_INVERTED_XOR8;
2270}
2271
2272int mlx5e_bits_invert(unsigned long a, int size)
2273{
2274        int inv = 0;
2275        int i;
2276
2277        for (i = 0; i < size; i++)
2278                inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
2279
2280        return inv;
2281}
2282
2283static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz,
2284                                struct mlx5e_redirect_rqt_param rrp, void *rqtc)
2285{
2286        int i;
2287
2288        for (i = 0; i < sz; i++) {
2289                u32 rqn;
2290
2291                if (rrp.is_rss) {
2292                        int ix = i;
2293
2294                        if (rrp.rss.hfunc == ETH_RSS_HASH_XOR)
2295                                ix = mlx5e_bits_invert(i, ilog2(sz));
2296
2297                        ix = priv->rss_params.indirection_rqt[ix];
2298                        rqn = rrp.rss.channels->c[ix]->rq.rqn;
2299                } else {
2300                        rqn = rrp.rqn;
2301                }
2302                MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
2303        }
2304}
2305
2306int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
2307                       struct mlx5e_redirect_rqt_param rrp)
2308{
2309        struct mlx5_core_dev *mdev = priv->mdev;
2310        void *rqtc;
2311        int inlen;
2312        u32 *in;
2313        int err;
2314
2315        inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2316        in = kvzalloc(inlen, GFP_KERNEL);
2317        if (!in)
2318                return -ENOMEM;
2319
2320        rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2321
2322        MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2323        MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2324        mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc);
2325        err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
2326
2327        kvfree(in);
2328        return err;
2329}
2330
2331static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix,
2332                                struct mlx5e_redirect_rqt_param rrp)
2333{
2334        if (!rrp.is_rss)
2335                return rrp.rqn;
2336
2337        if (ix >= rrp.rss.channels->num)
2338                return priv->drop_rq.rqn;
2339
2340        return rrp.rss.channels->c[ix]->rq.rqn;
2341}
2342
2343static void mlx5e_redirect_rqts(struct mlx5e_priv *priv,
2344                                struct mlx5e_redirect_rqt_param rrp,
2345                                struct mlx5e_redirect_rqt_param *ptp_rrp)
2346{
2347        u32 rqtn;
2348        int ix;
2349
2350        if (priv->indir_rqt.enabled) {
2351                /* RSS RQ table */
2352                rqtn = priv->indir_rqt.rqtn;
2353                mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
2354        }
2355
2356        for (ix = 0; ix < priv->max_nch; ix++) {
2357                struct mlx5e_redirect_rqt_param direct_rrp = {
2358                        .is_rss = false,
2359                        {
2360                                .rqn    = mlx5e_get_direct_rqn(priv, ix, rrp)
2361                        },
2362                };
2363
2364                /* Direct RQ Tables */
2365                if (!priv->direct_tir[ix].rqt.enabled)
2366                        continue;
2367
2368                rqtn = priv->direct_tir[ix].rqt.rqtn;
2369                mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
2370        }
2371        if (ptp_rrp) {
2372                rqtn = priv->ptp_tir.rqt.rqtn;
2373                mlx5e_redirect_rqt(priv, rqtn, 1, *ptp_rrp);
2374        }
2375}
2376
2377static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv,
2378                                            struct mlx5e_channels *chs)
2379{
2380        bool rx_ptp_support = priv->profile->rx_ptp_support;
2381        struct mlx5e_redirect_rqt_param *ptp_rrp_p = NULL;
2382        struct mlx5e_redirect_rqt_param rrp = {
2383                .is_rss        = true,
2384                {
2385                        .rss = {
2386                                .channels  = chs,
2387                                .hfunc     = priv->rss_params.hfunc,
2388                        }
2389                },
2390        };
2391        struct mlx5e_redirect_rqt_param ptp_rrp;
2392
2393        if (rx_ptp_support) {
2394                u32 ptp_rqn;
2395
2396                ptp_rrp.is_rss = false;
2397                ptp_rrp.rqn = mlx5e_ptp_get_rqn(priv->channels.ptp, &ptp_rqn) ?
2398                              priv->drop_rq.rqn : ptp_rqn;
2399                ptp_rrp_p = &ptp_rrp;
2400        }
2401        mlx5e_redirect_rqts(priv, rrp, ptp_rrp_p);
2402}
2403
2404static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv)
2405{
2406        bool rx_ptp_support = priv->profile->rx_ptp_support;
2407        struct mlx5e_redirect_rqt_param drop_rrp = {
2408                .is_rss = false,
2409                {
2410                        .rqn = priv->drop_rq.rqn,
2411                },
2412        };
2413
2414        mlx5e_redirect_rqts(priv, drop_rrp, rx_ptp_support ? &drop_rrp : NULL);
2415}
2416
2417static const struct mlx5e_tirc_config tirc_default_config[MLX5E_NUM_INDIR_TIRS] = {
2418        [MLX5E_TT_IPV4_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2419                                .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
2420                                .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2421        },
2422        [MLX5E_TT_IPV6_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2423                                .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
2424                                .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2425        },
2426        [MLX5E_TT_IPV4_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2427                                .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
2428                                .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2429        },
2430        [MLX5E_TT_IPV6_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2431                                .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
2432                                .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
2433        },
2434        [MLX5E_TT_IPV4_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2435                                     .l4_prot_type = 0,
2436                                     .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2437        },
2438        [MLX5E_TT_IPV6_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2439                                     .l4_prot_type = 0,
2440                                     .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2441        },
2442        [MLX5E_TT_IPV4_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2443                                      .l4_prot_type = 0,
2444                                      .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2445        },
2446        [MLX5E_TT_IPV6_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2447                                      .l4_prot_type = 0,
2448                                      .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
2449        },
2450        [MLX5E_TT_IPV4] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
2451                            .l4_prot_type = 0,
2452                            .rx_hash_fields = MLX5_HASH_IP,
2453        },
2454        [MLX5E_TT_IPV6] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
2455                            .l4_prot_type = 0,
2456                            .rx_hash_fields = MLX5_HASH_IP,
2457        },
2458};
2459
2460struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt)
2461{
2462        return tirc_default_config[tt];
2463}
2464
2465static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
2466{
2467        if (!params->lro_en)
2468                return;
2469
2470#define ROUGH_MAX_L2_L3_HDR_SZ 256
2471
2472        MLX5_SET(tirc, tirc, lro_enable_mask,
2473                 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2474                 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2475        MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
2476                 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2477        MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout);
2478}
2479
2480void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params,
2481                                    const struct mlx5e_tirc_config *ttconfig,
2482                                    void *tirc, bool inner)
2483{
2484        void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) :
2485                             MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2486
2487        MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(rss_params->hfunc));
2488        if (rss_params->hfunc == ETH_RSS_HASH_TOP) {
2489                void *rss_key = MLX5_ADDR_OF(tirc, tirc,
2490                                             rx_hash_toeplitz_key);
2491                size_t len = MLX5_FLD_SZ_BYTES(tirc,
2492                                               rx_hash_toeplitz_key);
2493
2494                MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2495                memcpy(rss_key, rss_params->toeplitz_hash_key, len);
2496        }
2497        MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2498                 ttconfig->l3_prot_type);
2499        MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2500                 ttconfig->l4_prot_type);
2501        MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2502                 ttconfig->rx_hash_fields);
2503}
2504
2505static void mlx5e_update_rx_hash_fields(struct mlx5e_tirc_config *ttconfig,
2506                                        enum mlx5e_traffic_types tt,
2507                                        u32 rx_hash_fields)
2508{
2509        *ttconfig                = tirc_default_config[tt];
2510        ttconfig->rx_hash_fields = rx_hash_fields;
2511}
2512
2513void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in)
2514{
2515        void *tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2516        struct mlx5e_rss_params *rss = &priv->rss_params;
2517        struct mlx5_core_dev *mdev = priv->mdev;
2518        int ctxlen = MLX5_ST_SZ_BYTES(tirc);
2519        struct mlx5e_tirc_config ttconfig;
2520        int tt;
2521
2522        MLX5_SET(modify_tir_in, in, bitmask.hash, 1);
2523
2524        for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2525                memset(tirc, 0, ctxlen);
2526                mlx5e_update_rx_hash_fields(&ttconfig, tt,
2527                                            rss->rx_hash_fields[tt]);
2528                mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, false);
2529                mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in);
2530        }
2531
2532        /* Verify inner tirs resources allocated */
2533        if (!priv->inner_indir_tir[0].tirn)
2534                return;
2535
2536        for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2537                memset(tirc, 0, ctxlen);
2538                mlx5e_update_rx_hash_fields(&ttconfig, tt,
2539                                            rss->rx_hash_fields[tt]);
2540                mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, true);
2541                mlx5_core_modify_tir(mdev, priv->inner_indir_tir[tt].tirn, in);
2542        }
2543}
2544
2545static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
2546{
2547        struct mlx5_core_dev *mdev = priv->mdev;
2548
2549        void *in;
2550        void *tirc;
2551        int inlen;
2552        int err;
2553        int tt;
2554        int ix;
2555
2556        inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
2557        in = kvzalloc(inlen, GFP_KERNEL);
2558        if (!in)
2559                return -ENOMEM;
2560
2561        MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
2562        tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2563
2564        mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
2565
2566        for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2567                err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in);
2568                if (err)
2569                        goto free_in;
2570        }
2571
2572        for (ix = 0; ix < priv->max_nch; ix++) {
2573                err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn, in);
2574                if (err)
2575                        goto free_in;
2576        }
2577
2578free_in:
2579        kvfree(in);
2580
2581        return err;
2582}
2583
2584static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_lro);
2585
2586static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2587                         struct mlx5e_params *params, u16 mtu)
2588{
2589        u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2590        int err;
2591
2592        err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2593        if (err)
2594                return err;
2595
2596        /* Update vport context MTU */
2597        mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2598        return 0;
2599}
2600
2601static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2602                            struct mlx5e_params *params, u16 *mtu)
2603{
2604        u16 hw_mtu = 0;
2605        int err;
2606
2607        err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2608        if (err || !hw_mtu) /* fallback to port oper mtu */
2609                mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2610
2611        *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2612}
2613
2614int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2615{
2616        struct mlx5e_params *params = &priv->channels.params;
2617        struct net_device *netdev = priv->netdev;
2618        struct mlx5_core_dev *mdev = priv->mdev;
2619        u16 mtu;
2620        int err;
2621
2622        err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2623        if (err)
2624                return err;
2625
2626        mlx5e_query_mtu(mdev, params, &mtu);
2627        if (mtu != params->sw_mtu)
2628                netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2629                            __func__, mtu, params->sw_mtu);
2630
2631        params->sw_mtu = mtu;
2632        return 0;
2633}
2634
2635MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);
2636
2637void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
2638{
2639        struct mlx5e_params *params = &priv->channels.params;
2640        struct net_device *netdev   = priv->netdev;
2641        struct mlx5_core_dev *mdev  = priv->mdev;
2642        u16 max_mtu;
2643
2644        /* MTU range: 68 - hw-specific max */
2645        netdev->min_mtu = ETH_MIN_MTU;
2646
2647        mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2648        netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu),
2649                                ETH_MAX_MTU);
2650}
2651
2652static void mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc)
2653{
2654        int tc;
2655
2656        netdev_reset_tc(netdev);
2657
2658        if (ntc == 1)
2659                return;
2660
2661        netdev_set_num_tc(netdev, ntc);
2662
2663        /* Map netdev TCs to offset 0
2664         * We have our own UP to TXQ mapping for QoS
2665         */
2666        for (tc = 0; tc < ntc; tc++)
2667                netdev_set_tc_queue(netdev, tc, nch, 0);
2668}
2669
2670int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
2671{
2672        int qos_queues, nch, ntc, num_txqs, err;
2673
2674        qos_queues = mlx5e_qos_cur_leaf_nodes(priv);
2675
2676        nch = priv->channels.params.num_channels;
2677        ntc = priv->channels.params.num_tc;
2678        num_txqs = nch * ntc + qos_queues;
2679        if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS))
2680                num_txqs += ntc;
2681
2682        mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs);
2683        err = netif_set_real_num_tx_queues(priv->netdev, num_txqs);
2684        if (err)
2685                netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err);
2686
2687        return err;
2688}
2689
2690static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
2691{
2692        struct net_device *netdev = priv->netdev;
2693        int old_num_txqs, old_ntc;
2694        int num_rxqs, nch, ntc;
2695        int err;
2696
2697        old_num_txqs = netdev->real_num_tx_queues;
2698        old_ntc = netdev->num_tc ? : 1;
2699
2700        nch = priv->channels.params.num_channels;
2701        ntc = priv->channels.params.num_tc;
2702        num_rxqs = nch * priv->profile->rq_groups;
2703
2704        mlx5e_netdev_set_tcs(netdev, nch, ntc);
2705
2706        err = mlx5e_update_tx_netdev_queues(priv);
2707        if (err)
2708                goto err_tcs;
2709        err = netif_set_real_num_rx_queues(netdev, num_rxqs);
2710        if (err) {
2711                netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err);
2712                goto err_txqs;
2713        }
2714
2715        return 0;
2716
2717err_txqs:
2718        /* netif_set_real_num_rx_queues could fail only when nch increased. Only
2719         * one of nch and ntc is changed in this function. That means, the call
2720         * to netif_set_real_num_tx_queues below should not fail, because it
2721         * decreases the number of TX queues.
2722         */
2723        WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev, old_num_txqs));
2724
2725err_tcs:
2726        mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc);
2727        return err;
2728}
2729
2730static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
2731                                           struct mlx5e_params *params)
2732{
2733        struct mlx5_core_dev *mdev = priv->mdev;
2734        int num_comp_vectors, ix, irq;
2735
2736        num_comp_vectors = mlx5_comp_vectors_count(mdev);
2737
2738        for (ix = 0; ix < params->num_channels; ix++) {
2739                cpumask_clear(priv->scratchpad.cpumask);
2740
2741                for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) {
2742                        int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev, irq));
2743
2744                        cpumask_set_cpu(cpu, priv->scratchpad.cpumask);
2745                }
2746
2747                netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix);
2748        }
2749}
2750
2751int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
2752{
2753        u16 count = priv->channels.params.num_channels;
2754        int err;
2755
2756        err = mlx5e_update_netdev_queues(priv);
2757        if (err)
2758                return err;
2759
2760        mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params);
2761
2762        if (!netif_is_rxfh_configured(priv->netdev))
2763                mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt,
2764                                              MLX5E_INDIR_RQT_SIZE, count);
2765
2766        return 0;
2767}
2768
2769MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);
2770
2771static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
2772{
2773        int i, ch, tc, num_tc;
2774
2775        ch = priv->channels.num;
2776        num_tc = priv->channels.params.num_tc;
2777
2778        for (i = 0; i < ch; i++) {
2779                for (tc = 0; tc < num_tc; tc++) {
2780                        struct mlx5e_channel *c = priv->channels.c[i];
2781                        struct mlx5e_txqsq *sq = &c->sq[tc];
2782
2783                        priv->txq2sq[sq->txq_ix] = sq;
2784                        priv->channel_tc2realtxq[i][tc] = i + tc * ch;
2785                }
2786        }
2787
2788        if (!priv->channels.ptp)
2789                return;
2790
2791        if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state))
2792                return;
2793
2794        for (tc = 0; tc < num_tc; tc++) {
2795                struct mlx5e_ptp *c = priv->channels.ptp;
2796                struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
2797
2798                priv->txq2sq[sq->txq_ix] = sq;
2799                priv->port_ptp_tc2realtxq[tc] = priv->num_tc_x_num_ch + tc;
2800        }
2801}
2802
2803static void mlx5e_update_num_tc_x_num_ch(struct mlx5e_priv *priv)
2804{
2805        /* Sync with mlx5e_select_queue. */
2806        WRITE_ONCE(priv->num_tc_x_num_ch,
2807                   priv->channels.params.num_tc * priv->channels.num);
2808}
2809
2810void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
2811{
2812        mlx5e_update_num_tc_x_num_ch(priv);
2813        mlx5e_build_txq_maps(priv);
2814        mlx5e_activate_channels(&priv->channels);
2815        mlx5e_qos_activate_queues(priv);
2816        mlx5e_xdp_tx_enable(priv);
2817        netif_tx_start_all_queues(priv->netdev);
2818
2819        if (mlx5e_is_vport_rep(priv))
2820                mlx5e_add_sqs_fwd_rules(priv);
2821
2822        mlx5e_wait_channels_min_rx_wqes(&priv->channels);
2823        mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
2824
2825        mlx5e_xsk_redirect_rqts_to_channels(priv, &priv->channels);
2826}
2827
2828void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
2829{
2830        mlx5e_xsk_redirect_rqts_to_drop(priv, &priv->channels);
2831
2832        mlx5e_redirect_rqts_to_drop(priv);
2833
2834        if (mlx5e_is_vport_rep(priv))
2835                mlx5e_remove_sqs_fwd_rules(priv);
2836
2837        /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
2838         * polling for inactive tx queues.
2839         */
2840        netif_tx_stop_all_queues(priv->netdev);
2841        netif_tx_disable(priv->netdev);
2842        mlx5e_xdp_tx_disable(priv);
2843        mlx5e_deactivate_channels(&priv->channels);
2844}
2845
2846static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
2847                                    struct mlx5e_params *new_params,
2848                                    mlx5e_fp_preactivate preactivate,
2849                                    void *context)
2850{
2851        struct mlx5e_params old_params;
2852
2853        old_params = priv->channels.params;
2854        priv->channels.params = *new_params;
2855
2856        if (preactivate) {
2857                int err;
2858
2859                err = preactivate(priv, context);
2860                if (err) {
2861                        priv->channels.params = old_params;
2862                        return err;
2863                }
2864        }
2865
2866        return 0;
2867}
2868
2869static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
2870                                      struct mlx5e_channels *new_chs,
2871                                      mlx5e_fp_preactivate preactivate,
2872                                      void *context)
2873{
2874        struct net_device *netdev = priv->netdev;
2875        struct mlx5e_channels old_chs;
2876        int carrier_ok;
2877        int err = 0;
2878
2879        carrier_ok = netif_carrier_ok(netdev);
2880        netif_carrier_off(netdev);
2881
2882        mlx5e_deactivate_priv_channels(priv);
2883
2884        old_chs = priv->channels;
2885        priv->channels = *new_chs;
2886
2887        /* New channels are ready to roll, call the preactivate hook if needed
2888         * to modify HW settings or update kernel parameters.
2889         */
2890        if (preactivate) {
2891                err = preactivate(priv, context);
2892                if (err) {
2893                        priv->channels = old_chs;
2894                        goto out;
2895                }
2896        }
2897
2898        mlx5e_close_channels(&old_chs);
2899        priv->profile->update_rx(priv);
2900
2901out:
2902        mlx5e_activate_priv_channels(priv);
2903
2904        /* return carrier back if needed */
2905        if (carrier_ok)
2906                netif_carrier_on(netdev);
2907
2908        return err;
2909}
2910
2911int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
2912                             struct mlx5e_params *params,
2913                             mlx5e_fp_preactivate preactivate,
2914                             void *context, bool reset)
2915{
2916        struct mlx5e_channels new_chs = {};
2917        int err;
2918
2919        reset &= test_bit(MLX5E_STATE_OPENED, &priv->state);
2920        if (!reset)
2921                return mlx5e_switch_priv_params(priv, params, preactivate, context);
2922
2923        new_chs.params = *params;
2924        err = mlx5e_open_channels(priv, &new_chs);
2925        if (err)
2926                return err;
2927        err = mlx5e_switch_priv_channels(priv, &new_chs, preactivate, context);
2928        if (err)
2929                mlx5e_close_channels(&new_chs);
2930
2931        return err;
2932}
2933
2934int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
2935{
2936        return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true);
2937}
2938
2939void mlx5e_timestamp_init(struct mlx5e_priv *priv)
2940{
2941        priv->tstamp.tx_type   = HWTSTAMP_TX_OFF;
2942        priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
2943}
2944
2945static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
2946                                     enum mlx5_port_status state)
2947{
2948        struct mlx5_eswitch *esw = mdev->priv.eswitch;
2949        int vport_admin_state;
2950
2951        mlx5_set_port_admin_status(mdev, state);
2952
2953        if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS ||
2954            !MLX5_CAP_GEN(mdev, uplink_follow))
2955                return;
2956
2957        if (state == MLX5_PORT_UP)
2958                vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO;
2959        else
2960                vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN;
2961
2962        mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state);
2963}
2964
2965int mlx5e_open_locked(struct net_device *netdev)
2966{
2967        struct mlx5e_priv *priv = netdev_priv(netdev);
2968        int err;
2969
2970        set_bit(MLX5E_STATE_OPENED, &priv->state);
2971
2972        err = mlx5e_open_channels(priv, &priv->channels);
2973        if (err)
2974                goto err_clear_state_opened_flag;
2975
2976        priv->profile->update_rx(priv);
2977        mlx5e_activate_priv_channels(priv);
2978        mlx5e_apply_traps(priv, true);
2979        if (priv->profile->update_carrier)
2980                priv->profile->update_carrier(priv);
2981
2982        mlx5e_queue_update_stats(priv);
2983        return 0;
2984
2985err_clear_state_opened_flag:
2986        clear_bit(MLX5E_STATE_OPENED, &priv->state);
2987        return err;
2988}
2989
2990int mlx5e_open(struct net_device *netdev)
2991{
2992        struct mlx5e_priv *priv = netdev_priv(netdev);
2993        int err;
2994
2995        mutex_lock(&priv->state_lock);
2996        err = mlx5e_open_locked(netdev);
2997        if (!err)
2998                mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
2999        mutex_unlock(&priv->state_lock);
3000
3001        return err;
3002}
3003
3004int mlx5e_close_locked(struct net_device *netdev)
3005{
3006        struct mlx5e_priv *priv = netdev_priv(netdev);
3007
3008        /* May already be CLOSED in case a previous configuration operation
3009         * (e.g RX/TX queue size change) that involves close&open failed.
3010         */
3011        if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3012                return 0;
3013
3014        mlx5e_apply_traps(priv, false);
3015        clear_bit(MLX5E_STATE_OPENED, &priv->state);
3016
3017        netif_carrier_off(priv->netdev);
3018        mlx5e_deactivate_priv_channels(priv);
3019        mlx5e_close_channels(&priv->channels);
3020
3021        return 0;
3022}
3023
3024int mlx5e_close(struct net_device *netdev)
3025{
3026        struct mlx5e_priv *priv = netdev_priv(netdev);
3027        int err;
3028
3029        if (!netif_device_present(netdev))
3030                return -ENODEV;
3031
3032        mutex_lock(&priv->state_lock);
3033        mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
3034        err = mlx5e_close_locked(netdev);
3035        mutex_unlock(&priv->state_lock);
3036
3037        return err;
3038}
3039
3040static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
3041{
3042        mlx5_wq_destroy(&rq->wq_ctrl);
3043}
3044
3045static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3046                               struct mlx5e_rq *rq,
3047                               struct mlx5e_rq_param *param)
3048{
3049        void *rqc = param->rqc;
3050        void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3051        int err;
3052
3053        param->wq.db_numa_node = param->wq.buf_numa_node;
3054
3055        err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
3056                                 &rq->wq_ctrl);
3057        if (err)
3058                return err;
3059
3060        /* Mark as unused given "Drop-RQ" packets never reach XDP */
3061        xdp_rxq_info_unused(&rq->xdp_rxq);
3062
3063        rq->mdev = mdev;
3064
3065        return 0;
3066}
3067
3068static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
3069                               struct mlx5e_cq *cq,
3070                               struct mlx5e_cq_param *param)
3071{
3072        struct mlx5_core_dev *mdev = priv->mdev;
3073
3074        param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
3075        param->wq.db_numa_node  = dev_to_node(mlx5_core_dma_dev(mdev));
3076
3077        return mlx5e_alloc_cq_common(priv, param, cq);
3078}
3079
3080int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
3081                       struct mlx5e_rq *drop_rq)
3082{
3083        struct mlx5_core_dev *mdev = priv->mdev;
3084        struct mlx5e_cq_param cq_param = {};
3085        struct mlx5e_rq_param rq_param = {};
3086        struct mlx5e_cq *cq = &drop_rq->cq;
3087        int err;
3088
3089        mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param);
3090
3091        err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
3092        if (err)
3093                return err;
3094
3095        err = mlx5e_create_cq(cq, &cq_param);
3096        if (err)
3097                goto err_free_cq;
3098
3099        err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
3100        if (err)
3101                goto err_destroy_cq;
3102
3103        err = mlx5e_create_rq(drop_rq, &rq_param);
3104        if (err)
3105                goto err_free_rq;
3106
3107        err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3108        if (err)
3109                mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3110
3111        return 0;
3112
3113err_free_rq:
3114        mlx5e_free_drop_rq(drop_rq);
3115
3116err_destroy_cq:
3117        mlx5e_destroy_cq(cq);
3118
3119err_free_cq:
3120        mlx5e_free_cq(cq);
3121
3122        return err;
3123}
3124
3125void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
3126{
3127        mlx5e_destroy_rq(drop_rq);
3128        mlx5e_free_drop_rq(drop_rq);
3129        mlx5e_destroy_cq(&drop_rq->cq);
3130        mlx5e_free_cq(&drop_rq->cq);
3131}
3132
3133int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn)
3134{
3135        void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3136
3137        MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn);
3138
3139        if (MLX5_GET(tisc, tisc, tls_en))
3140                MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn);
3141
3142        if (mlx5_lag_is_lacp_owner(mdev))
3143                MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3144
3145        return mlx5_core_create_tis(mdev, in, tisn);
3146}
3147
3148void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
3149{
3150        mlx5_core_destroy_tis(mdev, tisn);
3151}
3152
3153void mlx5e_destroy_tises(struct mlx5e_priv *priv)
3154{
3155        int tc, i;
3156
3157        for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++)
3158                for (tc = 0; tc < priv->profile->max_tc; tc++)
3159                        mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3160}
3161
3162static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev)
3163{
3164        return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1;
3165}
3166
3167int mlx5e_create_tises(struct mlx5e_priv *priv)
3168{
3169        int tc, i;
3170        int err;
3171
3172        for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) {
3173                for (tc = 0; tc < priv->profile->max_tc; tc++) {
3174                        u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
3175                        void *tisc;
3176
3177                        tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3178
3179                        MLX5_SET(tisc, tisc, prio, tc << 1);
3180
3181                        if (mlx5e_lag_should_assign_affinity(priv->mdev))
3182                                MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1);
3183
3184                        err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]);
3185                        if (err)
3186                                goto err_close_tises;
3187                }
3188        }
3189
3190        return 0;
3191
3192err_close_tises:
3193        for (; i >= 0; i--) {
3194                for (tc--; tc >= 0; tc--)
3195                        mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]);
3196                tc = priv->profile->max_tc;
3197        }
3198
3199        return err;
3200}
3201
3202static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
3203{
3204        mlx5e_destroy_tises(priv);
3205}
3206
3207static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv *priv,
3208                                             u32 rqtn, u32 *tirc)
3209{
3210        MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.hw_objs.td.tdn);
3211        MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
3212        MLX5_SET(tirc, tirc, indirect_table, rqtn);
3213        MLX5_SET(tirc, tirc, tunneled_offload_en,
3214                 priv->channels.params.tunneled_offload_en);
3215
3216        mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
3217}
3218
3219static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
3220                                      enum mlx5e_traffic_types tt,
3221                                      u32 *tirc)
3222{
3223        mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc);
3224        mlx5e_build_indir_tir_ctx_hash(&priv->rss_params,
3225                                       &tirc_default_config[tt], tirc, false);
3226}
3227
3228static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
3229{
3230        mlx5e_build_indir_tir_ctx_common(priv, rqtn, tirc);
3231        MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
3232}
3233
3234static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv,
3235                                            enum mlx5e_traffic_types tt,
3236                                            u32 *tirc)
3237{
3238        mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc);
3239        mlx5e_build_indir_tir_ctx_hash(&priv->rss_params,
3240                                       &tirc_default_config[tt], tirc, true);
3241}
3242
3243int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc)
3244{
3245        struct mlx5e_tir *tir;
3246        void *tirc;
3247        int inlen;
3248        int i = 0;
3249        int err;
3250        u32 *in;
3251        int tt;
3252
3253        inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3254        in = kvzalloc(inlen, GFP_KERNEL);
3255        if (!in)
3256                return -ENOMEM;
3257
3258        for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
3259                memset(in, 0, inlen);
3260                tir = &priv->indir_tir[tt];
3261                tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3262                mlx5e_build_indir_tir_ctx(priv, tt, tirc);
3263                err = mlx5e_create_tir(priv->mdev, tir, in);
3264                if (err) {
3265                        mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
3266                        goto err_destroy_inner_tirs;
3267                }
3268        }
3269
3270        if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev))
3271                goto out;
3272
3273        for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) {
3274                memset(in, 0, inlen);
3275                tir = &priv->inner_indir_tir[i];
3276                tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3277                mlx5e_build_inner_indir_tir_ctx(priv, i, tirc);
3278                err = mlx5e_create_tir(priv->mdev, tir, in);
3279                if (err) {
3280                        mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err);
3281                        goto err_destroy_inner_tirs;
3282                }
3283        }
3284
3285out:
3286        kvfree(in);
3287
3288        return 0;
3289
3290err_destroy_inner_tirs:
3291        for (i--; i >= 0; i--)
3292                mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3293
3294        for (tt--; tt >= 0; tt--)
3295                mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
3296
3297        kvfree(in);
3298
3299        return err;
3300}
3301
3302int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n)
3303{
3304        struct mlx5e_tir *tir;
3305        void *tirc;
3306        int inlen;
3307        int err = 0;
3308        u32 *in;
3309        int ix;
3310
3311        inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3312        in = kvzalloc(inlen, GFP_KERNEL);
3313        if (!in)
3314                return -ENOMEM;
3315
3316        for (ix = 0; ix < n; ix++) {
3317                memset(in, 0, inlen);
3318                tir = &tirs[ix];
3319                tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3320                mlx5e_build_direct_tir_ctx(priv, tir->rqt.rqtn, tirc);
3321                err = mlx5e_create_tir(priv->mdev, tir, in);
3322                if (unlikely(err))
3323                        goto err_destroy_ch_tirs;
3324        }
3325
3326        goto out;
3327
3328err_destroy_ch_tirs:
3329        mlx5_core_warn(priv->mdev, "create tirs failed, %d\n", err);
3330        for (ix--; ix >= 0; ix--)
3331                mlx5e_destroy_tir(priv->mdev, &tirs[ix]);
3332
3333out:
3334        kvfree(in);
3335
3336        return err;
3337}
3338
3339void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
3340{
3341        int i;
3342
3343        for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3344                mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
3345
3346        /* Verify inner tirs resources allocated */
3347        if (!priv->inner_indir_tir[0].tirn)
3348                return;
3349
3350        for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3351                mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3352}
3353
3354void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n)
3355{
3356        int i;
3357
3358        for (i = 0; i < n; i++)
3359                mlx5e_destroy_tir(priv->mdev, &tirs[i]);
3360}
3361
3362static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
3363{
3364        int err = 0;
3365        int i;
3366
3367        for (i = 0; i < chs->num; i++) {
3368                err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
3369                if (err)
3370                        return err;
3371        }
3372
3373        return 0;
3374}
3375
3376static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
3377{
3378        int err;
3379        int i;
3380
3381        for (i = 0; i < chs->num; i++) {
3382                err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
3383                if (err)
3384                        return err;
3385        }
3386        if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state))
3387                return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd);
3388
3389        return 0;
3390}
3391
3392static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
3393                                 struct tc_mqprio_qopt *mqprio)
3394{
3395        struct mlx5e_params new_params;
3396        u8 tc = mqprio->num_tc;
3397        int err = 0;
3398
3399        mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3400
3401        if (tc && tc != MLX5E_MAX_NUM_TC)
3402                return -EINVAL;
3403
3404        mutex_lock(&priv->state_lock);
3405
3406        /* MQPRIO is another toplevel qdisc that can't be attached
3407         * simultaneously with the offloaded HTB.
3408         */
3409        if (WARN_ON(priv->htb.maj_id)) {
3410                err = -EINVAL;
3411                goto out;
3412        }
3413
3414        new_params = priv->channels.params;
3415        new_params.num_tc = tc ? tc : 1;
3416
3417        err = mlx5e_safe_switch_params(priv, &new_params,
3418                                       mlx5e_num_channels_changed_ctx, NULL, true);
3419
3420out:
3421        priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3422                                    priv->channels.params.num_tc);
3423        mutex_unlock(&priv->state_lock);
3424        return err;
3425}
3426
3427static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb)
3428{
3429        int res;
3430
3431        switch (htb->command) {
3432        case TC_HTB_CREATE:
3433                return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid,
3434                                          htb->extack);
3435        case TC_HTB_DESTROY:
3436                return mlx5e_htb_root_del(priv);
3437        case TC_HTB_LEAF_ALLOC_QUEUE:
3438                res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid,
3439                                                 htb->rate, htb->ceil, htb->extack);
3440                if (res < 0)
3441                        return res;
3442                htb->qid = res;
3443                return 0;
3444        case TC_HTB_LEAF_TO_INNER:
3445                return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid,
3446                                               htb->rate, htb->ceil, htb->extack);
3447        case TC_HTB_LEAF_DEL:
3448                return mlx5e_htb_leaf_del(priv, htb->classid, &htb->moved_qid, &htb->qid,
3449                                          htb->extack);
3450        case TC_HTB_LEAF_DEL_LAST:
3451        case TC_HTB_LEAF_DEL_LAST_FORCE:
3452                return mlx5e_htb_leaf_del_last(priv, htb->classid,
3453                                               htb->command == TC_HTB_LEAF_DEL_LAST_FORCE,
3454                                               htb->extack);
3455        case TC_HTB_NODE_MODIFY:
3456                return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil,
3457                                             htb->extack);
3458        case TC_HTB_LEAF_QUERY_QUEUE:
3459                res = mlx5e_get_txq_by_classid(priv, htb->classid);
3460                if (res < 0)
3461                        return res;
3462                htb->qid = res;
3463                return 0;
3464        default:
3465                return -EOPNOTSUPP;
3466        }
3467}
3468
3469static LIST_HEAD(mlx5e_block_cb_list);
3470
3471static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3472                          void *type_data)
3473{
3474        struct mlx5e_priv *priv = netdev_priv(dev);
3475        bool tc_unbind = false;
3476        int err;
3477
3478        if (type == TC_SETUP_BLOCK &&
3479            ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND)
3480                tc_unbind = true;
3481
3482        if (!netif_device_present(dev) && !tc_unbind)
3483                return -ENODEV;
3484
3485        switch (type) {
3486        case TC_SETUP_BLOCK: {
3487                struct flow_block_offload *f = type_data;
3488
3489                f->unlocked_driver_cb = true;
3490                return flow_block_cb_setup_simple(type_data,
3491                                                  &mlx5e_block_cb_list,
3492                                                  mlx5e_setup_tc_block_cb,
3493                                                  priv, priv, true);
3494        }
3495        case TC_SETUP_QDISC_MQPRIO:
3496                return mlx5e_setup_tc_mqprio(priv, type_data);
3497        case TC_SETUP_QDISC_HTB:
3498                mutex_lock(&priv->state_lock);
3499                err = mlx5e_setup_tc_htb(priv, type_data);
3500                mutex_unlock(&priv->state_lock);
3501                return err;
3502        default:
3503                return -EOPNOTSUPP;
3504        }
3505}
3506
3507void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
3508{
3509        int i;
3510
3511        for (i = 0; i < priv->max_nch; i++) {
3512                struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i];
3513                struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
3514                struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3515                int j;
3516
3517                s->rx_packets   += rq_stats->packets + xskrq_stats->packets;
3518                s->rx_bytes     += rq_stats->bytes + xskrq_stats->bytes;
3519                s->multicast    += rq_stats->mcast_packets + xskrq_stats->mcast_packets;
3520
3521                for (j = 0; j < priv->max_opened_tc; j++) {
3522                        struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
3523
3524                        s->tx_packets    += sq_stats->packets;
3525                        s->tx_bytes      += sq_stats->bytes;
3526                        s->tx_dropped    += sq_stats->dropped;
3527                }
3528        }
3529        if (priv->tx_ptp_opened) {
3530                for (i = 0; i < priv->max_opened_tc; i++) {
3531                        struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i];
3532
3533                        s->tx_packets    += sq_stats->packets;
3534                        s->tx_bytes      += sq_stats->bytes;
3535                        s->tx_dropped    += sq_stats->dropped;
3536                }
3537        }
3538        if (priv->rx_ptp_opened) {
3539                struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq;
3540
3541                s->rx_packets   += rq_stats->packets;
3542                s->rx_bytes     += rq_stats->bytes;
3543                s->multicast    += rq_stats->mcast_packets;
3544        }
3545}
3546
3547void
3548mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3549{
3550        struct mlx5e_priv *priv = netdev_priv(dev);
3551        struct mlx5e_pport_stats *pstats = &priv->stats.pport;
3552
3553        if (!netif_device_present(dev))
3554                return;
3555
3556        /* In switchdev mode, monitor counters doesn't monitor
3557         * rx/tx stats of 802_3. The update stats mechanism
3558         * should keep the 802_3 layout counters updated
3559         */
3560        if (!mlx5e_monitor_counter_supported(priv) ||
3561            mlx5e_is_uplink_rep(priv)) {
3562                /* update HW stats in background for next time */
3563                mlx5e_queue_update_stats(priv);
3564        }
3565
3566        if (mlx5e_is_uplink_rep(priv)) {
3567                struct mlx5e_vport_stats *vstats = &priv->stats.vport;
3568
3569                stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3570                stats->rx_bytes   = PPORT_802_3_GET(pstats, a_octets_received_ok);
3571                stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3572                stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3573
3574                /* vport multicast also counts packets that are dropped due to steering
3575                 * or rx out of buffer
3576                 */
3577                stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
3578        } else {
3579                mlx5e_fold_sw_stats64(priv, stats);
3580        }
3581
3582        stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
3583
3584        stats->rx_length_errors =
3585                PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3586                PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3587                PPORT_802_3_GET(pstats, a_frame_too_long_errors);
3588        stats->rx_crc_errors =
3589                PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3590        stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3591        stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
3592        stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3593                           stats->rx_frame_errors;
3594        stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3595}
3596
3597static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv)
3598{
3599        if (mlx5e_is_uplink_rep(priv))
3600                return; /* no rx mode for uplink rep */
3601
3602        queue_work(priv->wq, &priv->set_rx_mode_work);
3603}
3604
3605static void mlx5e_set_rx_mode(struct net_device *dev)
3606{
3607        struct mlx5e_priv *priv = netdev_priv(dev);
3608
3609        mlx5e_nic_set_rx_mode(priv);
3610}
3611
3612static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3613{
3614        struct mlx5e_priv *priv = netdev_priv(netdev);
3615        struct sockaddr *saddr = addr;
3616
3617        if (!is_valid_ether_addr(saddr->sa_data))
3618                return -EADDRNOTAVAIL;
3619
3620        netif_addr_lock_bh(netdev);
3621        ether_addr_copy(netdev->dev_addr, saddr->sa_data);
3622        netif_addr_unlock_bh(netdev);
3623
3624        mlx5e_nic_set_rx_mode(priv);
3625
3626        return 0;
3627}
3628
3629#define MLX5E_SET_FEATURE(features, feature, enable)    \
3630        do {                                            \
3631                if (enable)                             \
3632                        *features |= feature;           \
3633                else                                    \
3634                        *features &= ~feature;          \
3635        } while (0)
3636
3637typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3638
3639static int set_feature_lro(struct net_device *netdev, bool enable)
3640{
3641        struct mlx5e_priv *priv = netdev_priv(netdev);
3642        struct mlx5_core_dev *mdev = priv->mdev;
3643        struct mlx5e_params *cur_params;
3644        struct mlx5e_params new_params;
3645        bool reset = true;
3646        int err = 0;
3647
3648        mutex_lock(&priv->state_lock);
3649
3650        if (enable && priv->xsk.refcnt) {
3651                netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
3652                            priv->xsk.refcnt);
3653                err = -EINVAL;
3654                goto out;
3655        }
3656
3657        cur_params = &priv->channels.params;
3658        if (enable && !MLX5E_GET_PFLAG(cur_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3659                netdev_warn(netdev, "can't set LRO with legacy RQ\n");
3660                err = -EINVAL;
3661                goto out;
3662        }
3663
3664        new_params = *cur_params;
3665        new_params.lro_en = enable;
3666
3667        if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3668                if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
3669                    mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
3670                        reset = false;
3671        }
3672
3673        err = mlx5e_safe_switch_params(priv, &new_params,
3674                                       mlx5e_modify_tirs_lro_ctx, NULL, reset);
3675out:
3676        mutex_unlock(&priv->state_lock);
3677        return err;
3678}
3679
3680static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
3681{
3682        struct mlx5e_priv *priv = netdev_priv(netdev);
3683
3684        if (enable)
3685                mlx5e_enable_cvlan_filter(priv);
3686        else
3687                mlx5e_disable_cvlan_filter(priv);
3688
3689        return 0;
3690}
3691
3692static int set_feature_hw_tc(struct net_device *netdev, bool enable)
3693{
3694        struct mlx5e_priv *priv = netdev_priv(netdev);
3695
3696#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3697        if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) {
3698                netdev_err(netdev,
3699                           "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3700                return -EINVAL;
3701        }
3702#endif
3703
3704        if (!enable && priv->htb.maj_id) {
3705                netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n");
3706                return -EINVAL;
3707        }
3708
3709        return 0;
3710}
3711
3712static int set_feature_rx_all(struct net_device *netdev, bool enable)
3713{
3714        struct mlx5e_priv *priv = netdev_priv(netdev);
3715        struct mlx5_core_dev *mdev = priv->mdev;
3716
3717        return mlx5_set_port_fcs(mdev, !enable);
3718}
3719
3720static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3721{
3722        struct mlx5e_priv *priv = netdev_priv(netdev);
3723        int err;
3724
3725        mutex_lock(&priv->state_lock);
3726
3727        priv->channels.params.scatter_fcs_en = enable;
3728        err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable);
3729        if (err)
3730                priv->channels.params.scatter_fcs_en = !enable;
3731
3732        mutex_unlock(&priv->state_lock);
3733
3734        return err;
3735}
3736
3737static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3738{
3739        struct mlx5e_priv *priv = netdev_priv(netdev);
3740        int err = 0;
3741
3742        mutex_lock(&priv->state_lock);
3743
3744        priv->channels.params.vlan_strip_disable = !enable;
3745        if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3746                goto unlock;
3747
3748        err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
3749        if (err)
3750                priv->channels.params.vlan_strip_disable = enable;
3751
3752unlock:
3753        mutex_unlock(&