linux/drivers/net/ethernet/intel/ice/ice_xsk.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2019, Intel Corporation. */
   3
   4#include <linux/bpf_trace.h>
   5#include <net/xdp_sock_drv.h>
   6#include <net/xdp.h>
   7#include "ice.h"
   8#include "ice_base.h"
   9#include "ice_type.h"
  10#include "ice_xsk.h"
  11#include "ice_txrx.h"
  12#include "ice_txrx_lib.h"
  13#include "ice_lib.h"
  14
  15/**
  16 * ice_qp_reset_stats - Resets all stats for rings of given index
  17 * @vsi: VSI that contains rings of interest
  18 * @q_idx: ring index in array
  19 */
  20static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)
  21{
  22        memset(&vsi->rx_rings[q_idx]->rx_stats, 0,
  23               sizeof(vsi->rx_rings[q_idx]->rx_stats));
  24        memset(&vsi->tx_rings[q_idx]->stats, 0,
  25               sizeof(vsi->tx_rings[q_idx]->stats));
  26        if (ice_is_xdp_ena_vsi(vsi))
  27                memset(&vsi->xdp_rings[q_idx]->stats, 0,
  28                       sizeof(vsi->xdp_rings[q_idx]->stats));
  29}
  30
  31/**
  32 * ice_qp_clean_rings - Cleans all the rings of a given index
  33 * @vsi: VSI that contains rings of interest
  34 * @q_idx: ring index in array
  35 */
  36static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)
  37{
  38        ice_clean_tx_ring(vsi->tx_rings[q_idx]);
  39        if (ice_is_xdp_ena_vsi(vsi))
  40                ice_clean_tx_ring(vsi->xdp_rings[q_idx]);
  41        ice_clean_rx_ring(vsi->rx_rings[q_idx]);
  42}
  43
  44/**
  45 * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector
  46 * @vsi: VSI that has netdev
  47 * @q_vector: q_vector that has NAPI context
  48 * @enable: true for enable, false for disable
  49 */
  50static void
  51ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector,
  52                     bool enable)
  53{
  54        if (!vsi->netdev || !q_vector)
  55                return;
  56
  57        if (enable)
  58                napi_enable(&q_vector->napi);
  59        else
  60                napi_disable(&q_vector->napi);
  61}
  62
  63/**
  64 * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring
  65 * @vsi: the VSI that contains queue vector being un-configured
  66 * @rx_ring: Rx ring that will have its IRQ disabled
  67 * @q_vector: queue vector
  68 */
  69static void
  70ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_ring *rx_ring,
  71                 struct ice_q_vector *q_vector)
  72{
  73        struct ice_pf *pf = vsi->back;
  74        struct ice_hw *hw = &pf->hw;
  75        int base = vsi->base_vector;
  76        u16 reg;
  77        u32 val;
  78
  79        /* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle
  80         * here only QINT_RQCTL
  81         */
  82        reg = rx_ring->reg_idx;
  83        val = rd32(hw, QINT_RQCTL(reg));
  84        val &= ~QINT_RQCTL_CAUSE_ENA_M;
  85        wr32(hw, QINT_RQCTL(reg), val);
  86
  87        if (q_vector) {
  88                u16 v_idx = q_vector->v_idx;
  89
  90                wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);
  91                ice_flush(hw);
  92                synchronize_irq(pf->msix_entries[v_idx + base].vector);
  93        }
  94}
  95
  96/**
  97 * ice_qvec_cfg_msix - Enable IRQ for given queue vector
  98 * @vsi: the VSI that contains queue vector
  99 * @q_vector: queue vector
 100 */
 101static void
 102ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
 103{
 104        u16 reg_idx = q_vector->reg_idx;
 105        struct ice_pf *pf = vsi->back;
 106        struct ice_hw *hw = &pf->hw;
 107        struct ice_ring *ring;
 108
 109        ice_cfg_itr(hw, q_vector);
 110
 111        ice_for_each_ring(ring, q_vector->tx)
 112                ice_cfg_txq_interrupt(vsi, ring->reg_idx, reg_idx,
 113                                      q_vector->tx.itr_idx);
 114
 115        ice_for_each_ring(ring, q_vector->rx)
 116                ice_cfg_rxq_interrupt(vsi, ring->reg_idx, reg_idx,
 117                                      q_vector->rx.itr_idx);
 118
 119        ice_flush(hw);
 120}
 121
 122/**
 123 * ice_qvec_ena_irq - Enable IRQ for given queue vector
 124 * @vsi: the VSI that contains queue vector
 125 * @q_vector: queue vector
 126 */
 127static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
 128{
 129        struct ice_pf *pf = vsi->back;
 130        struct ice_hw *hw = &pf->hw;
 131
 132        ice_irq_dynamic_ena(hw, vsi, q_vector);
 133
 134        ice_flush(hw);
 135}
 136
 137/**
 138 * ice_qp_dis - Disables a queue pair
 139 * @vsi: VSI of interest
 140 * @q_idx: ring index in array
 141 *
 142 * Returns 0 on success, negative on failure.
 143 */
 144static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx)
 145{
 146        struct ice_txq_meta txq_meta = { };
 147        struct ice_ring *tx_ring, *rx_ring;
 148        struct ice_q_vector *q_vector;
 149        int timeout = 50;
 150        int err;
 151
 152        if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
 153                return -EINVAL;
 154
 155        tx_ring = vsi->tx_rings[q_idx];
 156        rx_ring = vsi->rx_rings[q_idx];
 157        q_vector = rx_ring->q_vector;
 158
 159        while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) {
 160                timeout--;
 161                if (!timeout)
 162                        return -EBUSY;
 163                usleep_range(1000, 2000);
 164        }
 165        netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
 166
 167        ice_qvec_dis_irq(vsi, rx_ring, q_vector);
 168
 169        ice_fill_txq_meta(vsi, tx_ring, &txq_meta);
 170        err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta);
 171        if (err)
 172                return err;
 173        if (ice_is_xdp_ena_vsi(vsi)) {
 174                struct ice_ring *xdp_ring = vsi->xdp_rings[q_idx];
 175
 176                memset(&txq_meta, 0, sizeof(txq_meta));
 177                ice_fill_txq_meta(vsi, xdp_ring, &txq_meta);
 178                err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring,
 179                                           &txq_meta);
 180                if (err)
 181                        return err;
 182        }
 183        err = ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, true);
 184        if (err)
 185                return err;
 186
 187        ice_qvec_toggle_napi(vsi, q_vector, false);
 188        ice_qp_clean_rings(vsi, q_idx);
 189        ice_qp_reset_stats(vsi, q_idx);
 190
 191        return 0;
 192}
 193
 194/**
 195 * ice_qp_ena - Enables a queue pair
 196 * @vsi: VSI of interest
 197 * @q_idx: ring index in array
 198 *
 199 * Returns 0 on success, negative on failure.
 200 */
 201static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx)
 202{
 203        struct ice_aqc_add_tx_qgrp *qg_buf;
 204        struct ice_ring *tx_ring, *rx_ring;
 205        struct ice_q_vector *q_vector;
 206        u16 size;
 207        int err;
 208
 209        if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
 210                return -EINVAL;
 211
 212        size = struct_size(qg_buf, txqs, 1);
 213        qg_buf = kzalloc(size, GFP_KERNEL);
 214        if (!qg_buf)
 215                return -ENOMEM;
 216
 217        qg_buf->num_txqs = 1;
 218
 219        tx_ring = vsi->tx_rings[q_idx];
 220        rx_ring = vsi->rx_rings[q_idx];
 221        q_vector = rx_ring->q_vector;
 222
 223        err = ice_vsi_cfg_txq(vsi, tx_ring, qg_buf);
 224        if (err)
 225                goto free_buf;
 226
 227        if (ice_is_xdp_ena_vsi(vsi)) {
 228                struct ice_ring *xdp_ring = vsi->xdp_rings[q_idx];
 229
 230                memset(qg_buf, 0, size);
 231                qg_buf->num_txqs = 1;
 232                err = ice_vsi_cfg_txq(vsi, xdp_ring, qg_buf);
 233                if (err)
 234                        goto free_buf;
 235                ice_set_ring_xdp(xdp_ring);
 236                xdp_ring->xsk_pool = ice_xsk_pool(xdp_ring);
 237        }
 238
 239        err = ice_setup_rx_ctx(rx_ring);
 240        if (err)
 241                goto free_buf;
 242
 243        ice_qvec_cfg_msix(vsi, q_vector);
 244
 245        err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true);
 246        if (err)
 247                goto free_buf;
 248
 249        clear_bit(ICE_CFG_BUSY, vsi->state);
 250        ice_qvec_toggle_napi(vsi, q_vector, true);
 251        ice_qvec_ena_irq(vsi, q_vector);
 252
 253        netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
 254free_buf:
 255        kfree(qg_buf);
 256        return err;
 257}
 258
 259/**
 260 * ice_xsk_pool_disable - disable a buffer pool region
 261 * @vsi: Current VSI
 262 * @qid: queue ID
 263 *
 264 * Returns 0 on success, negative on failure
 265 */
 266static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid)
 267{
 268        struct xsk_buff_pool *pool = xsk_get_pool_from_qid(vsi->netdev, qid);
 269
 270        if (!pool)
 271                return -EINVAL;
 272
 273        clear_bit(qid, vsi->af_xdp_zc_qps);
 274        xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR);
 275
 276        return 0;
 277}
 278
 279/**
 280 * ice_xsk_pool_enable - enable a buffer pool region
 281 * @vsi: Current VSI
 282 * @pool: pointer to a requested buffer pool region
 283 * @qid: queue ID
 284 *
 285 * Returns 0 on success, negative on failure
 286 */
 287static int
 288ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
 289{
 290        int err;
 291
 292        if (vsi->type != ICE_VSI_PF)
 293                return -EINVAL;
 294
 295        if (qid >= vsi->netdev->real_num_rx_queues ||
 296            qid >= vsi->netdev->real_num_tx_queues)
 297                return -EINVAL;
 298
 299        err = xsk_pool_dma_map(pool, ice_pf_to_dev(vsi->back),
 300                               ICE_RX_DMA_ATTR);
 301        if (err)
 302                return err;
 303
 304        set_bit(qid, vsi->af_xdp_zc_qps);
 305
 306        return 0;
 307}
 308
 309/**
 310 * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state
 311 * @vsi: Current VSI
 312 * @pool: buffer pool to enable/associate to a ring, NULL to disable
 313 * @qid: queue ID
 314 *
 315 * Returns 0 on success, negative on failure
 316 */
 317int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
 318{
 319        bool if_running, pool_present = !!pool;
 320        int ret = 0, pool_failure = 0;
 321
 322        if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi);
 323
 324        if (if_running) {
 325                ret = ice_qp_dis(vsi, qid);
 326                if (ret) {
 327                        netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret);
 328                        goto xsk_pool_if_up;
 329                }
 330        }
 331
 332        pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) :
 333                                      ice_xsk_pool_disable(vsi, qid);
 334
 335xsk_pool_if_up:
 336        if (if_running) {
 337                ret = ice_qp_ena(vsi, qid);
 338                if (!ret && pool_present)
 339                        napi_schedule(&vsi->xdp_rings[qid]->q_vector->napi);
 340                else if (ret)
 341                        netdev_err(vsi->netdev, "ice_qp_ena error = %d\n", ret);
 342        }
 343
 344        if (pool_failure) {
 345                netdev_err(vsi->netdev, "Could not %sable buffer pool, error = %d\n",
 346                           pool_present ? "en" : "dis", pool_failure);
 347                return pool_failure;
 348        }
 349
 350        return ret;
 351}
 352
 353/**
 354 * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
 355 * @rx_ring: Rx ring
 356 * @count: The number of buffers to allocate
 357 *
 358 * This function allocates a number of Rx buffers from the fill ring
 359 * or the internal recycle mechanism and places them on the Rx ring.
 360 *
 361 * Returns true if all allocations were successful, false if any fail.
 362 */
 363bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count)
 364{
 365        union ice_32b_rx_flex_desc *rx_desc;
 366        u16 ntu = rx_ring->next_to_use;
 367        struct ice_rx_buf *rx_buf;
 368        bool ok = true;
 369        dma_addr_t dma;
 370
 371        if (!count)
 372                return true;
 373
 374        rx_desc = ICE_RX_DESC(rx_ring, ntu);
 375        rx_buf = &rx_ring->rx_buf[ntu];
 376
 377        do {
 378                rx_buf->xdp = xsk_buff_alloc(rx_ring->xsk_pool);
 379                if (!rx_buf->xdp) {
 380                        ok = false;
 381                        break;
 382                }
 383
 384                dma = xsk_buff_xdp_get_dma(rx_buf->xdp);
 385                rx_desc->read.pkt_addr = cpu_to_le64(dma);
 386                rx_desc->wb.status_error0 = 0;
 387
 388                rx_desc++;
 389                rx_buf++;
 390                ntu++;
 391
 392                if (unlikely(ntu == rx_ring->count)) {
 393                        rx_desc = ICE_RX_DESC(rx_ring, 0);
 394                        rx_buf = rx_ring->rx_buf;
 395                        ntu = 0;
 396                }
 397        } while (--count);
 398
 399        if (rx_ring->next_to_use != ntu) {
 400                /* clear the status bits for the next_to_use descriptor */
 401                rx_desc->wb.status_error0 = 0;
 402                ice_release_rx_desc(rx_ring, ntu);
 403        }
 404
 405        return ok;
 406}
 407
 408/**
 409 * ice_bump_ntc - Bump the next_to_clean counter of an Rx ring
 410 * @rx_ring: Rx ring
 411 */
 412static void ice_bump_ntc(struct ice_ring *rx_ring)
 413{
 414        int ntc = rx_ring->next_to_clean + 1;
 415
 416        ntc = (ntc < rx_ring->count) ? ntc : 0;
 417        rx_ring->next_to_clean = ntc;
 418        prefetch(ICE_RX_DESC(rx_ring, ntc));
 419}
 420
 421/**
 422 * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
 423 * @rx_ring: Rx ring
 424 * @rx_buf: zero-copy Rx buffer
 425 *
 426 * This function allocates a new skb from a zero-copy Rx buffer.
 427 *
 428 * Returns the skb on success, NULL on failure.
 429 */
 430static struct sk_buff *
 431ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
 432{
 433        unsigned int metasize = rx_buf->xdp->data - rx_buf->xdp->data_meta;
 434        unsigned int datasize = rx_buf->xdp->data_end - rx_buf->xdp->data;
 435        unsigned int datasize_hard = rx_buf->xdp->data_end -
 436                                     rx_buf->xdp->data_hard_start;
 437        struct sk_buff *skb;
 438
 439        skb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard,
 440                               GFP_ATOMIC | __GFP_NOWARN);
 441        if (unlikely(!skb))
 442                return NULL;
 443
 444        skb_reserve(skb, rx_buf->xdp->data - rx_buf->xdp->data_hard_start);
 445        memcpy(__skb_put(skb, datasize), rx_buf->xdp->data, datasize);
 446        if (metasize)
 447                skb_metadata_set(skb, metasize);
 448
 449        xsk_buff_free(rx_buf->xdp);
 450        rx_buf->xdp = NULL;
 451        return skb;
 452}
 453
 454/**
 455 * ice_run_xdp_zc - Executes an XDP program in zero-copy path
 456 * @rx_ring: Rx ring
 457 * @xdp: xdp_buff used as input to the XDP program
 458 *
 459 * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}
 460 */
 461static int
 462ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)
 463{
 464        int err, result = ICE_XDP_PASS;
 465        struct bpf_prog *xdp_prog;
 466        struct ice_ring *xdp_ring;
 467        u32 act;
 468
 469        rcu_read_lock();
 470        /* ZC patch is enabled only when XDP program is set,
 471         * so here it can not be NULL
 472         */
 473        xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 474
 475        act = bpf_prog_run_xdp(xdp_prog, xdp);
 476
 477        if (likely(act == XDP_REDIRECT)) {
 478                err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
 479                if (err)
 480                        goto out_failure;
 481                rcu_read_unlock();
 482                return ICE_XDP_REDIR;
 483        }
 484
 485        switch (act) {
 486        case XDP_PASS:
 487                break;
 488        case XDP_TX:
 489                xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->q_index];
 490                result = ice_xmit_xdp_buff(xdp, xdp_ring);
 491                if (result == ICE_XDP_CONSUMED)
 492                        goto out_failure;
 493                break;
 494        default:
 495                bpf_warn_invalid_xdp_action(act);
 496                fallthrough;
 497        case XDP_ABORTED:
 498out_failure:
 499                trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
 500                fallthrough;
 501        case XDP_DROP:
 502                result = ICE_XDP_CONSUMED;
 503                break;
 504        }
 505
 506        rcu_read_unlock();
 507        return result;
 508}
 509
 510/**
 511 * ice_clean_rx_irq_zc - consumes packets from the hardware ring
 512 * @rx_ring: AF_XDP Rx ring
 513 * @budget: NAPI budget
 514 *
 515 * Returns number of processed packets on success, remaining budget on failure.
 516 */
 517int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
 518{
 519        unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 520        u16 cleaned_count = ICE_DESC_UNUSED(rx_ring);
 521        unsigned int xdp_xmit = 0;
 522        bool failure = false;
 523
 524        while (likely(total_rx_packets < (unsigned int)budget)) {
 525                union ice_32b_rx_flex_desc *rx_desc;
 526                unsigned int size, xdp_res = 0;
 527                struct ice_rx_buf *rx_buf;
 528                struct sk_buff *skb;
 529                u16 stat_err_bits;
 530                u16 vlan_tag = 0;
 531                u8 rx_ptype;
 532
 533                rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean);
 534
 535                stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);
 536                if (!ice_test_staterr(rx_desc, stat_err_bits))
 537                        break;
 538
 539                /* This memory barrier is needed to keep us from reading
 540                 * any other fields out of the rx_desc until we have
 541                 * verified the descriptor has been written back.
 542                 */
 543                dma_rmb();
 544
 545                size = le16_to_cpu(rx_desc->wb.pkt_len) &
 546                                   ICE_RX_FLX_DESC_PKT_LEN_M;
 547                if (!size)
 548                        break;
 549
 550                rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
 551                rx_buf->xdp->data_end = rx_buf->xdp->data + size;
 552                xsk_buff_dma_sync_for_cpu(rx_buf->xdp, rx_ring->xsk_pool);
 553
 554                xdp_res = ice_run_xdp_zc(rx_ring, rx_buf->xdp);
 555                if (xdp_res) {
 556                        if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))
 557                                xdp_xmit |= xdp_res;
 558                        else
 559                                xsk_buff_free(rx_buf->xdp);
 560
 561                        rx_buf->xdp = NULL;
 562                        total_rx_bytes += size;
 563                        total_rx_packets++;
 564                        cleaned_count++;
 565
 566                        ice_bump_ntc(rx_ring);
 567                        continue;
 568                }
 569
 570                /* XDP_PASS path */
 571                skb = ice_construct_skb_zc(rx_ring, rx_buf);
 572                if (!skb) {
 573                        rx_ring->rx_stats.alloc_buf_failed++;
 574                        break;
 575                }
 576
 577                cleaned_count++;
 578                ice_bump_ntc(rx_ring);
 579
 580                if (eth_skb_pad(skb)) {
 581                        skb = NULL;
 582                        continue;
 583                }
 584
 585                total_rx_bytes += skb->len;
 586                total_rx_packets++;
 587
 588                stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S);
 589                if (ice_test_staterr(rx_desc, stat_err_bits))
 590                        vlan_tag = le16_to_cpu(rx_desc->wb.l2tag1);
 591
 592                rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) &
 593                                       ICE_RX_FLEX_DESC_PTYPE_M;
 594
 595                ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
 596                ice_receive_skb(rx_ring, skb, vlan_tag);
 597        }
 598
 599        if (cleaned_count >= ICE_RX_BUF_WRITE)
 600                failure = !ice_alloc_rx_bufs_zc(rx_ring, cleaned_count);
 601
 602        ice_finalize_xdp_rx(rx_ring, xdp_xmit);
 603        ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes);
 604
 605        if (xsk_uses_need_wakeup(rx_ring->xsk_pool)) {
 606                if (failure || rx_ring->next_to_clean == rx_ring->next_to_use)
 607                        xsk_set_rx_need_wakeup(rx_ring->xsk_pool);
 608                else
 609                        xsk_clear_rx_need_wakeup(rx_ring->xsk_pool);
 610
 611                return (int)total_rx_packets;
 612        }
 613
 614        return failure ? budget : (int)total_rx_packets;
 615}
 616
 617/**
 618 * ice_xmit_zc - Completes AF_XDP entries, and cleans XDP entries
 619 * @xdp_ring: XDP Tx ring
 620 * @budget: max number of frames to xmit
 621 *
 622 * Returns true if cleanup/transmission is done.
 623 */
 624static bool ice_xmit_zc(struct ice_ring *xdp_ring, int budget)
 625{
 626        struct ice_tx_desc *tx_desc = NULL;
 627        bool work_done = true;
 628        struct xdp_desc desc;
 629        dma_addr_t dma;
 630
 631        while (likely(budget-- > 0)) {
 632                struct ice_tx_buf *tx_buf;
 633
 634                if (unlikely(!ICE_DESC_UNUSED(xdp_ring))) {
 635                        xdp_ring->tx_stats.tx_busy++;
 636                        work_done = false;
 637                        break;
 638                }
 639
 640                tx_buf = &xdp_ring->tx_buf[xdp_ring->next_to_use];
 641
 642                if (!xsk_tx_peek_desc(xdp_ring->xsk_pool, &desc))
 643                        break;
 644
 645                dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc.addr);
 646                xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma,
 647                                                 desc.len);
 648
 649                tx_buf->bytecount = desc.len;
 650
 651                tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use);
 652                tx_desc->buf_addr = cpu_to_le64(dma);
 653                tx_desc->cmd_type_offset_bsz =
 654                        ice_build_ctob(ICE_TXD_LAST_DESC_CMD, 0, desc.len, 0);
 655
 656                xdp_ring->next_to_use++;
 657                if (xdp_ring->next_to_use == xdp_ring->count)
 658                        xdp_ring->next_to_use = 0;
 659        }
 660
 661        if (tx_desc) {
 662                ice_xdp_ring_update_tail(xdp_ring);
 663                xsk_tx_release(xdp_ring->xsk_pool);
 664        }
 665
 666        return budget > 0 && work_done;
 667}
 668
 669/**
 670 * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer
 671 * @xdp_ring: XDP Tx ring
 672 * @tx_buf: Tx buffer to clean
 673 */
 674static void
 675ice_clean_xdp_tx_buf(struct ice_ring *xdp_ring, struct ice_tx_buf *tx_buf)
 676{
 677        xdp_return_frame((struct xdp_frame *)tx_buf->raw_buf);
 678        dma_unmap_single(xdp_ring->dev, dma_unmap_addr(tx_buf, dma),
 679                         dma_unmap_len(tx_buf, len), DMA_TO_DEVICE);
 680        dma_unmap_len_set(tx_buf, len, 0);
 681}
 682
 683/**
 684 * ice_clean_tx_irq_zc - Completes AF_XDP entries, and cleans XDP entries
 685 * @xdp_ring: XDP Tx ring
 686 * @budget: NAPI budget
 687 *
 688 * Returns true if cleanup/tranmission is done.
 689 */
 690bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget)
 691{
 692        int total_packets = 0, total_bytes = 0;
 693        s16 ntc = xdp_ring->next_to_clean;
 694        struct ice_tx_desc *tx_desc;
 695        struct ice_tx_buf *tx_buf;
 696        u32 xsk_frames = 0;
 697        bool xmit_done;
 698
 699        tx_desc = ICE_TX_DESC(xdp_ring, ntc);
 700        tx_buf = &xdp_ring->tx_buf[ntc];
 701        ntc -= xdp_ring->count;
 702
 703        do {
 704                if (!(tx_desc->cmd_type_offset_bsz &
 705                      cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))
 706                        break;
 707
 708                total_bytes += tx_buf->bytecount;
 709                total_packets++;
 710
 711                if (tx_buf->raw_buf) {
 712                        ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
 713                        tx_buf->raw_buf = NULL;
 714                } else {
 715                        xsk_frames++;
 716                }
 717
 718                tx_desc->cmd_type_offset_bsz = 0;
 719                tx_buf++;
 720                tx_desc++;
 721                ntc++;
 722
 723                if (unlikely(!ntc)) {
 724                        ntc -= xdp_ring->count;
 725                        tx_buf = xdp_ring->tx_buf;
 726                        tx_desc = ICE_TX_DESC(xdp_ring, 0);
 727                }
 728
 729                prefetch(tx_desc);
 730
 731        } while (likely(--budget));
 732
 733        ntc += xdp_ring->count;
 734        xdp_ring->next_to_clean = ntc;
 735
 736        if (xsk_frames)
 737                xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
 738
 739        if (xsk_uses_need_wakeup(xdp_ring->xsk_pool))
 740                xsk_set_tx_need_wakeup(xdp_ring->xsk_pool);
 741
 742        ice_update_tx_ring_stats(xdp_ring, total_packets, total_bytes);
 743        xmit_done = ice_xmit_zc(xdp_ring, ICE_DFLT_IRQ_WORK);
 744
 745        return budget > 0 && xmit_done;
 746}
 747
 748/**
 749 * ice_xsk_wakeup - Implements ndo_xsk_wakeup
 750 * @netdev: net_device
 751 * @queue_id: queue to wake up
 752 * @flags: ignored in our case, since we have Rx and Tx in the same NAPI
 753 *
 754 * Returns negative on error, zero otherwise.
 755 */
 756int
 757ice_xsk_wakeup(struct net_device *netdev, u32 queue_id,
 758               u32 __always_unused flags)
 759{
 760        struct ice_netdev_priv *np = netdev_priv(netdev);
 761        struct ice_q_vector *q_vector;
 762        struct ice_vsi *vsi = np->vsi;
 763        struct ice_ring *ring;
 764
 765        if (test_bit(ICE_DOWN, vsi->state))
 766                return -ENETDOWN;
 767
 768        if (!ice_is_xdp_ena_vsi(vsi))
 769                return -ENXIO;
 770
 771        if (queue_id >= vsi->num_txq)
 772                return -ENXIO;
 773
 774        if (!vsi->xdp_rings[queue_id]->xsk_pool)
 775                return -ENXIO;
 776
 777        ring = vsi->xdp_rings[queue_id];
 778
 779        /* The idea here is that if NAPI is running, mark a miss, so
 780         * it will run again. If not, trigger an interrupt and
 781         * schedule the NAPI from interrupt context. If NAPI would be
 782         * scheduled here, the interrupt affinity would not be
 783         * honored.
 784         */
 785        q_vector = ring->q_vector;
 786        if (!napi_if_scheduled_mark_missed(&q_vector->napi))
 787                ice_trigger_sw_intr(&vsi->back->hw, q_vector);
 788
 789        return 0;
 790}
 791
 792/**
 793 * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP buff pool attached
 794 * @vsi: VSI to be checked
 795 *
 796 * Returns true if any of the Rx rings has an AF_XDP buff pool attached
 797 */
 798bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)
 799{
 800        int i;
 801
 802        ice_for_each_rxq(vsi, i) {
 803                if (xsk_get_pool_from_qid(vsi->netdev, i))
 804                        return true;
 805        }
 806
 807        return false;
 808}
 809
 810/**
 811 * ice_xsk_clean_rx_ring - clean buffer pool queues connected to a given Rx ring
 812 * @rx_ring: ring to be cleaned
 813 */
 814void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring)
 815{
 816        u16 i;
 817
 818        for (i = 0; i < rx_ring->count; i++) {
 819                struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];
 820
 821                if (!rx_buf->xdp)
 822                        continue;
 823
 824                rx_buf->xdp = NULL;
 825        }
 826}
 827
 828/**
 829 * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
 830 * @xdp_ring: XDP_Tx ring
 831 */
 832void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring)
 833{
 834        u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use;
 835        u32 xsk_frames = 0;
 836
 837        while (ntc != ntu) {
 838                struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];
 839
 840                if (tx_buf->raw_buf)
 841                        ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
 842                else
 843                        xsk_frames++;
 844
 845                tx_buf->raw_buf = NULL;
 846
 847                ntc++;
 848                if (ntc >= xdp_ring->count)
 849                        ntc = 0;
 850        }
 851
 852        if (xsk_frames)
 853                xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
 854}
 855