linux/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/* Marvell OcteonTx2 RVU Admin Function driver
   3 *
   4 * Copyright (C) 2020 Marvell.
   5 */
   6
   7#include <linux/bitfield.h>
   8
   9#include "rvu_struct.h"
  10#include "rvu_reg.h"
  11#include "rvu.h"
  12#include "npc.h"
  13
  14#define NPC_BYTESM              GENMASK_ULL(19, 16)
  15#define NPC_HDR_OFFSET          GENMASK_ULL(15, 8)
  16#define NPC_KEY_OFFSET          GENMASK_ULL(5, 0)
  17#define NPC_LDATA_EN            BIT_ULL(7)
  18
  19static const char * const npc_flow_names[] = {
  20        [NPC_DMAC]      = "dmac",
  21        [NPC_SMAC]      = "smac",
  22        [NPC_ETYPE]     = "ether type",
  23        [NPC_OUTER_VID] = "outer vlan id",
  24        [NPC_TOS]       = "tos",
  25        [NPC_SIP_IPV4]  = "ipv4 source ip",
  26        [NPC_DIP_IPV4]  = "ipv4 destination ip",
  27        [NPC_SIP_IPV6]  = "ipv6 source ip",
  28        [NPC_DIP_IPV6]  = "ipv6 destination ip",
  29        [NPC_IPPROTO_TCP] = "ip proto tcp",
  30        [NPC_IPPROTO_UDP] = "ip proto udp",
  31        [NPC_IPPROTO_SCTP] = "ip proto sctp",
  32        [NPC_IPPROTO_ICMP] = "ip proto icmp",
  33        [NPC_IPPROTO_ICMP6] = "ip proto icmp6",
  34        [NPC_IPPROTO_AH] = "ip proto AH",
  35        [NPC_IPPROTO_ESP] = "ip proto ESP",
  36        [NPC_SPORT_TCP] = "tcp source port",
  37        [NPC_DPORT_TCP] = "tcp destination port",
  38        [NPC_SPORT_UDP] = "udp source port",
  39        [NPC_DPORT_UDP] = "udp destination port",
  40        [NPC_SPORT_SCTP] = "sctp source port",
  41        [NPC_DPORT_SCTP] = "sctp destination port",
  42        [NPC_UNKNOWN]   = "unknown",
  43};
  44
  45const char *npc_get_field_name(u8 hdr)
  46{
  47        if (hdr >= ARRAY_SIZE(npc_flow_names))
  48                return npc_flow_names[NPC_UNKNOWN];
  49
  50        return npc_flow_names[hdr];
  51}
  52
  53/* Compute keyword masks and figure out the number of keywords a field
  54 * spans in the key.
  55 */
  56static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type,
  57                             u8 nr_bits, int start_kwi, int offset, u8 intf)
  58{
  59        struct npc_key_field *field = &mcam->rx_key_fields[type];
  60        u8 bits_in_kw;
  61        int max_kwi;
  62
  63        if (mcam->banks_per_entry == 1)
  64                max_kwi = 1; /* NPC_MCAM_KEY_X1 */
  65        else if (mcam->banks_per_entry == 2)
  66                max_kwi = 3; /* NPC_MCAM_KEY_X2 */
  67        else
  68                max_kwi = 6; /* NPC_MCAM_KEY_X4 */
  69
  70        if (is_npc_intf_tx(intf))
  71                field = &mcam->tx_key_fields[type];
  72
  73        if (offset + nr_bits <= 64) {
  74                /* one KW only */
  75                if (start_kwi > max_kwi)
  76                        return;
  77                field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0)
  78                                             << offset;
  79                field->nr_kws = 1;
  80        } else if (offset + nr_bits > 64 &&
  81                   offset + nr_bits <= 128) {
  82                /* two KWs */
  83                if (start_kwi + 1 > max_kwi)
  84                        return;
  85                /* first KW mask */
  86                bits_in_kw = 64 - offset;
  87                field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
  88                                             << offset;
  89                /* second KW mask i.e. mask for rest of bits */
  90                bits_in_kw = nr_bits + offset - 64;
  91                field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0);
  92                field->nr_kws = 2;
  93        } else {
  94                /* three KWs */
  95                if (start_kwi + 2 > max_kwi)
  96                        return;
  97                /* first KW mask */
  98                bits_in_kw = 64 - offset;
  99                field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
 100                                             << offset;
 101                /* second KW mask */
 102                field->kw_mask[start_kwi + 1] = ~0ULL;
 103                /* third KW mask i.e. mask for rest of bits */
 104                bits_in_kw = nr_bits + offset - 128;
 105                field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0);
 106                field->nr_kws = 3;
 107        }
 108}
 109
 110/* Helper function to figure out whether field exists in the key */
 111static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf)
 112{
 113        struct npc_mcam *mcam = &rvu->hw->mcam;
 114        struct npc_key_field *input;
 115
 116        input  = &mcam->rx_key_fields[type];
 117        if (is_npc_intf_tx(intf))
 118                input  = &mcam->tx_key_fields[type];
 119
 120        return input->nr_kws > 0;
 121}
 122
 123static bool npc_is_same(struct npc_key_field *input,
 124                        struct npc_key_field *field)
 125{
 126        return memcmp(&input->layer_mdata, &field->layer_mdata,
 127                     sizeof(struct npc_layer_mdata)) == 0;
 128}
 129
 130static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type,
 131                                u64 cfg, u8 lid, u8 lt, u8 intf)
 132{
 133        struct npc_key_field *input = &mcam->rx_key_fields[type];
 134
 135        if (is_npc_intf_tx(intf))
 136                input = &mcam->tx_key_fields[type];
 137
 138        input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 139        input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 140        input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1;
 141        input->layer_mdata.ltype = lt;
 142        input->layer_mdata.lid = lid;
 143}
 144
 145static bool npc_check_overlap_fields(struct npc_key_field *input1,
 146                                     struct npc_key_field *input2)
 147{
 148        int kwi;
 149
 150        /* Fields with same layer id and different ltypes are mutually
 151         * exclusive hence they can be overlapped
 152         */
 153        if (input1->layer_mdata.lid == input2->layer_mdata.lid &&
 154            input1->layer_mdata.ltype != input2->layer_mdata.ltype)
 155                return false;
 156
 157        for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) {
 158                if (input1->kw_mask[kwi] & input2->kw_mask[kwi])
 159                        return true;
 160        }
 161
 162        return false;
 163}
 164
 165/* Helper function to check whether given field overlaps with any other fields
 166 * in the key. Due to limitations on key size and the key extraction profile in
 167 * use higher layers can overwrite lower layer's header fields. Hence overlap
 168 * needs to be checked.
 169 */
 170static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 171                              enum key_fields type, u8 start_lid, u8 intf)
 172{
 173        struct npc_mcam *mcam = &rvu->hw->mcam;
 174        struct npc_key_field *dummy, *input;
 175        int start_kwi, offset;
 176        u8 nr_bits, lid, lt, ld;
 177        u64 cfg;
 178
 179        dummy = &mcam->rx_key_fields[NPC_UNKNOWN];
 180        input = &mcam->rx_key_fields[type];
 181
 182        if (is_npc_intf_tx(intf)) {
 183                dummy = &mcam->tx_key_fields[NPC_UNKNOWN];
 184                input = &mcam->tx_key_fields[type];
 185        }
 186
 187        for (lid = start_lid; lid < NPC_MAX_LID; lid++) {
 188                for (lt = 0; lt < NPC_MAX_LT; lt++) {
 189                        for (ld = 0; ld < NPC_MAX_LD; ld++) {
 190                                cfg = rvu_read64(rvu, blkaddr,
 191                                                 NPC_AF_INTFX_LIDX_LTX_LDX_CFG
 192                                                 (intf, lid, lt, ld));
 193                                if (!FIELD_GET(NPC_LDATA_EN, cfg))
 194                                        continue;
 195                                memset(dummy, 0, sizeof(struct npc_key_field));
 196                                npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg,
 197                                                    lid, lt, intf);
 198                                /* exclude input */
 199                                if (npc_is_same(input, dummy))
 200                                        continue;
 201                                start_kwi = dummy->layer_mdata.key / 8;
 202                                offset = (dummy->layer_mdata.key * 8) % 64;
 203                                nr_bits = dummy->layer_mdata.len * 8;
 204                                /* form KW masks */
 205                                npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits,
 206                                                 start_kwi, offset, intf);
 207                                /* check any input field bits falls in any
 208                                 * other field bits.
 209                                 */
 210                                if (npc_check_overlap_fields(dummy, input))
 211                                        return true;
 212                        }
 213                }
 214        }
 215
 216        return false;
 217}
 218
 219static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type,
 220                            u8 intf)
 221{
 222        if (!npc_is_field_present(rvu, type, intf) ||
 223            npc_check_overlap(rvu, blkaddr, type, 0, intf))
 224                return false;
 225        return true;
 226}
 227
 228static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
 229                                  u8 key_nibble, u8 intf)
 230{
 231        u8 offset = (key_nibble * 4) % 64; /* offset within key word */
 232        u8 kwi = (key_nibble * 4) / 64; /* which word in key */
 233        u8 nr_bits = 4; /* bits in a nibble */
 234        u8 type;
 235
 236        switch (bit_number) {
 237        case 0 ... 2:
 238                type = NPC_CHAN;
 239                break;
 240        case 3:
 241                type = NPC_ERRLEV;
 242                break;
 243        case 4 ... 5:
 244                type = NPC_ERRCODE;
 245                break;
 246        case 6:
 247                type = NPC_LXMB;
 248                break;
 249        /* check for LTYPE only as of now */
 250        case 9:
 251                type = NPC_LA;
 252                break;
 253        case 12:
 254                type = NPC_LB;
 255                break;
 256        case 15:
 257                type = NPC_LC;
 258                break;
 259        case 18:
 260                type = NPC_LD;
 261                break;
 262        case 21:
 263                type = NPC_LE;
 264                break;
 265        case 24:
 266                type = NPC_LF;
 267                break;
 268        case 27:
 269                type = NPC_LG;
 270                break;
 271        case 30:
 272                type = NPC_LH;
 273                break;
 274        default:
 275                return;
 276        }
 277        npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf);
 278}
 279
 280static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 281{
 282        struct npc_mcam *mcam = &rvu->hw->mcam;
 283        struct npc_key_field *key_fields;
 284        /* Ether type can come from three layers
 285         * (ethernet, single tagged, double tagged)
 286         */
 287        struct npc_key_field *etype_ether;
 288        struct npc_key_field *etype_tag1;
 289        struct npc_key_field *etype_tag2;
 290        /* Outer VLAN TCI can come from two layers
 291         * (single tagged, double tagged)
 292         */
 293        struct npc_key_field *vlan_tag1;
 294        struct npc_key_field *vlan_tag2;
 295        u64 *features;
 296        u8 start_lid;
 297        int i;
 298
 299        key_fields = mcam->rx_key_fields;
 300        features = &mcam->rx_features;
 301
 302        if (is_npc_intf_tx(intf)) {
 303                key_fields = mcam->tx_key_fields;
 304                features = &mcam->tx_features;
 305        }
 306
 307        /* Handle header fields which can come from multiple layers like
 308         * etype, outer vlan tci. These fields should have same position in
 309         * the key otherwise to install a mcam rule more than one entry is
 310         * needed which complicates mcam space management.
 311         */
 312        etype_ether = &key_fields[NPC_ETYPE_ETHER];
 313        etype_tag1 = &key_fields[NPC_ETYPE_TAG1];
 314        etype_tag2 = &key_fields[NPC_ETYPE_TAG2];
 315        vlan_tag1 = &key_fields[NPC_VLAN_TAG1];
 316        vlan_tag2 = &key_fields[NPC_VLAN_TAG2];
 317
 318        /* if key profile programmed does not extract Ethertype at all */
 319        if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
 320                goto vlan_tci;
 321
 322        /* if key profile programmed extracts Ethertype from one layer */
 323        if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
 324                key_fields[NPC_ETYPE] = *etype_ether;
 325        if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws)
 326                key_fields[NPC_ETYPE] = *etype_tag1;
 327        if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws)
 328                key_fields[NPC_ETYPE] = *etype_tag2;
 329
 330        /* if key profile programmed extracts Ethertype from multiple layers */
 331        if (etype_ether->nr_kws && etype_tag1->nr_kws) {
 332                for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 333                        if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i])
 334                                goto vlan_tci;
 335                }
 336                key_fields[NPC_ETYPE] = *etype_tag1;
 337        }
 338        if (etype_ether->nr_kws && etype_tag2->nr_kws) {
 339                for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 340                        if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i])
 341                                goto vlan_tci;
 342                }
 343                key_fields[NPC_ETYPE] = *etype_tag2;
 344        }
 345        if (etype_tag1->nr_kws && etype_tag2->nr_kws) {
 346                for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 347                        if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i])
 348                                goto vlan_tci;
 349                }
 350                key_fields[NPC_ETYPE] = *etype_tag2;
 351        }
 352
 353        /* check none of higher layers overwrite Ethertype */
 354        start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1;
 355        if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf))
 356                goto vlan_tci;
 357        *features |= BIT_ULL(NPC_ETYPE);
 358vlan_tci:
 359        /* if key profile does not extract outer vlan tci at all */
 360        if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
 361                goto done;
 362
 363        /* if key profile extracts outer vlan tci from one layer */
 364        if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
 365                key_fields[NPC_OUTER_VID] = *vlan_tag1;
 366        if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws)
 367                key_fields[NPC_OUTER_VID] = *vlan_tag2;
 368
 369        /* if key profile extracts outer vlan tci from multiple layers */
 370        if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) {
 371                for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 372                        if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i])
 373                                goto done;
 374                }
 375                key_fields[NPC_OUTER_VID] = *vlan_tag2;
 376        }
 377        /* check none of higher layers overwrite outer vlan tci */
 378        start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1;
 379        if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf))
 380                goto done;
 381        *features |= BIT_ULL(NPC_OUTER_VID);
 382done:
 383        return;
 384}
 385
 386static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid,
 387                           u8 lt, u64 cfg, u8 intf)
 388{
 389        struct npc_mcam *mcam = &rvu->hw->mcam;
 390        u8 hdr, key, nr_bytes, bit_offset;
 391        u8 la_ltype, la_start;
 392        /* starting KW index and starting bit position */
 393        int start_kwi, offset;
 394
 395        nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1;
 396        hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 397        key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 398        start_kwi = key / 8;
 399        offset = (key * 8) % 64;
 400
 401        /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding
 402         * ethernet header.
 403         */
 404        if (is_npc_intf_tx(intf)) {
 405                la_ltype = NPC_LT_LA_IH_NIX_ETHER;
 406                la_start = 8;
 407        } else {
 408                la_ltype = NPC_LT_LA_ETHER;
 409                la_start = 0;
 410        }
 411
 412#define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen)                            \
 413do {                                                                           \
 414        if (lid == (hlid) && lt == (hlt)) {                                    \
 415                if ((hstart) >= hdr &&                                         \
 416                    ((hstart) + (hlen)) <= (hdr + nr_bytes)) {                 \
 417                        bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \
 418                        npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \
 419                        npc_set_kw_masks(mcam, (name), (hlen) * 8,             \
 420                                         start_kwi, offset + bit_offset, intf);\
 421                }                                                              \
 422        }                                                                      \
 423} while (0)
 424
 425        /* List LID, LTYPE, start offset from layer and length(in bytes) of
 426         * packet header fields below.
 427         * Example: Source IP is 4 bytes and starts at 12th byte of IP header
 428         */
 429        NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1);
 430        NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4);
 431        NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4);
 432        NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16);
 433        NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16);
 434        NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2);
 435        NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2);
 436        NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2);
 437        NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2);
 438        NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2);
 439        NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2);
 440        NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2);
 441        NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2);
 442        NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2);
 443        NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2);
 444        NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2);
 445        NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6);
 446        NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6);
 447        /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */
 448        NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2);
 449}
 450
 451static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf)
 452{
 453        struct npc_mcam *mcam = &rvu->hw->mcam;
 454        u64 *features = &mcam->rx_features;
 455        u64 tcp_udp_sctp;
 456        int hdr;
 457
 458        if (is_npc_intf_tx(intf))
 459                features = &mcam->tx_features;
 460
 461        for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) {
 462                if (npc_check_field(rvu, blkaddr, hdr, intf))
 463                        *features |= BIT_ULL(hdr);
 464        }
 465
 466        tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) |
 467                       BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) |
 468                       BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP);
 469
 470        /* for tcp/udp/sctp corresponding layer type should be in the key */
 471        if (*features & tcp_udp_sctp) {
 472                if (!npc_check_field(rvu, blkaddr, NPC_LD, intf))
 473                        *features &= ~tcp_udp_sctp;
 474                else
 475                        *features |= BIT_ULL(NPC_IPPROTO_TCP) |
 476                                     BIT_ULL(NPC_IPPROTO_UDP) |
 477                                     BIT_ULL(NPC_IPPROTO_SCTP);
 478        }
 479
 480        /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */
 481        if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) {
 482                *features |= BIT_ULL(NPC_IPPROTO_AH);
 483                *features |= BIT_ULL(NPC_IPPROTO_ICMP);
 484                *features |= BIT_ULL(NPC_IPPROTO_ICMP6);
 485        }
 486
 487        /* for ESP, check if corresponding layer type is present in the key */
 488        if (npc_check_field(rvu, blkaddr, NPC_LE, intf))
 489                *features |= BIT_ULL(NPC_IPPROTO_ESP);
 490
 491        /* for vlan corresponding layer type should be in the key */
 492        if (*features & BIT_ULL(NPC_OUTER_VID))
 493                if (!npc_check_field(rvu, blkaddr, NPC_LB, intf))
 494                        *features &= ~BIT_ULL(NPC_OUTER_VID);
 495}
 496
 497/* Scan key extraction profile and record how fields of our interest
 498 * fill the key structure. Also verify Channel and DMAC exists in
 499 * key and not overwritten by other header fields.
 500 */
 501static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 502{
 503        struct npc_mcam *mcam = &rvu->hw->mcam;
 504        u8 lid, lt, ld, bitnr;
 505        u8 key_nibble = 0;
 506        u64 cfg;
 507
 508        /* Scan and note how parse result is going to be in key.
 509         * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from
 510         * parse result in the key. The enabled nibbles from parse result
 511         * will be concatenated in key.
 512         */
 513        cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
 514        cfg &= NPC_PARSE_NIBBLE;
 515        for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) {
 516                npc_scan_parse_result(mcam, bitnr, key_nibble, intf);
 517                key_nibble++;
 518        }
 519
 520        /* Scan and note how layer data is going to be in key */
 521        for (lid = 0; lid < NPC_MAX_LID; lid++) {
 522                for (lt = 0; lt < NPC_MAX_LT; lt++) {
 523                        for (ld = 0; ld < NPC_MAX_LD; ld++) {
 524                                cfg = rvu_read64(rvu, blkaddr,
 525                                                 NPC_AF_INTFX_LIDX_LTX_LDX_CFG
 526                                                 (intf, lid, lt, ld));
 527                                if (!FIELD_GET(NPC_LDATA_EN, cfg))
 528                                        continue;
 529                                npc_scan_ldata(rvu, blkaddr, lid, lt, cfg,
 530                                               intf);
 531                        }
 532                }
 533        }
 534
 535        return 0;
 536}
 537
 538static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr)
 539{
 540        int err;
 541
 542        err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX);
 543        if (err)
 544                return err;
 545
 546        err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX);
 547        if (err)
 548                return err;
 549
 550        /* Channel is mandatory */
 551        if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) {
 552                dev_err(rvu->dev, "Channel not present in Key\n");
 553                return -EINVAL;
 554        }
 555        /* check that none of the fields overwrite channel */
 556        if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) {
 557                dev_err(rvu->dev, "Channel cannot be overwritten\n");
 558                return -EINVAL;
 559        }
 560        /* DMAC should be present in key for unicast filter to work */
 561        if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) {
 562                dev_err(rvu->dev, "DMAC not present in Key\n");
 563                return -EINVAL;
 564        }
 565        /* check that none of the fields overwrite DMAC */
 566        if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) {
 567                dev_err(rvu->dev, "DMAC cannot be overwritten\n");
 568                return -EINVAL;
 569        }
 570
 571        npc_set_features(rvu, blkaddr, NIX_INTF_TX);
 572        npc_set_features(rvu, blkaddr, NIX_INTF_RX);
 573        npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX);
 574        npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX);
 575
 576        return 0;
 577}
 578
 579int npc_flow_steering_init(struct rvu *rvu, int blkaddr)
 580{
 581        struct npc_mcam *mcam = &rvu->hw->mcam;
 582
 583        INIT_LIST_HEAD(&mcam->mcam_rules);
 584
 585        return npc_scan_verify_kex(rvu, blkaddr);
 586}
 587
 588static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf)
 589{
 590        struct npc_mcam *mcam = &rvu->hw->mcam;
 591        u64 *mcam_features = &mcam->rx_features;
 592        u64 unsupported;
 593        u8 bit;
 594
 595        if (is_npc_intf_tx(intf))
 596                mcam_features = &mcam->tx_features;
 597
 598        unsupported = (*mcam_features ^ features) & ~(*mcam_features);
 599        if (unsupported) {
 600                dev_info(rvu->dev, "Unsupported flow(s):\n");
 601                for_each_set_bit(bit, (unsigned long *)&unsupported, 64)
 602                        dev_info(rvu->dev, "%s ", npc_get_field_name(bit));
 603                return NIX_AF_ERR_NPC_KEY_NOT_SUPP;
 604        }
 605
 606        return 0;
 607}
 608
 609/* npc_update_entry - Based on the masks generated during
 610 * the key scanning, updates the given entry with value and
 611 * masks for the field of interest. Maximum 16 bytes of a packet
 612 * header can be extracted by HW hence lo and hi are sufficient.
 613 * When field bytes are less than or equal to 8 then hi should be
 614 * 0 for value and mask.
 615 *
 616 * If exact match of value is required then mask should be all 1's.
 617 * If any bits in mask are 0 then corresponding bits in value are
 618 * dont care.
 619 */
 620static void npc_update_entry(struct rvu *rvu, enum key_fields type,
 621                             struct mcam_entry *entry, u64 val_lo,
 622                             u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf)
 623{
 624        struct npc_mcam *mcam = &rvu->hw->mcam;
 625        struct mcam_entry dummy = { {0} };
 626        struct npc_key_field *field;
 627        u64 kw1, kw2, kw3;
 628        u8 shift;
 629        int i;
 630
 631        field = &mcam->rx_key_fields[type];
 632        if (is_npc_intf_tx(intf))
 633                field = &mcam->tx_key_fields[type];
 634
 635        if (!field->nr_kws)
 636                return;
 637
 638        for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 639                if (!field->kw_mask[i])
 640                        continue;
 641                /* place key value in kw[x] */
 642                shift = __ffs64(field->kw_mask[i]);
 643                /* update entry value */
 644                kw1 = (val_lo << shift) & field->kw_mask[i];
 645                dummy.kw[i] = kw1;
 646                /* update entry mask */
 647                kw1 = (mask_lo << shift) & field->kw_mask[i];
 648                dummy.kw_mask[i] = kw1;
 649
 650                if (field->nr_kws == 1)
 651                        break;
 652                /* place remaining bits of key value in kw[x + 1] */
 653                if (field->nr_kws == 2) {
 654                        /* update entry value */
 655                        kw2 = shift ? val_lo >> (64 - shift) : 0;
 656                        kw2 |= (val_hi << shift);
 657                        kw2 &= field->kw_mask[i + 1];
 658                        dummy.kw[i + 1] = kw2;
 659                        /* update entry mask */
 660                        kw2 = shift ? mask_lo >> (64 - shift) : 0;
 661                        kw2 |= (mask_hi << shift);
 662                        kw2 &= field->kw_mask[i + 1];
 663                        dummy.kw_mask[i + 1] = kw2;
 664                        break;
 665                }
 666                /* place remaining bits of key value in kw[x + 1], kw[x + 2] */
 667                if (field->nr_kws == 3) {
 668                        /* update entry value */
 669                        kw2 = shift ? val_lo >> (64 - shift) : 0;
 670                        kw2 |= (val_hi << shift);
 671                        kw2 &= field->kw_mask[i + 1];
 672                        kw3 = shift ? val_hi >> (64 - shift) : 0;
 673                        kw3 &= field->kw_mask[i + 2];
 674                        dummy.kw[i + 1] = kw2;
 675                        dummy.kw[i + 2] = kw3;
 676                        /* update entry mask */
 677                        kw2 = shift ? mask_lo >> (64 - shift) : 0;
 678                        kw2 |= (mask_hi << shift);
 679                        kw2 &= field->kw_mask[i + 1];
 680                        kw3 = shift ? mask_hi >> (64 - shift) : 0;
 681                        kw3 &= field->kw_mask[i + 2];
 682                        dummy.kw_mask[i + 1] = kw2;
 683                        dummy.kw_mask[i + 2] = kw3;
 684                        break;
 685                }
 686        }
 687        /* dummy is ready with values and masks for given key
 688         * field now clear and update input entry with those
 689         */
 690        for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 691                if (!field->kw_mask[i])
 692                        continue;
 693                entry->kw[i] &= ~field->kw_mask[i];
 694                entry->kw_mask[i] &= ~field->kw_mask[i];
 695
 696                entry->kw[i] |= dummy.kw[i];
 697                entry->kw_mask[i] |= dummy.kw_mask[i];
 698        }
 699}
 700
 701#define IPV6_WORDS     4
 702
 703static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
 704                                 u64 features, struct flow_msg *pkt,
 705                                 struct flow_msg *mask,
 706                                 struct rvu_npc_mcam_rule *output, u8 intf)
 707{
 708        u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS];
 709        u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS];
 710        struct flow_msg *opkt = &output->packet;
 711        struct flow_msg *omask = &output->mask;
 712        u64 mask_lo, mask_hi;
 713        u64 val_lo, val_hi;
 714
 715        /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet
 716         * values to be programmed in MCAM should as below:
 717         * val_high: 0xfe80000000000000
 718         * val_low: 0x2c6863fffe5e2d0a
 719         */
 720        if (features & BIT_ULL(NPC_SIP_IPV6)) {
 721                be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS);
 722                be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS);
 723
 724                mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1];
 725                mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3];
 726                val_hi = (u64)src_ip[0] << 32 | src_ip[1];
 727                val_lo = (u64)src_ip[2] << 32 | src_ip[3];
 728
 729                npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi,
 730                                 mask_lo, mask_hi, intf);
 731                memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src));
 732                memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src));
 733        }
 734        if (features & BIT_ULL(NPC_DIP_IPV6)) {
 735                be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS);
 736                be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS);
 737
 738                mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1];
 739                mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3];
 740                val_hi = (u64)dst_ip[0] << 32 | dst_ip[1];
 741                val_lo = (u64)dst_ip[2] << 32 | dst_ip[3];
 742
 743                npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi,
 744                                 mask_lo, mask_hi, intf);
 745                memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst));
 746                memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst));
 747        }
 748}
 749
 750static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry,
 751                            u64 features, struct flow_msg *pkt,
 752                            struct flow_msg *mask,
 753                            struct rvu_npc_mcam_rule *output, u8 intf)
 754{
 755        u64 dmac_mask = ether_addr_to_u64(mask->dmac);
 756        u64 smac_mask = ether_addr_to_u64(mask->smac);
 757        u64 dmac_val = ether_addr_to_u64(pkt->dmac);
 758        u64 smac_val = ether_addr_to_u64(pkt->smac);
 759        struct flow_msg *opkt = &output->packet;
 760        struct flow_msg *omask = &output->mask;
 761
 762        if (!features)
 763                return;
 764
 765        /* For tcp/udp/sctp LTYPE should be present in entry */
 766        if (features & BIT_ULL(NPC_IPPROTO_TCP))
 767                npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP,
 768                                 0, ~0ULL, 0, intf);
 769        if (features & BIT_ULL(NPC_IPPROTO_UDP))
 770                npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP,
 771                                 0, ~0ULL, 0, intf);
 772        if (features & BIT_ULL(NPC_IPPROTO_SCTP))
 773                npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP,
 774                                 0, ~0ULL, 0, intf);
 775        if (features & BIT_ULL(NPC_IPPROTO_ICMP))
 776                npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP,
 777                                 0, ~0ULL, 0, intf);
 778        if (features & BIT_ULL(NPC_IPPROTO_ICMP6))
 779                npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6,
 780                                 0, ~0ULL, 0, intf);
 781
 782        if (features & BIT_ULL(NPC_OUTER_VID))
 783                npc_update_entry(rvu, NPC_LB, entry,
 784                                 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0,
 785                                 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf);
 786
 787        /* For AH, LTYPE should be present in entry */
 788        if (features & BIT_ULL(NPC_IPPROTO_AH))
 789                npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH,
 790                                 0, ~0ULL, 0, intf);
 791        /* For ESP, LTYPE should be present in entry */
 792        if (features & BIT_ULL(NPC_IPPROTO_ESP))
 793                npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP,
 794                                 0, ~0ULL, 0, intf);
 795
 796#define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi)       \
 797do {                                                                          \
 798        if (features & BIT_ULL((field))) {                                    \
 799                npc_update_entry(rvu, (field), entry, (val_lo), (val_hi),     \
 800                                 (mask_lo), (mask_hi), intf);                 \
 801                memcpy(&opkt->member, &pkt->member, sizeof(pkt->member));     \
 802                memcpy(&omask->member, &mask->member, sizeof(mask->member));  \
 803        }                                                                     \
 804} while (0)
 805
 806        NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0);
 807        NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0);
 808        NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0,
 809                       ntohs(mask->etype), 0);
 810        NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0);
 811        NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0,
 812                       ntohl(mask->ip4src), 0);
 813        NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0,
 814                       ntohl(mask->ip4dst), 0);
 815        NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0,
 816                       ntohs(mask->sport), 0);
 817        NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0,
 818                       ntohs(mask->sport), 0);
 819        NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0,
 820                       ntohs(mask->dport), 0);
 821        NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0,
 822                       ntohs(mask->dport), 0);
 823        NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0,
 824                       ntohs(mask->sport), 0);
 825        NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0,
 826                       ntohs(mask->dport), 0);
 827
 828        NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0,
 829                       ntohs(mask->vlan_tci), 0);
 830
 831        npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf);
 832}
 833
 834static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam,
 835                                                    u16 entry)
 836{
 837        struct rvu_npc_mcam_rule *iter;
 838
 839        mutex_lock(&mcam->lock);
 840        list_for_each_entry(iter, &mcam->mcam_rules, list) {
 841                if (iter->entry == entry) {
 842                        mutex_unlock(&mcam->lock);
 843                        return iter;
 844                }
 845        }
 846        mutex_unlock(&mcam->lock);
 847
 848        return NULL;
 849}
 850
 851static void rvu_mcam_add_rule(struct npc_mcam *mcam,
 852                              struct rvu_npc_mcam_rule *rule)
 853{
 854        struct list_head *head = &mcam->mcam_rules;
 855        struct rvu_npc_mcam_rule *iter;
 856
 857        mutex_lock(&mcam->lock);
 858        list_for_each_entry(iter, &mcam->mcam_rules, list) {
 859                if (iter->entry > rule->entry)
 860                        break;
 861                head = &iter->list;
 862        }
 863
 864        list_add(&rule->list, head);
 865        mutex_unlock(&mcam->lock);
 866}
 867
 868static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc,
 869                                              struct rvu_npc_mcam_rule *rule)
 870{
 871        struct npc_mcam_oper_counter_req free_req = { 0 };
 872        struct msg_rsp free_rsp;
 873
 874        if (!rule->has_cntr)
 875                return;
 876
 877        free_req.hdr.pcifunc = pcifunc;
 878        free_req.cntr = rule->cntr;
 879
 880        rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp);
 881        rule->has_cntr = false;
 882}
 883
 884static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc,
 885                                         struct rvu_npc_mcam_rule *rule,
 886                                         struct npc_install_flow_rsp *rsp)
 887{
 888        struct npc_mcam_alloc_counter_req cntr_req = { 0 };
 889        struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 };
 890        int err;
 891
 892        cntr_req.hdr.pcifunc = pcifunc;
 893        cntr_req.contig = true;
 894        cntr_req.count = 1;
 895
 896        /* we try to allocate a counter to track the stats of this
 897         * rule. If counter could not be allocated then proceed
 898         * without counter because counters are limited than entries.
 899         */
 900        err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req,
 901                                                      &cntr_rsp);
 902        if (!err && cntr_rsp.count) {
 903                rule->cntr = cntr_rsp.cntr;
 904                rule->has_cntr = true;
 905                rsp->counter = rule->cntr;
 906        } else {
 907                rsp->counter = err;
 908        }
 909}
 910
 911static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 912                                struct mcam_entry *entry,
 913                                struct npc_install_flow_req *req,
 914                                u16 target, bool pf_set_vfs_mac)
 915{
 916        struct rvu_switch *rswitch = &rvu->rswitch;
 917        struct nix_rx_action action;
 918
 919        if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac)
 920                req->chan_mask = 0x0; /* Do not care channel */
 921
 922        npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask,
 923                         0, NIX_INTF_RX);
 924
 925        *(u64 *)&action = 0x00;
 926        action.pf_func = target;
 927        action.op = req->op;
 928        action.index = req->index;
 929        action.match_id = req->match_id;
 930        action.flow_key_alg = req->flow_key_alg;
 931
 932        if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule)
 933                action = pfvf->def_ucast_rule->rx_action;
 934
 935        entry->action = *(u64 *)&action;
 936
 937        /* VTAG0 starts at 0th byte of LID_B.
 938         * VTAG1 starts at 4th byte of LID_B.
 939         */
 940        entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) |
 941                             FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) |
 942                             FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) |
 943                             FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) |
 944                             FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) |
 945                             FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) |
 946                             FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) |
 947                             FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4);
 948}
 949
 950static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 951                                struct mcam_entry *entry,
 952                                struct npc_install_flow_req *req, u16 target)
 953{
 954        struct nix_tx_action action;
 955        u64 mask = ~0ULL;
 956
 957        /* If AF is installing then do not care about
 958         * PF_FUNC in Send Descriptor
 959         */
 960        if (is_pffunc_af(req->hdr.pcifunc))
 961                mask = 0;
 962
 963        npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target),
 964                         0, mask, 0, NIX_INTF_TX);
 965
 966        *(u64 *)&action = 0x00;
 967        action.op = req->op;
 968        action.index = req->index;
 969        action.match_id = req->match_id;
 970
 971        entry->action = *(u64 *)&action;
 972
 973        /* VTAG0 starts at 0th byte of LID_B.
 974         * VTAG1 starts at 4th byte of LID_B.
 975         */
 976        entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) |
 977                             FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) |
 978                             FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) |
 979                             FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) |
 980                             FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) |
 981                             FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) |
 982                             FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) |
 983                             FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24);
 984}
 985
 986static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 987                            int nixlf, struct rvu_pfvf *pfvf,
 988                            struct npc_install_flow_req *req,
 989                            struct npc_install_flow_rsp *rsp, bool enable,
 990                            bool pf_set_vfs_mac)
 991{
 992        struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule;
 993        u64 features, installed_features, missing_features = 0;
 994        struct npc_mcam_write_entry_req write_req = { 0 };
 995        struct npc_mcam *mcam = &rvu->hw->mcam;
 996        struct rvu_npc_mcam_rule dummy = { 0 };
 997        struct rvu_npc_mcam_rule *rule;
 998        bool new = false, msg_from_vf;
 999        u16 owner = req->hdr.pcifunc;
1000        struct msg_rsp write_rsp;
1001        struct mcam_entry *entry;
1002        int entry_index, err;
1003
1004        msg_from_vf = !!(owner & RVU_PFVF_FUNC_MASK);
1005
1006        installed_features = req->features;
1007        features = req->features;
1008        entry = &write_req.entry_data;
1009        entry_index = req->entry;
1010
1011        npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy,
1012                        req->intf);
1013
1014        if (is_npc_intf_rx(req->intf))
1015                npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac);
1016        else
1017                npc_update_tx_entry(rvu, pfvf, entry, req, target);
1018
1019        /* Default unicast rules do not exist for TX */
1020        if (is_npc_intf_tx(req->intf))
1021                goto find_rule;
1022
1023        if (req->default_rule) {
1024                entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf,
1025                                                       NIXLF_UCAST_ENTRY);
1026                enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index);
1027        }
1028
1029        /* update mcam entry with default unicast rule attributes */
1030        if (def_ucast_rule && (msg_from_vf || (req->default_rule && req->append))) {
1031                missing_features = (def_ucast_rule->features ^ features) &
1032                                        def_ucast_rule->features;
1033                if (missing_features)
1034                        npc_update_flow(rvu, entry, missing_features,
1035                                        &def_ucast_rule->packet,
1036                                        &def_ucast_rule->mask,
1037                                        &dummy, req->intf);
1038                installed_features = req->features | missing_features;
1039        }
1040
1041find_rule:
1042        rule = rvu_mcam_find_rule(mcam, entry_index);
1043        if (!rule) {
1044                rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1045                if (!rule)
1046                        return -ENOMEM;
1047                new = true;
1048        }
1049
1050        /* allocate new counter if rule has no counter */
1051        if (!req->default_rule && req->set_cntr && !rule->has_cntr)
1052                rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp);
1053
1054        /* if user wants to delete an existing counter for a rule then
1055         * free the counter
1056         */
1057        if (!req->set_cntr && rule->has_cntr)
1058                rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1059
1060        write_req.hdr.pcifunc = owner;
1061
1062        /* AF owns the default rules so change the owner just to relax
1063         * the checks in rvu_mbox_handler_npc_mcam_write_entry
1064         */
1065        if (req->default_rule)
1066                write_req.hdr.pcifunc = 0;
1067
1068        write_req.entry = entry_index;
1069        write_req.intf = req->intf;
1070        write_req.enable_entry = (u8)enable;
1071        /* if counter is available then clear and use it */
1072        if (req->set_cntr && rule->has_cntr) {
1073                rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00);
1074                write_req.set_cntr = 1;
1075                write_req.cntr = rule->cntr;
1076        }
1077
1078        err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
1079                                                    &write_rsp);
1080        if (err) {
1081                rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1082                if (new)
1083                        kfree(rule);
1084                return err;
1085        }
1086        /* update rule */
1087        memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet));
1088        memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask));
1089        rule->entry = entry_index;
1090        memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action));
1091        if (is_npc_intf_tx(req->intf))
1092                memcpy(&rule->tx_action, &entry->action,
1093                       sizeof(struct nix_tx_action));
1094        rule->vtag_action = entry->vtag_action;
1095        rule->features = installed_features;
1096        rule->default_rule = req->default_rule;
1097        rule->owner = owner;
1098        rule->enable = enable;
1099        if (is_npc_intf_tx(req->intf))
1100                rule->intf = pfvf->nix_tx_intf;
1101        else
1102                rule->intf = pfvf->nix_rx_intf;
1103
1104        if (new)
1105                rvu_mcam_add_rule(mcam, rule);
1106        if (req->default_rule)
1107                pfvf->def_ucast_rule = rule;
1108
1109        /* VF's MAC address is being changed via PF  */
1110        if (pf_set_vfs_mac) {
1111                ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1112                ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1113                set_bit(PF_SET_VF_MAC, &pfvf->flags);
1114        }
1115
1116        if (test_bit(PF_SET_VF_CFG, &pfvf->flags) &&
1117            req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7)
1118                rule->vfvlan_cfg = true;
1119
1120        if (is_npc_intf_rx(req->intf) && req->match_id &&
1121            (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS))
1122                return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc,
1123                                             req->index, req->match_id);
1124
1125        return 0;
1126}
1127
1128int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
1129                                      struct npc_install_flow_req *req,
1130                                      struct npc_install_flow_rsp *rsp)
1131{
1132        bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK);
1133        int blkaddr, nixlf, err;
1134        struct rvu_pfvf *pfvf;
1135        bool pf_set_vfs_mac = false;
1136        bool enable = true;
1137        u16 target;
1138
1139        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1140        if (blkaddr < 0) {
1141                dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
1142                return -ENODEV;
1143        }
1144
1145        if (!is_npc_interface_valid(rvu, req->intf))
1146                return -EINVAL;
1147
1148        if (from_vf && req->default_rule)
1149                return NPC_MCAM_PERM_DENIED;
1150
1151        /* Each PF/VF info is maintained in struct rvu_pfvf.
1152         * rvu_pfvf for the target PF/VF needs to be retrieved
1153         * hence modify pcifunc accordingly.
1154         */
1155
1156        /* AF installing for a PF/VF */
1157        if (!req->hdr.pcifunc)
1158                target = req->vf;
1159        /* PF installing for its VF */
1160        else if (!from_vf && req->vf) {
1161                target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf;
1162                pf_set_vfs_mac = req->default_rule &&
1163                                (req->features & BIT_ULL(NPC_DMAC));
1164        }
1165        /* msg received from PF/VF */
1166        else
1167                target = req->hdr.pcifunc;
1168
1169        /* ignore chan_mask in case pf func is not AF, revisit later */
1170        if (!is_pffunc_af(req->hdr.pcifunc))
1171                req->chan_mask = 0xFFF;
1172
1173        err = npc_check_unsupported_flows(rvu, req->features, req->intf);
1174        if (err)
1175                return err;
1176
1177        /* Skip channel validation if AF is installing */
1178        if (!is_pffunc_af(req->hdr.pcifunc) &&
1179            npc_mcam_verify_channel(rvu, target, req->intf, req->channel))
1180                return -EINVAL;
1181
1182        pfvf = rvu_get_pfvf(rvu, target);
1183
1184        /* PF installing for its VF */
1185        if (req->hdr.pcifunc && !from_vf && req->vf)
1186                set_bit(PF_SET_VF_CFG, &pfvf->flags);
1187
1188        /* update req destination mac addr */
1189        if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) &&
1190            is_zero_ether_addr(req->packet.dmac)) {
1191                ether_addr_copy(req->packet.dmac, pfvf->mac_addr);
1192                eth_broadcast_addr((u8 *)&req->mask.dmac);
1193        }
1194
1195        /* Proceed if NIXLF is attached or not for TX rules */
1196        err = nix_get_nixlf(rvu, target, &nixlf, NULL);
1197        if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac)
1198                return -EINVAL;
1199
1200        /* don't enable rule when nixlf not attached or initialized */
1201        if (!(is_nixlf_attached(rvu, target) &&
1202              test_bit(NIXLF_INITIALIZED, &pfvf->flags)))
1203                enable = false;
1204
1205        /* Packets reaching NPC in Tx path implies that a
1206         * NIXLF is properly setup and transmitting.
1207         * Hence rules can be enabled for Tx.
1208         */
1209        if (is_npc_intf_tx(req->intf))
1210                enable = true;
1211
1212        /* Do not allow requests from uninitialized VFs */
1213        if (from_vf && !enable)
1214                return -EINVAL;
1215
1216        /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */
1217        if (pf_set_vfs_mac && !enable) {
1218                ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1219                ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1220                set_bit(PF_SET_VF_MAC, &pfvf->flags);
1221                return 0;
1222        }
1223
1224        /* If message is from VF then its flow should not overlap with
1225         * reserved unicast flow.
1226         */
1227        if (from_vf && pfvf->def_ucast_rule && is_npc_intf_rx(req->intf) &&
1228            pfvf->def_ucast_rule->features & req->features)
1229                return -EINVAL;
1230
1231        return npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, req, rsp,
1232                                enable, pf_set_vfs_mac);
1233}
1234
1235static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule,
1236                           u16 pcifunc)
1237{
1238        struct npc_mcam_ena_dis_entry_req dis_req = { 0 };
1239        struct msg_rsp dis_rsp;
1240
1241        if (rule->default_rule)
1242                return 0;
1243
1244        if (rule->has_cntr)
1245                rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule);
1246
1247        dis_req.hdr.pcifunc = pcifunc;
1248        dis_req.entry = rule->entry;
1249
1250        list_del(&rule->list);
1251        kfree(rule);
1252
1253        return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp);
1254}
1255
1256int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu,
1257                                     struct npc_delete_flow_req *req,
1258                                     struct msg_rsp *rsp)
1259{
1260        struct npc_mcam *mcam = &rvu->hw->mcam;
1261        struct rvu_npc_mcam_rule *iter, *tmp;
1262        u16 pcifunc = req->hdr.pcifunc;
1263        struct list_head del_list;
1264
1265        INIT_LIST_HEAD(&del_list);
1266
1267        mutex_lock(&mcam->lock);
1268        list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) {
1269                if (iter->owner == pcifunc) {
1270                        /* All rules */
1271                        if (req->all) {
1272                                list_move_tail(&iter->list, &del_list);
1273                        /* Range of rules */
1274                        } else if (req->end && iter->entry >= req->start &&
1275                                   iter->entry <= req->end) {
1276                                list_move_tail(&iter->list, &del_list);
1277                        /* single rule */
1278                        } else if (req->entry == iter->entry) {
1279                                list_move_tail(&iter->list, &del_list);
1280                                break;
1281                        }
1282                }
1283        }
1284        mutex_unlock(&mcam->lock);
1285
1286        list_for_each_entry_safe(iter, tmp, &del_list, list) {
1287                u16 entry = iter->entry;
1288
1289                /* clear the mcam entry target pcifunc */
1290                mcam->entry2target_pffunc[entry] = 0x0;
1291                if (npc_delete_flow(rvu, iter, pcifunc))
1292                        dev_err(rvu->dev, "rule deletion failed for entry:%u",
1293                                entry);
1294        }
1295
1296        return 0;
1297}
1298
1299static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr,
1300                                 struct rvu_npc_mcam_rule *rule,
1301                                 struct rvu_pfvf *pfvf)
1302{
1303        struct npc_mcam_write_entry_req write_req = { 0 };
1304        struct mcam_entry *entry = &write_req.entry_data;
1305        struct npc_mcam *mcam = &rvu->hw->mcam;
1306        struct msg_rsp rsp;
1307        u8 intf, enable;
1308        int err;
1309
1310        ether_addr_copy(rule->packet.dmac, pfvf->mac_addr);
1311
1312        npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
1313                            entry, &intf,  &enable);
1314
1315        npc_update_entry(rvu, NPC_DMAC, entry,
1316                         ether_addr_to_u64(pfvf->mac_addr), 0,
1317                         0xffffffffffffull, 0, intf);
1318
1319        write_req.hdr.pcifunc = rule->owner;
1320        write_req.entry = rule->entry;
1321        write_req.intf = pfvf->nix_rx_intf;
1322
1323        mutex_unlock(&mcam->lock);
1324        err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp);
1325        mutex_lock(&mcam->lock);
1326
1327        return err;
1328}
1329
1330void npc_mcam_enable_flows(struct rvu *rvu, u16 target)
1331{
1332        struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target);
1333        struct rvu_npc_mcam_rule *def_ucast_rule;
1334        struct npc_mcam *mcam = &rvu->hw->mcam;
1335        struct rvu_npc_mcam_rule *rule;
1336        int blkaddr, bank, index;
1337        u64 def_action;
1338
1339        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1340        if (blkaddr < 0)
1341                return;
1342
1343        def_ucast_rule = pfvf->def_ucast_rule;
1344
1345        mutex_lock(&mcam->lock);
1346        list_for_each_entry(rule, &mcam->mcam_rules, list) {
1347                if (is_npc_intf_rx(rule->intf) &&
1348                    rule->rx_action.pf_func == target && !rule->enable) {
1349                        if (rule->default_rule) {
1350                                npc_enable_mcam_entry(rvu, mcam, blkaddr,
1351                                                      rule->entry, true);
1352                                rule->enable = true;
1353                                continue;
1354                        }
1355
1356                        if (rule->vfvlan_cfg)
1357                                npc_update_dmac_value(rvu, blkaddr, rule, pfvf);
1358
1359                        if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) {
1360                                if (!def_ucast_rule)
1361                                        continue;
1362                                /* Use default unicast entry action */
1363                                rule->rx_action = def_ucast_rule->rx_action;
1364                                def_action = *(u64 *)&def_ucast_rule->rx_action;
1365                                bank = npc_get_bank(mcam, rule->entry);
1366                                rvu_write64(rvu, blkaddr,
1367                                            NPC_AF_MCAMEX_BANKX_ACTION
1368                                            (rule->entry, bank), def_action);
1369                        }
1370
1371                        npc_enable_mcam_entry(rvu, mcam, blkaddr,
1372                                              rule->entry, true);
1373                        rule->enable = true;
1374                }
1375        }
1376
1377        /* Enable MCAM entries installed by PF with target as VF pcifunc */
1378        for (index = 0; index < mcam->bmap_entries; index++) {
1379                if (mcam->entry2target_pffunc[index] == target)
1380                        npc_enable_mcam_entry(rvu, mcam, blkaddr,
1381                                              index, true);
1382        }
1383        mutex_unlock(&mcam->lock);
1384}
1385
1386void npc_mcam_disable_flows(struct rvu *rvu, u16 target)
1387{
1388        struct npc_mcam *mcam = &rvu->hw->mcam;
1389        int blkaddr, index;
1390
1391        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1392        if (blkaddr < 0)
1393                return;
1394
1395        mutex_lock(&mcam->lock);
1396        /* Disable MCAM entries installed by PF with target as VF pcifunc */
1397        for (index = 0; index < mcam->bmap_entries; index++) {
1398                if (mcam->entry2target_pffunc[index] == target)
1399                        npc_enable_mcam_entry(rvu, mcam, blkaddr,
1400                                              index, false);
1401        }
1402        mutex_unlock(&mcam->lock);
1403}
1404