linux/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2019 Mellanox Technologies. */
   3
   4#include <net/netfilter/nf_conntrack.h>
   5#include <net/netfilter/nf_conntrack_core.h>
   6#include <net/netfilter/nf_conntrack_zones.h>
   7#include <net/netfilter/nf_conntrack_labels.h>
   8#include <net/netfilter/nf_conntrack_helper.h>
   9#include <net/netfilter/nf_conntrack_acct.h>
  10#include <uapi/linux/tc_act/tc_pedit.h>
  11#include <net/tc_act/tc_ct.h>
  12#include <net/flow_offload.h>
  13#include <net/netfilter/nf_flow_table.h>
  14#include <linux/workqueue.h>
  15#include <linux/refcount.h>
  16#include <linux/xarray.h>
  17
  18#include "lib/fs_chains.h"
  19#include "en/tc_ct.h"
  20#include "en/mod_hdr.h"
  21#include "en/mapping.h"
  22#include "en.h"
  23#include "en_tc.h"
  24#include "en_rep.h"
  25
  26#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen)
  27#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0)
  28#define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1)
  29#define MLX5_CT_STATE_TRK_BIT BIT(2)
  30#define MLX5_CT_STATE_NAT_BIT BIT(3)
  31#define MLX5_CT_STATE_REPLY_BIT BIT(4)
  32#define MLX5_CT_STATE_RELATED_BIT BIT(5)
  33#define MLX5_CT_STATE_INVALID_BIT BIT(6)
  34
  35#define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen)
  36#define MLX5_FTE_ID_MAX GENMASK(MLX5_FTE_ID_BITS - 1, 0)
  37#define MLX5_FTE_ID_MASK MLX5_FTE_ID_MAX
  38
  39#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen)
  40#define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0)
  41
  42#define ct_dbg(fmt, args...)\
  43        netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args)
  44
  45struct mlx5_tc_ct_priv {
  46        struct mlx5_core_dev *dev;
  47        const struct net_device *netdev;
  48        struct mod_hdr_tbl *mod_hdr_tbl;
  49        struct idr fte_ids;
  50        struct xarray tuple_ids;
  51        struct rhashtable zone_ht;
  52        struct rhashtable ct_tuples_ht;
  53        struct rhashtable ct_tuples_nat_ht;
  54        struct mlx5_flow_table *ct;
  55        struct mlx5_flow_table *ct_nat;
  56        struct mlx5_flow_table *post_ct;
  57        struct mutex control_lock; /* guards parallel adds/dels */
  58        struct mapping_ctx *zone_mapping;
  59        struct mapping_ctx *labels_mapping;
  60        enum mlx5_flow_namespace_type ns_type;
  61        struct mlx5_fs_chains *chains;
  62        spinlock_t ht_lock; /* protects ft entries */
  63};
  64
  65struct mlx5_ct_flow {
  66        struct mlx5_flow_attr *pre_ct_attr;
  67        struct mlx5_flow_attr *post_ct_attr;
  68        struct mlx5_flow_handle *pre_ct_rule;
  69        struct mlx5_flow_handle *post_ct_rule;
  70        struct mlx5_ct_ft *ft;
  71        u32 fte_id;
  72        u32 chain_mapping;
  73};
  74
  75struct mlx5_ct_zone_rule {
  76        struct mlx5_flow_handle *rule;
  77        struct mlx5e_mod_hdr_handle *mh;
  78        struct mlx5_flow_attr *attr;
  79        bool nat;
  80};
  81
  82struct mlx5_tc_ct_pre {
  83        struct mlx5_flow_table *ft;
  84        struct mlx5_flow_group *flow_grp;
  85        struct mlx5_flow_group *miss_grp;
  86        struct mlx5_flow_handle *flow_rule;
  87        struct mlx5_flow_handle *miss_rule;
  88        struct mlx5_modify_hdr *modify_hdr;
  89};
  90
  91struct mlx5_ct_ft {
  92        struct rhash_head node;
  93        u16 zone;
  94        u32 zone_restore_id;
  95        refcount_t refcount;
  96        struct nf_flowtable *nf_ft;
  97        struct mlx5_tc_ct_priv *ct_priv;
  98        struct rhashtable ct_entries_ht;
  99        struct mlx5_tc_ct_pre pre_ct;
 100        struct mlx5_tc_ct_pre pre_ct_nat;
 101};
 102
 103struct mlx5_ct_tuple {
 104        u16 addr_type;
 105        __be16 n_proto;
 106        u8 ip_proto;
 107        struct {
 108                union {
 109                        __be32 src_v4;
 110                        struct in6_addr src_v6;
 111                };
 112                union {
 113                        __be32 dst_v4;
 114                        struct in6_addr dst_v6;
 115                };
 116        } ip;
 117        struct {
 118                __be16 src;
 119                __be16 dst;
 120        } port;
 121
 122        u16 zone;
 123};
 124
 125struct mlx5_ct_counter {
 126        struct mlx5_fc *counter;
 127        refcount_t refcount;
 128        bool is_shared;
 129};
 130
 131enum {
 132        MLX5_CT_ENTRY_FLAG_VALID,
 133};
 134
 135struct mlx5_ct_entry {
 136        struct rhash_head node;
 137        struct rhash_head tuple_node;
 138        struct rhash_head tuple_nat_node;
 139        struct mlx5_ct_counter *counter;
 140        unsigned long cookie;
 141        unsigned long restore_cookie;
 142        struct mlx5_ct_tuple tuple;
 143        struct mlx5_ct_tuple tuple_nat;
 144        struct mlx5_ct_zone_rule zone_rules[2];
 145
 146        struct mlx5_tc_ct_priv *ct_priv;
 147        struct work_struct work;
 148
 149        refcount_t refcnt;
 150        unsigned long flags;
 151};
 152
 153static void
 154mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 155                                 struct mlx5_flow_attr *attr,
 156                                 struct mlx5e_mod_hdr_handle *mh);
 157
 158static const struct rhashtable_params cts_ht_params = {
 159        .head_offset = offsetof(struct mlx5_ct_entry, node),
 160        .key_offset = offsetof(struct mlx5_ct_entry, cookie),
 161        .key_len = sizeof(((struct mlx5_ct_entry *)0)->cookie),
 162        .automatic_shrinking = true,
 163        .min_size = 16 * 1024,
 164};
 165
 166static const struct rhashtable_params zone_params = {
 167        .head_offset = offsetof(struct mlx5_ct_ft, node),
 168        .key_offset = offsetof(struct mlx5_ct_ft, zone),
 169        .key_len = sizeof(((struct mlx5_ct_ft *)0)->zone),
 170        .automatic_shrinking = true,
 171};
 172
 173static const struct rhashtable_params tuples_ht_params = {
 174        .head_offset = offsetof(struct mlx5_ct_entry, tuple_node),
 175        .key_offset = offsetof(struct mlx5_ct_entry, tuple),
 176        .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple),
 177        .automatic_shrinking = true,
 178        .min_size = 16 * 1024,
 179};
 180
 181static const struct rhashtable_params tuples_nat_ht_params = {
 182        .head_offset = offsetof(struct mlx5_ct_entry, tuple_nat_node),
 183        .key_offset = offsetof(struct mlx5_ct_entry, tuple_nat),
 184        .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple_nat),
 185        .automatic_shrinking = true,
 186        .min_size = 16 * 1024,
 187};
 188
 189static bool
 190mlx5_tc_ct_entry_has_nat(struct mlx5_ct_entry *entry)
 191{
 192        return !!(entry->tuple_nat_node.next);
 193}
 194
 195static int
 196mlx5_get_label_mapping(struct mlx5_tc_ct_priv *ct_priv,
 197                       u32 *labels, u32 *id)
 198{
 199        if (!memchr_inv(labels, 0, sizeof(u32) * 4)) {
 200                *id = 0;
 201                return 0;
 202        }
 203
 204        if (mapping_add(ct_priv->labels_mapping, labels, id))
 205                return -EOPNOTSUPP;
 206
 207        return 0;
 208}
 209
 210static void
 211mlx5_put_label_mapping(struct mlx5_tc_ct_priv *ct_priv, u32 id)
 212{
 213        if (id)
 214                mapping_remove(ct_priv->labels_mapping, id);
 215}
 216
 217static int
 218mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple *tuple, struct flow_rule *rule)
 219{
 220        struct flow_match_control control;
 221        struct flow_match_basic basic;
 222
 223        flow_rule_match_basic(rule, &basic);
 224        flow_rule_match_control(rule, &control);
 225
 226        tuple->n_proto = basic.key->n_proto;
 227        tuple->ip_proto = basic.key->ip_proto;
 228        tuple->addr_type = control.key->addr_type;
 229
 230        if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 231                struct flow_match_ipv4_addrs match;
 232
 233                flow_rule_match_ipv4_addrs(rule, &match);
 234                tuple->ip.src_v4 = match.key->src;
 235                tuple->ip.dst_v4 = match.key->dst;
 236        } else if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 237                struct flow_match_ipv6_addrs match;
 238
 239                flow_rule_match_ipv6_addrs(rule, &match);
 240                tuple->ip.src_v6 = match.key->src;
 241                tuple->ip.dst_v6 = match.key->dst;
 242        } else {
 243                return -EOPNOTSUPP;
 244        }
 245
 246        if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 247                struct flow_match_ports match;
 248
 249                flow_rule_match_ports(rule, &match);
 250                switch (tuple->ip_proto) {
 251                case IPPROTO_TCP:
 252                case IPPROTO_UDP:
 253                        tuple->port.src = match.key->src;
 254                        tuple->port.dst = match.key->dst;
 255                        break;
 256                default:
 257                        return -EOPNOTSUPP;
 258                }
 259        } else {
 260                return -EOPNOTSUPP;
 261        }
 262
 263        return 0;
 264}
 265
 266static int
 267mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple *tuple,
 268                             struct flow_rule *rule)
 269{
 270        struct flow_action *flow_action = &rule->action;
 271        struct flow_action_entry *act;
 272        u32 offset, val, ip6_offset;
 273        int i;
 274
 275        flow_action_for_each(i, act, flow_action) {
 276                if (act->id != FLOW_ACTION_MANGLE)
 277                        continue;
 278
 279                offset = act->mangle.offset;
 280                val = act->mangle.val;
 281                switch (act->mangle.htype) {
 282                case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 283                        if (offset == offsetof(struct iphdr, saddr))
 284                                tuple->ip.src_v4 = cpu_to_be32(val);
 285                        else if (offset == offsetof(struct iphdr, daddr))
 286                                tuple->ip.dst_v4 = cpu_to_be32(val);
 287                        else
 288                                return -EOPNOTSUPP;
 289                        break;
 290
 291                case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 292                        ip6_offset = (offset - offsetof(struct ipv6hdr, saddr));
 293                        ip6_offset /= 4;
 294                        if (ip6_offset < 4)
 295                                tuple->ip.src_v6.s6_addr32[ip6_offset] = cpu_to_be32(val);
 296                        else if (ip6_offset < 8)
 297                                tuple->ip.dst_v6.s6_addr32[ip6_offset - 4] = cpu_to_be32(val);
 298                        else
 299                                return -EOPNOTSUPP;
 300                        break;
 301
 302                case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 303                        if (offset == offsetof(struct tcphdr, source))
 304                                tuple->port.src = cpu_to_be16(val);
 305                        else if (offset == offsetof(struct tcphdr, dest))
 306                                tuple->port.dst = cpu_to_be16(val);
 307                        else
 308                                return -EOPNOTSUPP;
 309                        break;
 310
 311                case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 312                        if (offset == offsetof(struct udphdr, source))
 313                                tuple->port.src = cpu_to_be16(val);
 314                        else if (offset == offsetof(struct udphdr, dest))
 315                                tuple->port.dst = cpu_to_be16(val);
 316                        else
 317                                return -EOPNOTSUPP;
 318                        break;
 319
 320                default:
 321                        return -EOPNOTSUPP;
 322                }
 323        }
 324
 325        return 0;
 326}
 327
 328static int
 329mlx5_tc_ct_set_tuple_match(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec,
 330                           struct flow_rule *rule)
 331{
 332        void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 333                                       outer_headers);
 334        void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 335                                       outer_headers);
 336        u16 addr_type = 0;
 337        u8 ip_proto = 0;
 338
 339        if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
 340                struct flow_match_basic match;
 341
 342                flow_rule_match_basic(rule, &match);
 343
 344                mlx5e_tc_set_ethertype(priv->mdev, &match, true, headers_c,
 345                                       headers_v);
 346                MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 347                         match.mask->ip_proto);
 348                MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 349                         match.key->ip_proto);
 350
 351                ip_proto = match.key->ip_proto;
 352        }
 353
 354        if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
 355                struct flow_match_control match;
 356
 357                flow_rule_match_control(rule, &match);
 358                addr_type = match.key->addr_type;
 359        }
 360
 361        if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 362                struct flow_match_ipv4_addrs match;
 363
 364                flow_rule_match_ipv4_addrs(rule, &match);
 365                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 366                                    src_ipv4_src_ipv6.ipv4_layout.ipv4),
 367                       &match.mask->src, sizeof(match.mask->src));
 368                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 369                                    src_ipv4_src_ipv6.ipv4_layout.ipv4),
 370                       &match.key->src, sizeof(match.key->src));
 371                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 372                                    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 373                       &match.mask->dst, sizeof(match.mask->dst));
 374                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 375                                    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 376                       &match.key->dst, sizeof(match.key->dst));
 377        }
 378
 379        if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 380                struct flow_match_ipv6_addrs match;
 381
 382                flow_rule_match_ipv6_addrs(rule, &match);
 383                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 384                                    src_ipv4_src_ipv6.ipv6_layout.ipv6),
 385                       &match.mask->src, sizeof(match.mask->src));
 386                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 387                                    src_ipv4_src_ipv6.ipv6_layout.ipv6),
 388                       &match.key->src, sizeof(match.key->src));
 389
 390                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 391                                    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 392                       &match.mask->dst, sizeof(match.mask->dst));
 393                memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 394                                    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 395                       &match.key->dst, sizeof(match.key->dst));
 396        }
 397
 398        if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 399                struct flow_match_ports match;
 400
 401                flow_rule_match_ports(rule, &match);
 402                switch (ip_proto) {
 403                case IPPROTO_TCP:
 404                        MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 405                                 tcp_sport, ntohs(match.mask->src));
 406                        MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 407                                 tcp_sport, ntohs(match.key->src));
 408
 409                        MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 410                                 tcp_dport, ntohs(match.mask->dst));
 411                        MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 412                                 tcp_dport, ntohs(match.key->dst));
 413                        break;
 414
 415                case IPPROTO_UDP:
 416                        MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 417                                 udp_sport, ntohs(match.mask->src));
 418                        MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 419                                 udp_sport, ntohs(match.key->src));
 420
 421                        MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 422                                 udp_dport, ntohs(match.mask->dst));
 423                        MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 424                                 udp_dport, ntohs(match.key->dst));
 425                        break;
 426                default:
 427                        break;
 428                }
 429        }
 430
 431        if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
 432                struct flow_match_tcp match;
 433
 434                flow_rule_match_tcp(rule, &match);
 435                MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
 436                         ntohs(match.mask->flags));
 437                MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
 438                         ntohs(match.key->flags));
 439        }
 440
 441        return 0;
 442}
 443
 444static void
 445mlx5_tc_ct_counter_put(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry)
 446{
 447        if (entry->counter->is_shared &&
 448            !refcount_dec_and_test(&entry->counter->refcount))
 449                return;
 450
 451        mlx5_fc_destroy(ct_priv->dev, entry->counter->counter);
 452        kfree(entry->counter);
 453}
 454
 455static void
 456mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv,
 457                          struct mlx5_ct_entry *entry,
 458                          bool nat)
 459{
 460        struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
 461        struct mlx5_flow_attr *attr = zone_rule->attr;
 462
 463        ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone);
 464
 465        mlx5_tc_rule_delete(netdev_priv(ct_priv->netdev), zone_rule->rule, attr);
 466        mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
 467        mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
 468        kfree(attr);
 469}
 470
 471static void
 472mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv,
 473                           struct mlx5_ct_entry *entry)
 474{
 475        mlx5_tc_ct_entry_del_rule(ct_priv, entry, true);
 476        mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
 477}
 478
 479static struct flow_action_entry *
 480mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule)
 481{
 482        struct flow_action *flow_action = &flow_rule->action;
 483        struct flow_action_entry *act;
 484        int i;
 485
 486        flow_action_for_each(i, act, flow_action) {
 487                if (act->id == FLOW_ACTION_CT_METADATA)
 488                        return act;
 489        }
 490
 491        return NULL;
 492}
 493
 494static int
 495mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv,
 496                               struct mlx5e_tc_mod_hdr_acts *mod_acts,
 497                               u8 ct_state,
 498                               u32 mark,
 499                               u32 labels_id,
 500                               u8 zone_restore_id)
 501{
 502        enum mlx5_flow_namespace_type ns = ct_priv->ns_type;
 503        struct mlx5_core_dev *dev = ct_priv->dev;
 504        int err;
 505
 506        err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 507                                        CTSTATE_TO_REG, ct_state);
 508        if (err)
 509                return err;
 510
 511        err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 512                                        MARK_TO_REG, mark);
 513        if (err)
 514                return err;
 515
 516        err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 517                                        LABELS_TO_REG, labels_id);
 518        if (err)
 519                return err;
 520
 521        err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 522                                        ZONE_RESTORE_TO_REG, zone_restore_id);
 523        if (err)
 524                return err;
 525
 526        /* Make another copy of zone id in reg_b for
 527         * NIC rx flows since we don't copy reg_c1 to
 528         * reg_b upon miss.
 529         */
 530        if (ns != MLX5_FLOW_NAMESPACE_FDB) {
 531                err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 532                                                NIC_ZONE_RESTORE_TO_REG, zone_restore_id);
 533                if (err)
 534                        return err;
 535        }
 536        return 0;
 537}
 538
 539static int
 540mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
 541                                   char *modact)
 542{
 543        u32 offset = act->mangle.offset, field;
 544
 545        switch (act->mangle.htype) {
 546        case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 547                MLX5_SET(set_action_in, modact, length, 0);
 548                if (offset == offsetof(struct iphdr, saddr))
 549                        field = MLX5_ACTION_IN_FIELD_OUT_SIPV4;
 550                else if (offset == offsetof(struct iphdr, daddr))
 551                        field = MLX5_ACTION_IN_FIELD_OUT_DIPV4;
 552                else
 553                        return -EOPNOTSUPP;
 554                break;
 555
 556        case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 557                MLX5_SET(set_action_in, modact, length, 0);
 558                if (offset == offsetof(struct ipv6hdr, saddr) + 12)
 559                        field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0;
 560                else if (offset == offsetof(struct ipv6hdr, saddr) + 8)
 561                        field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
 562                else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
 563                        field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64;
 564                else if (offset == offsetof(struct ipv6hdr, saddr))
 565                        field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96;
 566                else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
 567                        field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0;
 568                else if (offset == offsetof(struct ipv6hdr, daddr) + 8)
 569                        field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
 570                else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
 571                        field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64;
 572                else if (offset == offsetof(struct ipv6hdr, daddr))
 573                        field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96;
 574                else
 575                        return -EOPNOTSUPP;
 576                break;
 577
 578        case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 579                MLX5_SET(set_action_in, modact, length, 16);
 580                if (offset == offsetof(struct tcphdr, source))
 581                        field = MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT;
 582                else if (offset == offsetof(struct tcphdr, dest))
 583                        field = MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT;
 584                else
 585                        return -EOPNOTSUPP;
 586                break;
 587
 588        case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 589                MLX5_SET(set_action_in, modact, length, 16);
 590                if (offset == offsetof(struct udphdr, source))
 591                        field = MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT;
 592                else if (offset == offsetof(struct udphdr, dest))
 593                        field = MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT;
 594                else
 595                        return -EOPNOTSUPP;
 596                break;
 597
 598        default:
 599                return -EOPNOTSUPP;
 600        }
 601
 602        MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
 603        MLX5_SET(set_action_in, modact, offset, 0);
 604        MLX5_SET(set_action_in, modact, field, field);
 605        MLX5_SET(set_action_in, modact, data, act->mangle.val);
 606
 607        return 0;
 608}
 609
 610static int
 611mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv,
 612                            struct flow_rule *flow_rule,
 613                            struct mlx5e_tc_mod_hdr_acts *mod_acts)
 614{
 615        struct flow_action *flow_action = &flow_rule->action;
 616        struct mlx5_core_dev *mdev = ct_priv->dev;
 617        struct flow_action_entry *act;
 618        size_t action_size;
 619        char *modact;
 620        int err, i;
 621
 622        action_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto);
 623
 624        flow_action_for_each(i, act, flow_action) {
 625                switch (act->id) {
 626                case FLOW_ACTION_MANGLE: {
 627                        err = alloc_mod_hdr_actions(mdev, ct_priv->ns_type,
 628                                                    mod_acts);
 629                        if (err)
 630                                return err;
 631
 632                        modact = mod_acts->actions +
 633                                 mod_acts->num_actions * action_size;
 634
 635                        err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact);
 636                        if (err)
 637                                return err;
 638
 639                        mod_acts->num_actions++;
 640                }
 641                break;
 642
 643                case FLOW_ACTION_CT_METADATA:
 644                        /* Handled earlier */
 645                        continue;
 646                default:
 647                        return -EOPNOTSUPP;
 648                }
 649        }
 650
 651        return 0;
 652}
 653
 654static int
 655mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 656                                struct mlx5_flow_attr *attr,
 657                                struct flow_rule *flow_rule,
 658                                struct mlx5e_mod_hdr_handle **mh,
 659                                u8 zone_restore_id, bool nat)
 660{
 661        struct mlx5e_tc_mod_hdr_acts mod_acts = {};
 662        struct flow_action_entry *meta;
 663        u16 ct_state = 0;
 664        int err;
 665
 666        meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
 667        if (!meta)
 668                return -EOPNOTSUPP;
 669
 670        err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels,
 671                                     &attr->ct_attr.ct_labels_id);
 672        if (err)
 673                return -EOPNOTSUPP;
 674        if (nat) {
 675                err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule,
 676                                                  &mod_acts);
 677                if (err)
 678                        goto err_mapping;
 679
 680                ct_state |= MLX5_CT_STATE_NAT_BIT;
 681        }
 682
 683        ct_state |= MLX5_CT_STATE_ESTABLISHED_BIT | MLX5_CT_STATE_TRK_BIT;
 684        ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT;
 685        err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,
 686                                             ct_state,
 687                                             meta->ct_metadata.mark,
 688                                             attr->ct_attr.ct_labels_id,
 689                                             zone_restore_id);
 690        if (err)
 691                goto err_mapping;
 692
 693        if (nat) {
 694                attr->modify_hdr = mlx5_modify_header_alloc(ct_priv->dev, ct_priv->ns_type,
 695                                                            mod_acts.num_actions,
 696                                                            mod_acts.actions);
 697                if (IS_ERR(attr->modify_hdr)) {
 698                        err = PTR_ERR(attr->modify_hdr);
 699                        goto err_mapping;
 700                }
 701
 702                *mh = NULL;
 703        } else {
 704                *mh = mlx5e_mod_hdr_attach(ct_priv->dev,
 705                                           ct_priv->mod_hdr_tbl,
 706                                           ct_priv->ns_type,
 707                                           &mod_acts);
 708                if (IS_ERR(*mh)) {
 709                        err = PTR_ERR(*mh);
 710                        goto err_mapping;
 711                }
 712                attr->modify_hdr = mlx5e_mod_hdr_get(*mh);
 713        }
 714
 715        dealloc_mod_hdr_actions(&mod_acts);
 716        return 0;
 717
 718err_mapping:
 719        dealloc_mod_hdr_actions(&mod_acts);
 720        mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
 721        return err;
 722}
 723
 724static void
 725mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 726                                 struct mlx5_flow_attr *attr,
 727                                 struct mlx5e_mod_hdr_handle *mh)
 728{
 729        if (mh)
 730                mlx5e_mod_hdr_detach(ct_priv->dev, ct_priv->mod_hdr_tbl, mh);
 731        else
 732                mlx5_modify_header_dealloc(ct_priv->dev, attr->modify_hdr);
 733}
 734
 735static int
 736mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
 737                          struct flow_rule *flow_rule,
 738                          struct mlx5_ct_entry *entry,
 739                          bool nat, u8 zone_restore_id)
 740{
 741        struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
 742        struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
 743        struct mlx5_flow_spec *spec = NULL;
 744        struct mlx5_flow_attr *attr;
 745        int err;
 746
 747        zone_rule->nat = nat;
 748
 749        spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 750        if (!spec)
 751                return -ENOMEM;
 752
 753        attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
 754        if (!attr) {
 755                err = -ENOMEM;
 756                goto err_attr;
 757        }
 758
 759        err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule,
 760                                              &zone_rule->mh,
 761                                              zone_restore_id, nat);
 762        if (err) {
 763                ct_dbg("Failed to create ct entry mod hdr");
 764                goto err_mod_hdr;
 765        }
 766
 767        attr->action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
 768                       MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 769                       MLX5_FLOW_CONTEXT_ACTION_COUNT;
 770        attr->dest_chain = 0;
 771        attr->dest_ft = ct_priv->post_ct;
 772        attr->ft = nat ? ct_priv->ct_nat : ct_priv->ct;
 773        attr->outer_match_level = MLX5_MATCH_L4;
 774        attr->counter = entry->counter->counter;
 775        attr->flags |= MLX5_ESW_ATTR_FLAG_NO_IN_PORT;
 776        if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB)
 777                attr->esw_attr->in_mdev = priv->mdev;
 778
 779        mlx5_tc_ct_set_tuple_match(netdev_priv(ct_priv->netdev), spec, flow_rule);
 780        mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK);
 781
 782        zone_rule->rule = mlx5_tc_rule_insert(priv, spec, attr);
 783        if (IS_ERR(zone_rule->rule)) {
 784                err = PTR_ERR(zone_rule->rule);
 785                ct_dbg("Failed to add ct entry rule, nat: %d", nat);
 786                goto err_rule;
 787        }
 788
 789        zone_rule->attr = attr;
 790
 791        kvfree(spec);
 792        ct_dbg("Offloaded ct entry rule in zone %d", entry->tuple.zone);
 793
 794        return 0;
 795
 796err_rule:
 797        mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
 798        mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
 799err_mod_hdr:
 800        kfree(attr);
 801err_attr:
 802        kvfree(spec);
 803        return err;
 804}
 805
 806static bool
 807mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry)
 808{
 809        return test_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags);
 810}
 811
 812static struct mlx5_ct_entry *
 813mlx5_tc_ct_entry_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_tuple *tuple)
 814{
 815        struct mlx5_ct_entry *entry;
 816
 817        entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_ht, tuple,
 818                                       tuples_ht_params);
 819        if (entry && mlx5_tc_ct_entry_valid(entry) &&
 820            refcount_inc_not_zero(&entry->refcnt)) {
 821                return entry;
 822        } else if (!entry) {
 823                entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_nat_ht,
 824                                               tuple, tuples_nat_ht_params);
 825                if (entry && mlx5_tc_ct_entry_valid(entry) &&
 826                    refcount_inc_not_zero(&entry->refcnt))
 827                        return entry;
 828        }
 829
 830        return entry ? ERR_PTR(-EINVAL) : NULL;
 831}
 832
 833static void mlx5_tc_ct_entry_remove_from_tuples(struct mlx5_ct_entry *entry)
 834{
 835        struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv;
 836
 837        rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht,
 838                               &entry->tuple_nat_node,
 839                               tuples_nat_ht_params);
 840        rhashtable_remove_fast(&ct_priv->ct_tuples_ht, &entry->tuple_node,
 841                               tuples_ht_params);
 842}
 843
 844static void mlx5_tc_ct_entry_del(struct mlx5_ct_entry *entry)
 845{
 846        struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv;
 847
 848        mlx5_tc_ct_entry_del_rules(ct_priv, entry);
 849
 850        spin_lock_bh(&ct_priv->ht_lock);
 851        mlx5_tc_ct_entry_remove_from_tuples(entry);
 852        spin_unlock_bh(&ct_priv->ht_lock);
 853
 854        mlx5_tc_ct_counter_put(ct_priv, entry);
 855        kfree(entry);
 856}
 857
 858static void
 859mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry)
 860{
 861        if (!refcount_dec_and_test(&entry->refcnt))
 862                return;
 863
 864        mlx5_tc_ct_entry_del(entry);
 865}
 866
 867static void mlx5_tc_ct_entry_del_work(struct work_struct *work)
 868{
 869        struct mlx5_ct_entry *entry = container_of(work, struct mlx5_ct_entry, work);
 870
 871        mlx5_tc_ct_entry_del(entry);
 872}
 873
 874static void
 875__mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry)
 876{
 877        struct mlx5e_priv *priv;
 878
 879        if (!refcount_dec_and_test(&entry->refcnt))
 880                return;
 881
 882        priv = netdev_priv(entry->ct_priv->netdev);
 883        INIT_WORK(&entry->work, mlx5_tc_ct_entry_del_work);
 884        queue_work(priv->wq, &entry->work);
 885}
 886
 887static struct mlx5_ct_counter *
 888mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv *ct_priv)
 889{
 890        struct mlx5_ct_counter *counter;
 891        int ret;
 892
 893        counter = kzalloc(sizeof(*counter), GFP_KERNEL);
 894        if (!counter)
 895                return ERR_PTR(-ENOMEM);
 896
 897        counter->is_shared = false;
 898        counter->counter = mlx5_fc_create(ct_priv->dev, true);
 899        if (IS_ERR(counter->counter)) {
 900                ct_dbg("Failed to create counter for ct entry");
 901                ret = PTR_ERR(counter->counter);
 902                kfree(counter);
 903                return ERR_PTR(ret);
 904        }
 905
 906        return counter;
 907}
 908
 909static struct mlx5_ct_counter *
 910mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv,
 911                              struct mlx5_ct_entry *entry)
 912{
 913        struct mlx5_ct_tuple rev_tuple = entry->tuple;
 914        struct mlx5_ct_counter *shared_counter;
 915        struct mlx5_ct_entry *rev_entry;
 916        __be16 tmp_port;
 917
 918        /* get the reversed tuple */
 919        tmp_port = rev_tuple.port.src;
 920        rev_tuple.port.src = rev_tuple.port.dst;
 921        rev_tuple.port.dst = tmp_port;
 922
 923        if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 924                __be32 tmp_addr = rev_tuple.ip.src_v4;
 925
 926                rev_tuple.ip.src_v4 = rev_tuple.ip.dst_v4;
 927                rev_tuple.ip.dst_v4 = tmp_addr;
 928        } else if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 929                struct in6_addr tmp_addr = rev_tuple.ip.src_v6;
 930
 931                rev_tuple.ip.src_v6 = rev_tuple.ip.dst_v6;
 932                rev_tuple.ip.dst_v6 = tmp_addr;
 933        } else {
 934                return ERR_PTR(-EOPNOTSUPP);
 935        }
 936
 937        /* Use the same counter as the reverse direction */
 938        spin_lock_bh(&ct_priv->ht_lock);
 939        rev_entry = mlx5_tc_ct_entry_get(ct_priv, &rev_tuple);
 940
 941        if (IS_ERR(rev_entry)) {
 942                spin_unlock_bh(&ct_priv->ht_lock);
 943                goto create_counter;
 944        }
 945
 946        if (rev_entry && refcount_inc_not_zero(&rev_entry->counter->refcount)) {
 947                ct_dbg("Using shared counter entry=0x%p rev=0x%p", entry, rev_entry);
 948                shared_counter = rev_entry->counter;
 949                spin_unlock_bh(&ct_priv->ht_lock);
 950
 951                mlx5_tc_ct_entry_put(rev_entry);
 952                return shared_counter;
 953        }
 954
 955        spin_unlock_bh(&ct_priv->ht_lock);
 956
 957create_counter:
 958
 959        shared_counter = mlx5_tc_ct_counter_create(ct_priv);
 960        if (IS_ERR(shared_counter))
 961                return shared_counter;
 962
 963        shared_counter->is_shared = true;
 964        refcount_set(&shared_counter->refcount, 1);
 965        return shared_counter;
 966}
 967
 968static int
 969mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv,
 970                           struct flow_rule *flow_rule,
 971                           struct mlx5_ct_entry *entry,
 972                           u8 zone_restore_id)
 973{
 974        int err;
 975
 976        if (nf_ct_acct_enabled(dev_net(ct_priv->netdev)))
 977                entry->counter = mlx5_tc_ct_counter_create(ct_priv);
 978        else
 979                entry->counter = mlx5_tc_ct_shared_counter_get(ct_priv, entry);
 980
 981        if (IS_ERR(entry->counter)) {
 982                err = PTR_ERR(entry->counter);
 983                return err;
 984        }
 985
 986        err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false,
 987                                        zone_restore_id);
 988        if (err)
 989                goto err_orig;
 990
 991        err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true,
 992                                        zone_restore_id);
 993        if (err)
 994                goto err_nat;
 995
 996        return 0;
 997
 998err_nat:
 999        mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
1000err_orig:
1001        mlx5_tc_ct_counter_put(ct_priv, entry);
1002        return err;
1003}
1004
1005static int
1006mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
1007                                  struct flow_cls_offload *flow)
1008{
1009        struct flow_rule *flow_rule = flow_cls_offload_flow_rule(flow);
1010        struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1011        struct flow_action_entry *meta_action;
1012        unsigned long cookie = flow->cookie;
1013        struct mlx5_ct_entry *entry;
1014        int err;
1015
1016        meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
1017        if (!meta_action)
1018                return -EOPNOTSUPP;
1019
1020        spin_lock_bh(&ct_priv->ht_lock);
1021        entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1022        if (entry && refcount_inc_not_zero(&entry->refcnt)) {
1023                spin_unlock_bh(&ct_priv->ht_lock);
1024                mlx5_tc_ct_entry_put(entry);
1025                return -EEXIST;
1026        }
1027        spin_unlock_bh(&ct_priv->ht_lock);
1028
1029        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1030        if (!entry)
1031                return -ENOMEM;
1032
1033        entry->tuple.zone = ft->zone;
1034        entry->cookie = flow->cookie;
1035        entry->restore_cookie = meta_action->ct_metadata.cookie;
1036        refcount_set(&entry->refcnt, 2);
1037        entry->ct_priv = ct_priv;
1038
1039        err = mlx5_tc_ct_rule_to_tuple(&entry->tuple, flow_rule);
1040        if (err)
1041                goto err_set;
1042
1043        memcpy(&entry->tuple_nat, &entry->tuple, sizeof(entry->tuple));
1044        err = mlx5_tc_ct_rule_to_tuple_nat(&entry->tuple_nat, flow_rule);
1045        if (err)
1046                goto err_set;
1047
1048        spin_lock_bh(&ct_priv->ht_lock);
1049
1050        err = rhashtable_lookup_insert_fast(&ft->ct_entries_ht, &entry->node,
1051                                            cts_ht_params);
1052        if (err)
1053                goto err_entries;
1054
1055        err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_ht,
1056                                            &entry->tuple_node,
1057                                            tuples_ht_params);
1058        if (err)
1059                goto err_tuple;
1060
1061        if (memcmp(&entry->tuple, &entry->tuple_nat, sizeof(entry->tuple))) {
1062                err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_nat_ht,
1063                                                    &entry->tuple_nat_node,
1064                                                    tuples_nat_ht_params);
1065                if (err)
1066                        goto err_tuple_nat;
1067        }
1068        spin_unlock_bh(&ct_priv->ht_lock);
1069
1070        err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry,
1071                                         ft->zone_restore_id);
1072        if (err)
1073                goto err_rules;
1074
1075        set_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags);
1076        mlx5_tc_ct_entry_put(entry); /* this function reference */
1077
1078        return 0;
1079
1080err_rules:
1081        spin_lock_bh(&ct_priv->ht_lock);
1082        if (mlx5_tc_ct_entry_has_nat(entry))
1083                rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht,
1084                                       &entry->tuple_nat_node, tuples_nat_ht_params);
1085err_tuple_nat:
1086        rhashtable_remove_fast(&ct_priv->ct_tuples_ht,
1087                               &entry->tuple_node,
1088                               tuples_ht_params);
1089err_tuple:
1090        rhashtable_remove_fast(&ft->ct_entries_ht,
1091                               &entry->node,
1092                               cts_ht_params);
1093err_entries:
1094        spin_unlock_bh(&ct_priv->ht_lock);
1095err_set:
1096        kfree(entry);
1097        if (err != -EEXIST)
1098                netdev_warn(ct_priv->netdev, "Failed to offload ct entry, err: %d\n", err);
1099        return err;
1100}
1101
1102static int
1103mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft,
1104                                  struct flow_cls_offload *flow)
1105{
1106        struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1107        unsigned long cookie = flow->cookie;
1108        struct mlx5_ct_entry *entry;
1109
1110        spin_lock_bh(&ct_priv->ht_lock);
1111        entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1112        if (!entry) {
1113                spin_unlock_bh(&ct_priv->ht_lock);
1114                return -ENOENT;
1115        }
1116
1117        if (!mlx5_tc_ct_entry_valid(entry)) {
1118                spin_unlock_bh(&ct_priv->ht_lock);
1119                return -EINVAL;
1120        }
1121
1122        rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);
1123        mlx5_tc_ct_entry_remove_from_tuples(entry);
1124        spin_unlock_bh(&ct_priv->ht_lock);
1125
1126        mlx5_tc_ct_entry_put(entry);
1127
1128        return 0;
1129}
1130
1131static int
1132mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft,
1133                                    struct flow_cls_offload *f)
1134{
1135        struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1136        unsigned long cookie = f->cookie;
1137        struct mlx5_ct_entry *entry;
1138        u64 lastuse, packets, bytes;
1139
1140        spin_lock_bh(&ct_priv->ht_lock);
1141        entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1142        if (!entry) {
1143                spin_unlock_bh(&ct_priv->ht_lock);
1144                return -ENOENT;
1145        }
1146
1147        if (!mlx5_tc_ct_entry_valid(entry) || !refcount_inc_not_zero(&entry->refcnt)) {
1148                spin_unlock_bh(&ct_priv->ht_lock);
1149                return -EINVAL;
1150        }
1151
1152        spin_unlock_bh(&ct_priv->ht_lock);
1153
1154        mlx5_fc_query_cached(entry->counter->counter, &bytes, &packets, &lastuse);
1155        flow_stats_update(&f->stats, bytes, packets, 0, lastuse,
1156                          FLOW_ACTION_HW_STATS_DELAYED);
1157
1158        mlx5_tc_ct_entry_put(entry);
1159        return 0;
1160}
1161
1162static int
1163mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data,
1164                              void *cb_priv)
1165{
1166        struct flow_cls_offload *f = type_data;
1167        struct mlx5_ct_ft *ft = cb_priv;
1168
1169        if (type != TC_SETUP_CLSFLOWER)
1170                return -EOPNOTSUPP;
1171
1172        switch (f->command) {
1173        case FLOW_CLS_REPLACE:
1174                return mlx5_tc_ct_block_flow_offload_add(ft, f);
1175        case FLOW_CLS_DESTROY:
1176                return mlx5_tc_ct_block_flow_offload_del(ft, f);
1177        case FLOW_CLS_STATS:
1178                return mlx5_tc_ct_block_flow_offload_stats(ft, f);
1179        default:
1180                break;
1181        }
1182
1183        return -EOPNOTSUPP;
1184}
1185
1186static bool
1187mlx5_tc_ct_skb_to_tuple(struct sk_buff *skb, struct mlx5_ct_tuple *tuple,
1188                        u16 zone)
1189{
1190        struct flow_keys flow_keys;
1191
1192        skb_reset_network_header(skb);
1193        skb_flow_dissect_flow_keys(skb, &flow_keys, 0);
1194
1195        tuple->zone = zone;
1196
1197        if (flow_keys.basic.ip_proto != IPPROTO_TCP &&
1198            flow_keys.basic.ip_proto != IPPROTO_UDP)
1199                return false;
1200
1201        tuple->port.src = flow_keys.ports.src;
1202        tuple->port.dst = flow_keys.ports.dst;
1203        tuple->n_proto = flow_keys.basic.n_proto;
1204        tuple->ip_proto = flow_keys.basic.ip_proto;
1205
1206        switch (flow_keys.basic.n_proto) {
1207        case htons(ETH_P_IP):
1208                tuple->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1209                tuple->ip.src_v4 = flow_keys.addrs.v4addrs.src;
1210                tuple->ip.dst_v4 = flow_keys.addrs.v4addrs.dst;
1211                break;
1212
1213        case htons(ETH_P_IPV6):
1214                tuple->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1215                tuple->ip.src_v6 = flow_keys.addrs.v6addrs.src;
1216                tuple->ip.dst_v6 = flow_keys.addrs.v6addrs.dst;
1217                break;
1218        default:
1219                goto out;
1220        }
1221
1222        return true;
1223
1224out:
1225        return false;
1226}
1227
1228int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec)
1229{
1230        u32 ctstate = 0, ctstate_mask = 0;
1231
1232        mlx5e_tc_match_to_reg_get_match(spec, CTSTATE_TO_REG,
1233                                        &ctstate, &ctstate_mask);
1234
1235        if ((ctstate & ctstate_mask) == MLX5_CT_STATE_TRK_BIT)
1236                return -EOPNOTSUPP;
1237
1238        ctstate_mask |= MLX5_CT_STATE_TRK_BIT;
1239        mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
1240                                    ctstate, ctstate_mask);
1241
1242        return 0;
1243}
1244
1245void mlx5_tc_ct_match_del(struct mlx5_tc_ct_priv *priv, struct mlx5_ct_attr *ct_attr)
1246{
1247        if (!priv || !ct_attr->ct_labels_id)
1248                return;
1249
1250        mlx5_put_label_mapping(priv, ct_attr->ct_labels_id);
1251}
1252
1253int
1254mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
1255                     struct mlx5_flow_spec *spec,
1256                     struct flow_cls_offload *f,
1257                     struct mlx5_ct_attr *ct_attr,
1258                     struct netlink_ext_ack *extack)
1259{
1260        bool trk, est, untrk, unest, new, rpl, unrpl, rel, unrel, inv, uninv;
1261        struct flow_rule *rule = flow_cls_offload_flow_rule(f);
1262        struct flow_dissector_key_ct *mask, *key;
1263        u32 ctstate = 0, ctstate_mask = 0;
1264        u16 ct_state_on, ct_state_off;
1265        u16 ct_state, ct_state_mask;
1266        struct flow_match_ct match;
1267        u32 ct_labels[4];
1268
1269        if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT))
1270                return 0;
1271
1272        if (!priv) {
1273                NL_SET_ERR_MSG_MOD(extack,
1274                                   "offload of ct matching isn't available");
1275                return -EOPNOTSUPP;
1276        }
1277
1278        flow_rule_match_ct(rule, &match);
1279
1280        key = match.key;
1281        mask = match.mask;
1282
1283        ct_state = key->ct_state;
1284        ct_state_mask = mask->ct_state;
1285
1286        if (ct_state_mask & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
1287                              TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED |
1288                              TCA_FLOWER_KEY_CT_FLAGS_NEW |
1289                              TCA_FLOWER_KEY_CT_FLAGS_REPLY |
1290                              TCA_FLOWER_KEY_CT_FLAGS_RELATED |
1291                              TCA_FLOWER_KEY_CT_FLAGS_INVALID)) {
1292                NL_SET_ERR_MSG_MOD(extack,
1293                                   "only ct_state trk, est, new and rpl are supported for offload");
1294                return -EOPNOTSUPP;
1295        }
1296
1297        ct_state_on = ct_state & ct_state_mask;
1298        ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask;
1299        trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
1300        new = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_NEW;
1301        est = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
1302        rpl = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
1303        rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
1304        inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
1305        untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
1306        unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
1307        unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
1308        unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
1309        uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
1310
1311        ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;
1312        ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
1313        ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0;
1314        ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;
1315        ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
1316        ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0;
1317        ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0;
1318        ctstate_mask |= uninv ? MLX5_CT_STATE_INVALID_BIT : 0;
1319
1320        if (rel) {
1321                NL_SET_ERR_MSG_MOD(extack,
1322                                   "matching on ct_state +rel isn't supported");
1323                return -EOPNOTSUPP;
1324        }
1325
1326        if (inv) {
1327                NL_SET_ERR_MSG_MOD(extack,
1328                                   "matching on ct_state +inv isn't supported");
1329                return -EOPNOTSUPP;
1330        }
1331
1332        if (new) {
1333                NL_SET_ERR_MSG_MOD(extack,
1334                                   "matching on ct_state +new isn't supported");
1335                return -EOPNOTSUPP;
1336        }
1337
1338        if (mask->ct_zone)
1339                mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
1340                                            key->ct_zone, MLX5_CT_ZONE_MASK);
1341        if (ctstate_mask)
1342                mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
1343                                            ctstate, ctstate_mask);
1344        if (mask->ct_mark)
1345                mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG,
1346                                            key->ct_mark, mask->ct_mark);
1347        if (mask->ct_labels[0] || mask->ct_labels[1] || mask->ct_labels[2] ||
1348            mask->ct_labels[3]) {
1349                ct_labels[0] = key->ct_labels[0] & mask->ct_labels[0];
1350                ct_labels[1] = key->ct_labels[1] & mask->ct_labels[1];
1351                ct_labels[2] = key->ct_labels[2] & mask->ct_labels[2];
1352                ct_labels[3] = key->ct_labels[3] & mask->ct_labels[3];
1353                if (mlx5_get_label_mapping(priv, ct_labels, &ct_attr->ct_labels_id))
1354                        return -EOPNOTSUPP;
1355                mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG, ct_attr->ct_labels_id,
1356                                            MLX5_CT_LABELS_MASK);
1357        }
1358
1359        return 0;
1360}
1361
1362int
1363mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
1364                        struct mlx5_flow_attr *attr,
1365                        const struct flow_action_entry *act,
1366                        struct netlink_ext_ack *extack)
1367{
1368        if (!priv) {
1369                NL_SET_ERR_MSG_MOD(extack,
1370                                   "offload of ct action isn't available");
1371                return -EOPNOTSUPP;
1372        }
1373
1374        attr->ct_attr.zone = act->ct.zone;
1375        attr->ct_attr.ct_action = act->ct.action;
1376        attr->ct_attr.nf_ft = act->ct.flow_table;
1377
1378        return 0;
1379}
1380
1381static int tc_ct_pre_ct_add_rules(struct mlx5_ct_ft *ct_ft,
1382                                  struct mlx5_tc_ct_pre *pre_ct,
1383                                  bool nat)
1384{
1385        struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1386        struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {};
1387        struct mlx5_core_dev *dev = ct_priv->dev;
1388        struct mlx5_flow_table *ft = pre_ct->ft;
1389        struct mlx5_flow_destination dest = {};
1390        struct mlx5_flow_act flow_act = {};
1391        struct mlx5_modify_hdr *mod_hdr;
1392        struct mlx5_flow_handle *rule;
1393        struct mlx5_flow_spec *spec;
1394        u32 ctstate;
1395        u16 zone;
1396        int err;
1397
1398        spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1399        if (!spec)
1400                return -ENOMEM;
1401
1402        zone = ct_ft->zone & MLX5_CT_ZONE_MASK;
1403        err = mlx5e_tc_match_to_reg_set(dev, &pre_mod_acts, ct_priv->ns_type,
1404                                        ZONE_TO_REG, zone);
1405        if (err) {
1406                ct_dbg("Failed to set zone register mapping");
1407                goto err_mapping;
1408        }
1409
1410        mod_hdr = mlx5_modify_header_alloc(dev, ct_priv->ns_type,
1411                                           pre_mod_acts.num_actions,
1412                                           pre_mod_acts.actions);
1413
1414        if (IS_ERR(mod_hdr)) {
1415                err = PTR_ERR(mod_hdr);
1416                ct_dbg("Failed to create pre ct mod hdr");
1417                goto err_mapping;
1418        }
1419        pre_ct->modify_hdr = mod_hdr;
1420
1421        flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1422                          MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1423        flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
1424        flow_act.modify_hdr = mod_hdr;
1425        dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1426
1427        /* add flow rule */
1428        mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
1429                                    zone, MLX5_CT_ZONE_MASK);
1430        ctstate = MLX5_CT_STATE_TRK_BIT;
1431        if (nat)
1432                ctstate |= MLX5_CT_STATE_NAT_BIT;
1433        mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, ctstate, ctstate);
1434
1435        dest.ft = ct_priv->post_ct;
1436        rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1437        if (IS_ERR(rule)) {
1438                err = PTR_ERR(rule);
1439                ct_dbg("Failed to add pre ct flow rule zone %d", zone);
1440                goto err_flow_rule;
1441        }
1442        pre_ct->flow_rule = rule;
1443
1444        /* add miss rule */
1445        dest.ft = nat ? ct_priv->ct_nat : ct_priv->ct;
1446        rule = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1);
1447        if (IS_ERR(rule)) {
1448                err = PTR_ERR(rule);
1449                ct_dbg("Failed to add pre ct miss rule zone %d", zone);
1450                goto err_miss_rule;
1451        }
1452        pre_ct->miss_rule = rule;
1453
1454        dealloc_mod_hdr_actions(&pre_mod_acts);
1455        kvfree(spec);
1456        return 0;
1457
1458err_miss_rule:
1459        mlx5_del_flow_rules(pre_ct->flow_rule);
1460err_flow_rule:
1461        mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr);
1462err_mapping:
1463        dealloc_mod_hdr_actions(&pre_mod_acts);
1464        kvfree(spec);
1465        return err;
1466}
1467
1468static void
1469tc_ct_pre_ct_del_rules(struct mlx5_ct_ft *ct_ft,
1470                       struct mlx5_tc_ct_pre *pre_ct)
1471{
1472        struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1473        struct mlx5_core_dev *dev = ct_priv->dev;
1474
1475        mlx5_del_flow_rules(pre_ct->flow_rule);
1476        mlx5_del_flow_rules(pre_ct->miss_rule);
1477        mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr);
1478}
1479
1480static int
1481mlx5_tc_ct_alloc_pre_ct(struct mlx5_ct_ft *ct_ft,
1482                        struct mlx5_tc_ct_pre *pre_ct,
1483                        bool nat)
1484{
1485        int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1486        struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1487        struct mlx5_core_dev *dev = ct_priv->dev;
1488        struct mlx5_flow_table_attr ft_attr = {};
1489        struct mlx5_flow_namespace *ns;
1490        struct mlx5_flow_table *ft;
1491        struct mlx5_flow_group *g;
1492        u32 metadata_reg_c_2_mask;
1493        u32 *flow_group_in;
1494        void *misc;
1495        int err;
1496
1497        ns = mlx5_get_flow_namespace(dev, ct_priv->ns_type);
1498        if (!ns) {
1499                err = -EOPNOTSUPP;
1500                ct_dbg("Failed to get flow namespace");
1501                return err;
1502        }
1503
1504        flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1505        if (!flow_group_in)
1506                return -ENOMEM;
1507
1508        ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
1509        ft_attr.prio =  ct_priv->ns_type ==  MLX5_FLOW_NAMESPACE_FDB ?
1510                        FDB_TC_OFFLOAD : MLX5E_TC_PRIO;
1511        ft_attr.max_fte = 2;
1512        ft_attr.level = 1;
1513        ft = mlx5_create_flow_table(ns, &ft_attr);
1514        if (IS_ERR(ft)) {
1515                err = PTR_ERR(ft);
1516                ct_dbg("Failed to create pre ct table");
1517                goto out_free;
1518        }
1519        pre_ct->ft = ft;
1520
1521        /* create flow group */
1522        MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1523        MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
1524        MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1525                 MLX5_MATCH_MISC_PARAMETERS_2);
1526
1527        misc = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1528                            match_criteria.misc_parameters_2);
1529
1530        metadata_reg_c_2_mask = MLX5_CT_ZONE_MASK;
1531        metadata_reg_c_2_mask |= (MLX5_CT_STATE_TRK_BIT << 16);
1532        if (nat)
1533                metadata_reg_c_2_mask |= (MLX5_CT_STATE_NAT_BIT << 16);
1534
1535        MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_2,
1536                 metadata_reg_c_2_mask);
1537
1538        g = mlx5_create_flow_group(ft, flow_group_in);
1539        if (IS_ERR(g)) {
1540                err = PTR_ERR(g);
1541                ct_dbg("Failed to create pre ct group");
1542                goto err_flow_grp;
1543        }
1544        pre_ct->flow_grp = g;
1545
1546        /* create miss group */
1547        memset(flow_group_in, 0, inlen);
1548        MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
1549        MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
1550        g = mlx5_create_flow_group(ft, flow_group_in);
1551        if (IS_ERR(g)) {
1552                err = PTR_ERR(g);
1553                ct_dbg("Failed to create pre ct miss group");
1554                goto err_miss_grp;
1555        }
1556        pre_ct->miss_grp = g;
1557
1558        err = tc_ct_pre_ct_add_rules(ct_ft, pre_ct, nat);
1559        if (err)
1560                goto err_add_rules;
1561
1562        kvfree(flow_group_in);
1563        return 0;
1564
1565err_add_rules:
1566        mlx5_destroy_flow_group(pre_ct->miss_grp);
1567err_miss_grp:
1568        mlx5_destroy_flow_group(pre_ct->flow_grp);
1569err_flow_grp:
1570        mlx5_destroy_flow_table(ft);
1571out_free:
1572        kvfree(flow_group_in);
1573        return err;
1574}
1575
1576static void
1577mlx5_tc_ct_free_pre_ct(struct mlx5_ct_ft *ct_ft,
1578                       struct mlx5_tc_ct_pre *pre_ct)
1579{
1580        tc_ct_pre_ct_del_rules(ct_ft, pre_ct);
1581        mlx5_destroy_flow_group(pre_ct->miss_grp);
1582        mlx5_destroy_flow_group(pre_ct->flow_grp);
1583        mlx5_destroy_flow_table(pre_ct->ft);
1584}
1585
1586static int
1587mlx5_tc_ct_alloc_pre_ct_tables(struct mlx5_ct_ft *ft)
1588{
1589        int err;
1590
1591        err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct, false);
1592        if (err)
1593                return err;
1594
1595        err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct_nat, true);
1596        if (err)
1597                goto err_pre_ct_nat;
1598
1599        return 0;
1600
1601err_pre_ct_nat:
1602        mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct);
1603        return err;
1604}
1605
1606static void
1607mlx5_tc_ct_free_pre_ct_tables(struct mlx5_ct_ft *ft)
1608{
1609        mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct_nat);
1610        mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct);
1611}
1612
1613/* To avoid false lock dependency warning set the ct_entries_ht lock
1614 * class different than the lock class of the ht being used when deleting
1615 * last flow from a group and then deleting a group, we get into del_sw_flow_group()
1616 * which call rhashtable_destroy on fg->ftes_hash which will take ht->mutex but
1617 * it's different than the ht->mutex here.
1618 */
1619static struct lock_class_key ct_entries_ht_lock_key;
1620
1621static struct mlx5_ct_ft *
1622mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone,
1623                     struct nf_flowtable *nf_ft)
1624{
1625        struct mlx5_ct_ft *ft;
1626        int err;
1627
1628        ft = rhashtable_lookup_fast(&ct_priv->zone_ht, &zone, zone_params);
1629        if (ft) {
1630                refcount_inc(&ft->refcount);
1631                return ft;
1632        }
1633
1634        ft = kzalloc(sizeof(*ft), GFP_KERNEL);
1635        if (!ft)
1636                return ERR_PTR(-ENOMEM);
1637
1638        err = mapping_add(ct_priv->zone_mapping, &zone, &ft->zone_restore_id);
1639        if (err)
1640                goto err_mapping;
1641
1642        ft->zone = zone;
1643        ft->nf_ft = nf_ft;
1644        ft->ct_priv = ct_priv;
1645        refcount_set(&ft->refcount, 1);
1646
1647        err = mlx5_tc_ct_alloc_pre_ct_tables(ft);
1648        if (err)
1649                goto err_alloc_pre_ct;
1650
1651        err = rhashtable_init(&ft->ct_entries_ht, &cts_ht_params);
1652        if (err)
1653                goto err_init;
1654
1655        lockdep_set_class(&ft->ct_entries_ht.mutex, &ct_entries_ht_lock_key);
1656
1657        err = rhashtable_insert_fast(&ct_priv->zone_ht, &ft->node,
1658                                     zone_params);
1659        if (err)
1660                goto err_insert;
1661
1662        err = nf_flow_table_offload_add_cb(ft->nf_ft,
1663                                           mlx5_tc_ct_block_flow_offload, ft);
1664        if (err)
1665                goto err_add_cb;
1666
1667        return ft;
1668
1669err_add_cb:
1670        rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
1671err_insert:
1672        rhashtable_destroy(&ft->ct_entries_ht);
1673err_init:
1674        mlx5_tc_ct_free_pre_ct_tables(ft);
1675err_alloc_pre_ct:
1676        mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
1677err_mapping:
1678        kfree(ft);
1679        return ERR_PTR(err);
1680}
1681
1682static void
1683mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg)
1684{
1685        struct mlx5_ct_entry *entry = ptr;
1686
1687        mlx5_tc_ct_entry_put(entry);
1688}
1689
1690static void
1691mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
1692{
1693        if (!refcount_dec_and_test(&ft->refcount))
1694                return;
1695
1696        nf_flow_table_offload_del_cb(ft->nf_ft,
1697                                     mlx5_tc_ct_block_flow_offload, ft);
1698        rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
1699        rhashtable_free_and_destroy(&ft->ct_entries_ht,
1700                                    mlx5_tc_ct_flush_ft_entry,
1701                                    ct_priv);
1702        mlx5_tc_ct_free_pre_ct_tables(ft);
1703        mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
1704        kfree(ft);
1705}
1706
1707/* We translate the tc filter with CT action to the following HW model:
1708 *
1709 * +---------------------+
1710 * + ft prio (tc chain) +
1711 * + original match      +
1712 * +---------------------+
1713 *      | set chain miss mapping
1714 *      | set fte_id
1715 *      | set tunnel_id
1716 *      | do decap
1717 *      v
1718 * +---------------------+
1719 * + pre_ct/pre_ct_nat   +  if matches     +---------------------+
1720 * + zone+nat match      +---------------->+ post_ct (see below) +
1721 * +---------------------+  set zone       +---------------------+
1722 *      | set zone
1723 *      v
1724 * +--------------------+
1725 * + CT (nat or no nat) +
1726 * + tuple + zone match +
1727 * +--------------------+
1728 *      | set mark
1729 *      | set labels_id
1730 *      | set established
1731 *      | set zone_restore
1732 *      | do nat (if needed)
1733 *      v
1734 * +--------------+
1735 * + post_ct      + original filter actions
1736 * + fte_id match +------------------------>
1737 * +--------------+
1738 */
1739static struct mlx5_flow_handle *
1740__mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv,
1741                          struct mlx5e_tc_flow *flow,
1742                          struct mlx5_flow_spec *orig_spec,
1743                          struct mlx5_flow_attr *attr)
1744{
1745        bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
1746        struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
1747        struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {};
1748        u32 attr_sz = ns_to_attr_sz(ct_priv->ns_type);
1749        struct mlx5_flow_spec *post_ct_spec = NULL;
1750        struct mlx5_flow_attr *pre_ct_attr;
1751        struct mlx5_modify_hdr *mod_hdr;
1752        struct mlx5_flow_handle *rule;
1753        struct mlx5_ct_flow *ct_flow;
1754        int chain_mapping = 0, err;
1755        struct mlx5_ct_ft *ft;
1756        u32 fte_id = 1;
1757
1758        post_ct_spec = kvzalloc(sizeof(*post_ct_spec), GFP_KERNEL);
1759        ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
1760        if (!post_ct_spec || !ct_flow) {
1761                kvfree(post_ct_spec);
1762                kfree(ct_flow);
1763                return ERR_PTR(-ENOMEM);
1764        }
1765
1766        /* Register for CT established events */
1767        ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone,
1768                                  attr->ct_attr.nf_ft);
1769        if (IS_ERR(ft)) {
1770                err = PTR_ERR(ft);
1771                ct_dbg("Failed to register to ft callback");
1772                goto err_ft;
1773        }
1774        ct_flow->ft = ft;
1775
1776        err = idr_alloc_u32(&ct_priv->fte_ids, ct_flow, &fte_id,
1777                            MLX5_FTE_ID_MAX, GFP_KERNEL);
1778        if (err) {
1779                netdev_warn(priv->netdev,
1780                            "Failed to allocate fte id, err: %d\n", err);
1781                goto err_idr;
1782        }
1783        ct_flow->fte_id = fte_id;
1784
1785        /* Base flow attributes of both rules on original rule attribute */
1786        ct_flow->pre_ct_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
1787        if (!ct_flow->pre_ct_attr) {
1788                err = -ENOMEM;
1789                goto err_alloc_pre;
1790        }
1791
1792        ct_flow->post_ct_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
1793        if (!ct_flow->post_ct_attr) {
1794                err = -ENOMEM;
1795                goto err_alloc_post;
1796        }
1797
1798        pre_ct_attr = ct_flow->pre_ct_attr;
1799        memcpy(pre_ct_attr, attr, attr_sz);
1800        memcpy(ct_flow->post_ct_attr, attr, attr_sz);
1801
1802        /* Modify the original rule's action to fwd and modify, leave decap */
1803        pre_ct_attr->action = attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP;
1804        pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1805                               MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1806
1807        /* Write chain miss tag for miss in ct table as we
1808         * don't go though all prios of this chain as normal tc rules
1809         * miss.
1810         */
1811        err = mlx5_chains_get_chain_mapping(ct_priv->chains, attr->chain,
1812                                            &chain_mapping);
1813        if (err) {
1814                ct_dbg("Failed to get chain register mapping for chain");
1815                goto err_get_chain;
1816        }
1817        ct_flow->chain_mapping = chain_mapping;
1818
1819        err = mlx5e_tc_match_to_reg_set(priv->mdev, &pre_mod_acts, ct_priv->ns_type,
1820                                        CHAIN_TO_REG, chain_mapping);
1821        if (err) {
1822                ct_dbg("Failed to set chain register mapping");
1823                goto err_mapping;
1824        }
1825
1826        err = mlx5e_tc_match_to_reg_set(priv->mdev, &pre_mod_acts, ct_priv->ns_type,
1827                                        FTEID_TO_REG, fte_id);
1828        if (err) {
1829                ct_dbg("Failed to set fte_id register mapping");
1830                goto err_mapping;
1831        }
1832
1833        /* If original flow is decap, we do it before going into ct table
1834         * so add a rewrite for the tunnel match_id.
1835         */
1836        if ((pre_ct_attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) &&
1837            attr->chain == 0) {
1838                u32 tun_id = mlx5e_tc_get_flow_tun_id(flow);
1839
1840                err = mlx5e_tc_match_to_reg_set(priv->mdev, &pre_mod_acts,
1841                                                ct_priv->ns_type,
1842                                                TUNNEL_TO_REG,
1843                                                tun_id);
1844                if (err) {
1845                        ct_dbg("Failed to set tunnel register mapping");
1846                        goto err_mapping;
1847                }
1848        }
1849
1850        mod_hdr = mlx5_modify_header_alloc(priv->mdev, ct_priv->ns_type,
1851                                           pre_mod_acts.num_actions,
1852                                           pre_mod_acts.actions);
1853        if (IS_ERR(mod_hdr)) {
1854                err = PTR_ERR(mod_hdr);
1855                ct_dbg("Failed to create pre ct mod hdr");
1856                goto err_mapping;
1857        }
1858        pre_ct_attr->modify_hdr = mod_hdr;
1859
1860        /* Post ct rule matches on fte_id and executes original rule's
1861         * tc rule action
1862         */
1863        mlx5e_tc_match_to_reg_match(post_ct_spec, FTEID_TO_REG,
1864                                    fte_id, MLX5_FTE_ID_MASK);
1865
1866        /* Put post_ct rule on post_ct flow table */
1867        ct_flow->post_ct_attr->chain = 0;
1868        ct_flow->post_ct_attr->prio = 0;
1869        ct_flow->post_ct_attr->ft = ct_priv->post_ct;
1870
1871        /* Splits were handled before CT */
1872        if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB)
1873                ct_flow->post_ct_attr->esw_attr->split_count = 0;
1874
1875        ct_flow->post_ct_attr->inner_match_level = MLX5_MATCH_NONE;
1876        ct_flow->post_ct_attr->outer_match_level = MLX5_MATCH_NONE;
1877        ct_flow->post_ct_attr->action &= ~(MLX5_FLOW_CONTEXT_ACTION_DECAP);
1878        rule = mlx5_tc_rule_insert(priv, post_ct_spec,
1879                                   ct_flow->post_ct_attr);
1880        ct_flow->post_ct_rule = rule;
1881        if (IS_ERR(ct_flow->post_ct_rule)) {
1882                err = PTR_ERR(ct_flow->post_ct_rule);
1883                ct_dbg("Failed to add post ct rule");
1884                goto err_insert_post_ct;
1885        }
1886
1887        /* Change original rule point to ct table */
1888        pre_ct_attr->dest_chain = 0;
1889        pre_ct_attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft;
1890        ct_flow->pre_ct_rule = mlx5_tc_rule_insert(priv, orig_spec,
1891                                                   pre_ct_attr);
1892        if (IS_ERR(ct_flow->pre_ct_rule)) {
1893                err = PTR_ERR(ct_flow->pre_ct_rule);
1894                ct_dbg("Failed to add pre ct rule");
1895                goto err_insert_orig;
1896        }
1897
1898        attr->ct_attr.ct_flow = ct_flow;
1899        dealloc_mod_hdr_actions(&pre_mod_acts);
1900        kvfree(post_ct_spec);
1901
1902        return rule;
1903
1904err_insert_orig:
1905        mlx5_tc_rule_delete(priv, ct_flow->post_ct_rule,
1906                            ct_flow->post_ct_attr);
1907err_insert_post_ct:
1908        mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
1909err_mapping:
1910        dealloc_mod_hdr_actions(&pre_mod_acts);
1911        mlx5_chains_put_chain_mapping(ct_priv->chains, ct_flow->chain_mapping);
1912err_get_chain:
1913        kfree(ct_flow->post_ct_attr);
1914err_alloc_post:
1915        kfree(ct_flow->pre_ct_attr);
1916err_alloc_pre:
1917        idr_remove(&ct_priv->fte_ids, fte_id);
1918err_idr:
1919        mlx5_tc_ct_del_ft_cb(ct_priv, ft);
1920err_ft:
1921        kvfree(post_ct_spec);
1922        kfree(ct_flow);
1923        netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err);
1924        return ERR_PTR(err);
1925}
1926
1927static struct mlx5_flow_handle *
1928__mlx5_tc_ct_flow_offload_clear(struct mlx5_tc_ct_priv *ct_priv,
1929                                struct mlx5_flow_spec *orig_spec,
1930                                struct mlx5_flow_attr *attr,
1931                                struct mlx5e_tc_mod_hdr_acts *mod_acts)
1932{
1933        struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
1934        u32 attr_sz = ns_to_attr_sz(ct_priv->ns_type);
1935        struct mlx5_flow_attr *pre_ct_attr;
1936        struct mlx5_modify_hdr *mod_hdr;
1937        struct mlx5_flow_handle *rule;
1938        struct mlx5_ct_flow *ct_flow;
1939        int err;
1940
1941        ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
1942        if (!ct_flow)
1943                return ERR_PTR(-ENOMEM);
1944
1945        /* Base esw attributes on original rule attribute */
1946        pre_ct_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
1947        if (!pre_ct_attr) {
1948                err = -ENOMEM;
1949                goto err_attr;
1950        }
1951
1952        memcpy(pre_ct_attr, attr, attr_sz);
1953
1954        err = mlx5_tc_ct_entry_set_registers(ct_priv, mod_acts, 0, 0, 0, 0);
1955        if (err) {
1956                ct_dbg("Failed to set register for ct clear");
1957                goto err_set_registers;
1958        }
1959
1960        mod_hdr = mlx5_modify_header_alloc(priv->mdev, ct_priv->ns_type,
1961                                           mod_acts->num_actions,
1962                                           mod_acts->actions);
1963        if (IS_ERR(mod_hdr)) {
1964                err = PTR_ERR(mod_hdr);
1965                ct_dbg("Failed to add create ct clear mod hdr");
1966                goto err_set_registers;
1967        }
1968
1969        pre_ct_attr->modify_hdr = mod_hdr;
1970        pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1971
1972        rule = mlx5_tc_rule_insert(priv, orig_spec, pre_ct_attr);
1973        if (IS_ERR(rule)) {
1974                err = PTR_ERR(rule);
1975                ct_dbg("Failed to add ct clear rule");
1976                goto err_insert;
1977        }
1978
1979        attr->ct_attr.ct_flow = ct_flow;
1980        ct_flow->pre_ct_attr = pre_ct_attr;
1981        ct_flow->pre_ct_rule = rule;
1982        return rule;
1983
1984err_insert:
1985        mlx5_modify_header_dealloc(priv->mdev, mod_hdr);
1986err_set_registers:
1987        netdev_warn(priv->netdev,
1988                    "Failed to offload ct clear flow, err %d\n", err);
1989        kfree(pre_ct_attr);
1990err_attr:
1991        kfree(ct_flow);
1992
1993        return ERR_PTR(err);
1994}
1995
1996struct mlx5_flow_handle *
1997mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv,
1998                        struct mlx5e_tc_flow *flow,
1999                        struct mlx5_flow_spec *spec,
2000                        struct mlx5_flow_attr *attr,
2001                        struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
2002{
2003        bool clear_action = attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR;
2004        struct mlx5_flow_handle *rule;
2005
2006        if (!priv)
2007                return ERR_PTR(-EOPNOTSUPP);
2008
2009        mutex_lock(&priv->control_lock);
2010
2011        if (clear_action)
2012                rule = __mlx5_tc_ct_flow_offload_clear(priv, spec, attr, mod_hdr_acts);
2013        else
2014                rule = __mlx5_tc_ct_flow_offload(priv, flow, spec, attr);
2015        mutex_unlock(&priv->control_lock);
2016
2017        return rule;
2018}
2019
2020static void
2021__mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv,
2022                         struct mlx5e_tc_flow *flow,
2023                         struct mlx5_ct_flow *ct_flow)
2024{
2025        struct mlx5_flow_attr *pre_ct_attr = ct_flow->pre_ct_attr;
2026        struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
2027
2028        mlx5_tc_rule_delete(priv, ct_flow->pre_ct_rule,
2029                            pre_ct_attr);
2030        mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
2031
2032        if (ct_flow->post_ct_rule) {
2033                mlx5_tc_rule_delete(priv, ct_flow->post_ct_rule,
2034                                    ct_flow->post_ct_attr);
2035                mlx5_chains_put_chain_mapping(ct_priv->chains, ct_flow->chain_mapping);
2036                idr_remove(&ct_priv->fte_ids, ct_flow->fte_id);
2037                mlx5_tc_ct_del_ft_cb(ct_priv, ct_flow->ft);
2038        }
2039
2040        kfree(ct_flow->pre_ct_attr);
2041        kfree(ct_flow->post_ct_attr);
2042        kfree(ct_flow);
2043}
2044
2045void
2046mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv,
2047                       struct mlx5e_tc_flow *flow,
2048                       struct mlx5_flow_attr *attr)
2049{
2050        struct mlx5_ct_flow *ct_flow = attr->ct_attr.ct_flow;
2051
2052        /* We are called on error to clean up stuff from parsing
2053         * but we don't have anything for now
2054         */
2055        if (!ct_flow)
2056                return;
2057
2058        mutex_lock(&priv->control_lock);
2059        __mlx5_tc_ct_delete_flow(priv, flow, ct_flow);
2060        mutex_unlock(&priv->control_lock);
2061}
2062
2063static int
2064mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch *esw,
2065                                  const char **err_msg)
2066{
2067        if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) {
2068                *err_msg = "firmware level support is missing";
2069                return -EOPNOTSUPP;
2070        }
2071
2072        if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) {
2073                /* vlan workaround should be avoided for multi chain rules.
2074                 * This is just a sanity check as pop vlan action should
2075                 * be supported by any FW that supports ignore_flow_level
2076                 */
2077
2078                *err_msg = "firmware vlan actions support is missing";
2079                return -EOPNOTSUPP;
2080        }
2081
2082        if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev,
2083                                    fdb_modify_header_fwd_to_table)) {
2084                /* CT always writes to registers which are mod header actions.
2085                 * Therefore, mod header and goto is required
2086                 */
2087
2088                *err_msg = "firmware fwd and modify support is missing";
2089                return -EOPNOTSUPP;
2090        }
2091
2092        if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
2093                *err_msg = "register loopback isn't supported";
2094                return -EOPNOTSUPP;
2095        }
2096
2097        return 0;
2098}
2099
2100static int
2101mlx5_tc_ct_init_check_nic_support(struct mlx5e_priv *priv,
2102                                  const char **err_msg)
2103{
2104        if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ignore_flow_level)) {
2105                *err_msg = "firmware level support is missing";
2106                return -EOPNOTSUPP;
2107        }
2108
2109        return 0;
2110}
2111
2112static int
2113mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv,
2114                              enum mlx5_flow_namespace_type ns_type,
2115                              const char **err_msg)
2116{
2117        struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2118
2119#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
2120        /* cannot restore chain ID on HW miss */
2121
2122        *err_msg = "tc skb extension missing";
2123        return -EOPNOTSUPP;
2124#endif
2125        if (ns_type == MLX5_FLOW_NAMESPACE_FDB)
2126                return mlx5_tc_ct_init_check_esw_support(esw, err_msg);
2127        else
2128                return mlx5_tc_ct_init_check_nic_support(priv, err_msg);
2129}
2130
2131#define INIT_ERR_PREFIX "tc ct offload init failed"
2132
2133struct mlx5_tc_ct_priv *
2134mlx5_tc_ct_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains,
2135                struct mod_hdr_tbl *mod_hdr,
2136                enum mlx5_flow_namespace_type ns_type)
2137{
2138        struct mlx5_tc_ct_priv *ct_priv;
2139        struct mlx5_core_dev *dev;
2140        const char *msg;
2141        int err;
2142
2143        dev = priv->mdev;
2144        err = mlx5_tc_ct_init_check_support(priv, ns_type, &msg);
2145        if (err) {
2146                mlx5_core_warn(dev,
2147                               "tc ct offload not supported, %s\n",
2148                               msg);
2149                goto err_support;
2150        }
2151
2152        ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL);
2153        if (!ct_priv)
2154                goto err_alloc;
2155
2156        ct_priv->zone_mapping = mapping_create(sizeof(u16), 0, true);
2157        if (IS_ERR(ct_priv->zone_mapping)) {
2158                err = PTR_ERR(ct_priv->zone_mapping);
2159                goto err_mapping_zone;
2160        }
2161
2162        ct_priv->labels_mapping = mapping_create(sizeof(u32) * 4, 0, true);
2163        if (IS_ERR(ct_priv->labels_mapping)) {
2164                err = PTR_ERR(ct_priv->labels_mapping);
2165                goto err_mapping_labels;
2166        }
2167
2168        spin_lock_init(&ct_priv->ht_lock);
2169        ct_priv->ns_type = ns_type;
2170        ct_priv->chains = chains;
2171        ct_priv->netdev = priv->netdev;
2172        ct_priv->dev = priv->mdev;
2173        ct_priv->mod_hdr_tbl = mod_hdr;
2174        ct_priv->ct = mlx5_chains_create_global_table(chains);
2175        if (IS_ERR(ct_priv->ct)) {
2176                err = PTR_ERR(ct_priv->ct);
2177                mlx5_core_warn(dev,
2178                               "%s, failed to create ct table err: %d\n",
2179                               INIT_ERR_PREFIX, err);
2180                goto err_ct_tbl;
2181        }
2182
2183        ct_priv->ct_nat = mlx5_chains_create_global_table(chains);
2184        if (IS_ERR(ct_priv->ct_nat)) {
2185                err = PTR_ERR(ct_priv->ct_nat);
2186                mlx5_core_warn(dev,
2187                               "%s, failed to create ct nat table err: %d\n",
2188                               INIT_ERR_PREFIX, err);
2189                goto err_ct_nat_tbl;
2190        }
2191
2192        ct_priv->post_ct = mlx5_chains_create_global_table(chains);
2193        if (IS_ERR(ct_priv->post_ct)) {
2194                err = PTR_ERR(ct_priv->post_ct);
2195                mlx5_core_warn(dev,
2196                               "%s, failed to create post ct table err: %d\n",
2197                               INIT_ERR_PREFIX, err);
2198                goto err_post_ct_tbl;
2199        }
2200
2201        idr_init(&ct_priv->fte_ids);
2202        mutex_init(&ct_priv->control_lock);
2203        rhashtable_init(&ct_priv->zone_ht, &zone_params);
2204        rhashtable_init(&ct_priv->ct_tuples_ht, &tuples_ht_params);
2205        rhashtable_init(&ct_priv->ct_tuples_nat_ht, &tuples_nat_ht_params);
2206
2207        return ct_priv;
2208
2209err_post_ct_tbl:
2210        mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat);
2211err_ct_nat_tbl:
2212        mlx5_chains_destroy_global_table(chains, ct_priv->ct);
2213err_ct_tbl:
2214        mapping_destroy(ct_priv->labels_mapping);
2215err_mapping_labels:
2216        mapping_destroy(ct_priv->zone_mapping);
2217err_mapping_zone:
2218        kfree(ct_priv);
2219err_alloc:
2220err_support:
2221
2222        return NULL;
2223}
2224
2225void
2226mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv)
2227{
2228        struct mlx5_fs_chains *chains;
2229
2230        if (!ct_priv)
2231                return;
2232
2233        chains = ct_priv->chains;
2234
2235        mlx5_chains_destroy_global_table(chains, ct_priv->post_ct);
2236        mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat);
2237        mlx5_chains_destroy_global_table(chains, ct_priv->ct);
2238        mapping_destroy(ct_priv->zone_mapping);
2239        mapping_destroy(ct_priv->labels_mapping);
2240
2241        rhashtable_destroy(&ct_priv->ct_tuples_ht);
2242        rhashtable_destroy(&ct_priv->ct_tuples_nat_ht);
2243        rhashtable_destroy(&ct_priv->zone_ht);
2244        mutex_destroy(&ct_priv->control_lock);
2245        idr_destroy(&ct_priv->fte_ids);
2246        kfree(ct_priv);
2247}
2248
2249bool
2250mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv *ct_priv,
2251                         struct sk_buff *skb, u8 zone_restore_id)
2252{
2253        struct mlx5_ct_tuple tuple = {};
2254        struct mlx5_ct_entry *entry;
2255        u16 zone;
2256
2257        if (!ct_priv || !zone_restore_id)
2258                return true;
2259
2260        if (mapping_find(ct_priv->zone_mapping, zone_restore_id, &zone))
2261                return false;
2262
2263        if (!mlx5_tc_ct_skb_to_tuple(skb, &tuple, zone))
2264                return false;
2265
2266        spin_lock(&ct_priv->ht_lock);
2267
2268        entry = mlx5_tc_ct_entry_get(ct_priv, &tuple);
2269        if (!entry) {
2270                spin_unlock(&ct_priv->ht_lock);
2271                return false;
2272        }
2273
2274        if (IS_ERR(entry)) {
2275                spin_unlock(&ct_priv->ht_lock);
2276                return false;
2277        }
2278        spin_unlock(&ct_priv->ht_lock);
2279
2280        tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie);
2281        __mlx5_tc_ct_entry_put(entry);
2282
2283        return true;
2284}
2285