// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2021 Corigine, Inc. */

#include "conntrack.h"

const struct rhashtable_params nfp_tc_ct_merge_params = {
        .head_offset            = offsetof(struct nfp_fl_ct_tc_merge,
                                           hash_node),
        .key_len                = sizeof(unsigned long) * 2,
        .key_offset             = offsetof(struct nfp_fl_ct_tc_merge, cookie),
        .automatic_shrinking    = true,
};

const struct rhashtable_params nfp_nft_ct_merge_params = {
        .head_offset            = offsetof(struct nfp_fl_nft_tc_merge,
                                           hash_node),
        .key_len                = sizeof(unsigned long) * 3,
        .key_offset             = offsetof(struct nfp_fl_nft_tc_merge, cookie),
        .automatic_shrinking    = true,
};

static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
                                              enum flow_action_id act_id);

/**
 * get_hashentry() - Wrapper around hashtable lookup.
 * @ht:         hashtable where entry could be found
 * @key:        key to lookup
 * @params:     hashtable params
 * @size:       size of entry to allocate if not in table
 *
 * Returns an entry from a hashtable. If entry does not exist
 * yet allocate the memory for it and return the new entry.
 */
static void *get_hashentry(struct rhashtable *ht, void *key,
                           const struct rhashtable_params params, size_t size)
{
        void *result;

        result = rhashtable_lookup_fast(ht, key, params);

        if (result)
                return result;

        result = kzalloc(size, GFP_KERNEL);
        if (!result)
                return ERR_PTR(-ENOMEM);

        return result;
}

bool is_pre_ct_flow(struct flow_cls_offload *flow)
{
        struct flow_action_entry *act;
        int i;

        flow_action_for_each(i, act, &flow->rule->action) {
                if (act->id == FLOW_ACTION_CT && !act->ct.action)
                        return true;
        }
        return false;
}

bool is_post_ct_flow(struct flow_cls_offload *flow)
{
        struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
        struct flow_dissector *dissector = rule->match.dissector;
        struct flow_match_ct ct;

        if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
                flow_rule_match_ct(rule, &ct);
                if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
                        return true;
        }
        return false;
}

static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
                              struct nfp_fl_ct_flow_entry *entry2)
{
        unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys &
                                 entry2->rule->match.dissector->used_keys;
        bool out;

        /* check the overlapped fields one by one, the unmasked part
         * should not conflict with each other.
         */
        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) {
                struct flow_match_control match1, match2;

                flow_rule_match_control(entry1->rule, &match1);
                flow_rule_match_control(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) {
                struct flow_match_basic match1, match2;

                flow_rule_match_basic(entry1->rule, &match1);
                flow_rule_match_basic(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
                struct flow_match_ipv4_addrs match1, match2;

                flow_rule_match_ipv4_addrs(entry1->rule, &match1);
                flow_rule_match_ipv4_addrs(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
                struct flow_match_ipv6_addrs match1, match2;

                flow_rule_match_ipv6_addrs(entry1->rule, &match1);
                flow_rule_match_ipv6_addrs(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) {
                struct flow_match_ports match1, match2;

                flow_rule_match_ports(entry1->rule, &match1);
                flow_rule_match_ports(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
                struct flow_match_eth_addrs match1, match2;

                flow_rule_match_eth_addrs(entry1->rule, &match1);
                flow_rule_match_eth_addrs(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) {
                struct flow_match_vlan match1, match2;

                flow_rule_match_vlan(entry1->rule, &match1);
                flow_rule_match_vlan(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) {
                struct flow_match_mpls match1, match2;

                flow_rule_match_mpls(entry1->rule, &match1);
                flow_rule_match_mpls(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) {
                struct flow_match_tcp match1, match2;

                flow_rule_match_tcp(entry1->rule, &match1);
                flow_rule_match_tcp(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) {
                struct flow_match_ip match1, match2;

                flow_rule_match_ip(entry1->rule, &match1);
                flow_rule_match_ip(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
                struct flow_match_enc_keyid match1, match2;

                flow_rule_match_enc_keyid(entry1->rule, &match1);
                flow_rule_match_enc_keyid(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
                struct flow_match_ipv4_addrs match1, match2;

                flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
                flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
                struct flow_match_ipv6_addrs match1, match2;

                flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
                flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
                struct flow_match_control match1, match2;

                flow_rule_match_enc_control(entry1->rule, &match1);
                flow_rule_match_enc_control(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) {
                struct flow_match_ip match1, match2;

                flow_rule_match_enc_ip(entry1->rule, &match1);
                flow_rule_match_enc_ip(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
                struct flow_match_enc_opts match1, match2;

                flow_rule_match_enc_opts(entry1->rule, &match1);
                flow_rule_match_enc_opts(entry2->rule, &match2);
                COMPARE_UNMASKED_FIELDS(match1, match2, &out);
                if (out)
                        goto check_failed;
        }

        return 0;

check_failed:
        return -EINVAL;
}

static int nfp_ct_check_mangle_merge(struct flow_action_entry *a_in,
                                     struct flow_rule *rule)
{
        enum flow_action_mangle_base htype = a_in->mangle.htype;
        u32 offset = a_in->mangle.offset;

        switch (htype) {
        case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
                if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS))
                        return -EOPNOTSUPP;
                break;
        case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
                if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
                        struct flow_match_ip match;

                        flow_rule_match_ip(rule, &match);
                        if (offset == offsetof(struct iphdr, ttl) &&
                            match.mask->ttl)
                                return -EOPNOTSUPP;
                        if (offset == round_down(offsetof(struct iphdr, tos), 4) &&
                            match.mask->tos)
                                return -EOPNOTSUPP;
                }
                if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
                        struct flow_match_ipv4_addrs match;

                        flow_rule_match_ipv4_addrs(rule, &match);
                        if (offset == offsetof(struct iphdr, saddr) &&
                            match.mask->src)
                                return -EOPNOTSUPP;
                        if (offset == offsetof(struct iphdr, daddr) &&
                            match.mask->dst)
                                return -EOPNOTSUPP;
                }
                break;
        case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
                if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
                        struct flow_match_ip match;

                        flow_rule_match_ip(rule, &match);
                        if (offset == round_down(offsetof(struct ipv6hdr, hop_limit), 4) &&
                            match.mask->ttl)
                                return -EOPNOTSUPP;
                        /* for ipv6, tos and flow_lbl are in the same word */
                        if (offset == round_down(offsetof(struct ipv6hdr, flow_lbl), 4) &&
                            match.mask->tos)
                                return -EOPNOTSUPP;
                }
                if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
                        struct flow_match_ipv6_addrs match;

                        flow_rule_match_ipv6_addrs(rule, &match);
                        if (offset >= offsetof(struct ipv6hdr, saddr) &&
                            offset < offsetof(struct ipv6hdr, daddr) &&
                            memchr_inv(&match.mask->src, 0, sizeof(match.mask->src)))
                                return -EOPNOTSUPP;
                        if (offset >= offsetof(struct ipv6hdr, daddr) &&
                            offset < sizeof(struct ipv6hdr) &&
                            memchr_inv(&match.mask->dst, 0, sizeof(match.mask->dst)))
                                return -EOPNOTSUPP;
                }
                break;
        case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
        case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
                /* currently only can modify ports */
                if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS))
                        return -EOPNOTSUPP;
                break;
        default:
                break;
        }
        return 0;
}

static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
                                  struct nfp_fl_ct_flow_entry *post_ct_entry,
                                  struct nfp_fl_ct_flow_entry *nft_entry)
{
        struct flow_action_entry *act;
        int err, i;

        /* Check for pre_ct->action conflicts */
        flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
                switch (act->id) {
                case FLOW_ACTION_MANGLE:
                        err = nfp_ct_check_mangle_merge(act, nft_entry->rule);
                        if (err)
                                return err;
                        err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
                        if (err)
                                return err;
                        break;
                case FLOW_ACTION_VLAN_PUSH:
                case FLOW_ACTION_VLAN_POP:
                case FLOW_ACTION_VLAN_MANGLE:
                case FLOW_ACTION_MPLS_PUSH:
                case FLOW_ACTION_MPLS_POP:
                case FLOW_ACTION_MPLS_MANGLE:
                        return -EOPNOTSUPP;
                default:
                        break;
                }
        }

        /* Check for nft->action conflicts */
        flow_action_for_each(i, act, &nft_entry->rule->action) {
                switch (act->id) {
                case FLOW_ACTION_MANGLE:
                        err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
                        if (err)
                                return err;
                        break;
                case FLOW_ACTION_VLAN_PUSH:
                case FLOW_ACTION_VLAN_POP:
                case FLOW_ACTION_VLAN_MANGLE:
                case FLOW_ACTION_MPLS_PUSH:
                case FLOW_ACTION_MPLS_POP:
                case FLOW_ACTION_MPLS_MANGLE:
                        return -EOPNOTSUPP;
                default:
                        break;
                }
        }
        return 0;
}

static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
                             struct nfp_fl_ct_flow_entry *nft_entry)
{
        struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
        struct flow_action_entry *ct_met;
        struct flow_match_ct ct;
        int i;

        ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
        if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) {
                u32 *act_lbl;

                act_lbl = ct_met->ct_metadata.labels;
                flow_rule_match_ct(post_ct_entry->rule, &ct);
                for (i = 0; i < 4; i++) {
                        if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
                            (act_lbl[i] & ct.mask->ct_labels[i]))
                                return -EINVAL;
                }

                if ((ct.key->ct_mark & ct.mask->ct_mark) ^
                    (ct_met->ct_metadata.mark & ct.mask->ct_mark))
                        return -EINVAL;

                return 0;
        }

        return -EINVAL;
}

static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
{
        return 0;
}

static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
                                 struct net_device *netdev)
{
        return 0;
}

static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
                               struct nfp_fl_ct_flow_entry *nft_entry,
                               struct nfp_fl_ct_tc_merge *tc_m_entry)
{
        struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
        struct nfp_fl_nft_tc_merge *nft_m_entry;
        unsigned long new_cookie[3];
        int err;

        pre_ct_entry = tc_m_entry->pre_ct_parent;
        post_ct_entry = tc_m_entry->post_ct_parent;

        err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
        if (err)
                return err;

        /* Check that the two tc flows are also compatible with
         * the nft entry. No need to check the pre_ct and post_ct
         * entries as that was already done during pre_merge.
         * The nft entry does not have a netdev or chain populated, so
         * skip this check.
         */
        err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
        if (err)
                return err;
        err = nfp_ct_merge_check(post_ct_entry, nft_entry);
        if (err)
                return err;
        err = nfp_ct_check_meta(post_ct_entry, nft_entry);
        if (err)
                return err;

        /* Combine tc_merge and nft cookies for this cookie. */
        new_cookie[0] = tc_m_entry->cookie[0];
        new_cookie[1] = tc_m_entry->cookie[1];
        new_cookie[2] = nft_entry->cookie;
        nft_m_entry = get_hashentry(&zt->nft_merge_tb,
                                    &new_cookie,
                                    nfp_nft_ct_merge_params,
                                    sizeof(*nft_m_entry));

        if (IS_ERR(nft_m_entry))
                return PTR_ERR(nft_m_entry);

        /* nft_m_entry already present, not merging again */
        if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
                return 0;

        memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
        nft_m_entry->zt = zt;
        nft_m_entry->tc_m_parent = tc_m_entry;
        nft_m_entry->nft_parent = nft_entry;
        nft_m_entry->tc_flower_cookie = 0;
        /* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created
         * it only combined them if the netdevs were the same, so can use any of them.
         */
        nft_m_entry->netdev = pre_ct_entry->netdev;

        /* Add this entry to the tc_m_list and nft_flow lists */
        list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
        list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);

        /* Generate offload structure and send to nfp */
        err = nfp_fl_ct_add_offload(nft_m_entry);
        if (err)
                goto err_nft_ct_offload;

        err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
                                     nfp_nft_ct_merge_params);
        if (err)
                goto err_nft_ct_merge_insert;

        zt->nft_merge_count++;

        return err;

err_nft_ct_merge_insert:
        nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
                              nft_m_entry->netdev);
err_nft_ct_offload:
        list_del(&nft_m_entry->tc_merge_list);
        list_del(&nft_m_entry->nft_flow_list);
        kfree(nft_m_entry);
        return err;
}

static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
                              struct nfp_fl_ct_flow_entry *ct_entry1,
                              struct nfp_fl_ct_flow_entry *ct_entry2)
{
        struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
        struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
        struct nfp_fl_ct_tc_merge *m_entry;
        unsigned long new_cookie[2];
        int err;

        if (ct_entry1->type == CT_TYPE_PRE_CT) {
                pre_ct_entry = ct_entry1;
                post_ct_entry = ct_entry2;
        } else {
                post_ct_entry = ct_entry1;
                pre_ct_entry = ct_entry2;
        }

        if (post_ct_entry->netdev != pre_ct_entry->netdev)
                return -EINVAL;
        /* Checks that the chain_index of the filter matches the
         * chain_index of the GOTO action.
         */
        if (post_ct_entry->chain_index != pre_ct_entry->chain_index)
                return -EINVAL;

        err = nfp_ct_merge_check(post_ct_entry, pre_ct_entry);
        if (err)
                return err;

        new_cookie[0] = pre_ct_entry->cookie;
        new_cookie[1] = post_ct_entry->cookie;
        m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
                                nfp_tc_ct_merge_params, sizeof(*m_entry));
        if (IS_ERR(m_entry))
                return PTR_ERR(m_entry);

        /* m_entry already present, not merging again */
        if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
                return 0;

        memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
        m_entry->zt = zt;
        m_entry->post_ct_parent = post_ct_entry;
        m_entry->pre_ct_parent = pre_ct_entry;

        /* Add this entry to the pre_ct and post_ct lists */
        list_add(&m_entry->post_ct_list, &post_ct_entry->children);
        list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
        INIT_LIST_HEAD(&m_entry->children);

        err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
                                     nfp_tc_ct_merge_params);
        if (err)
                goto err_ct_tc_merge_insert;
        zt->tc_merge_count++;

        /* Merge with existing nft flows */
        list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
                                 list_node) {
                nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
        }

        return 0;

err_ct_tc_merge_insert:
        list_del(&m_entry->post_ct_list);
        list_del(&m_entry->pre_ct_list);
        kfree(m_entry);
        return err;
}

static struct
nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
                                         u16 zone, bool wildcarded)
{
        struct nfp_fl_ct_zone_entry *zt;
        int err;

        if (wildcarded && priv->ct_zone_wc)
                return priv->ct_zone_wc;

        if (!wildcarded) {
                zt = get_hashentry(&priv->ct_zone_table, &zone,
                                   nfp_zone_table_params, sizeof(*zt));

                /* If priv is set this is an existing entry, just return it */
                if (IS_ERR(zt) || zt->priv)
                        return zt;
        } else {
                zt = kzalloc(sizeof(*zt), GFP_KERNEL);
                if (!zt)
                        return ERR_PTR(-ENOMEM);
        }

        zt->zone = zone;
        zt->priv = priv;
        zt->nft = NULL;

        /* init the various hash tables and lists*/
        INIT_LIST_HEAD(&zt->pre_ct_list);
        INIT_LIST_HEAD(&zt->post_ct_list);
        INIT_LIST_HEAD(&zt->nft_flows_list);

        err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
        if (err)
                goto err_tc_merge_tb_init;

        err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
        if (err)
                goto err_nft_merge_tb_init;

        if (wildcarded) {
                priv->ct_zone_wc = zt;
        } else {
                err = rhashtable_insert_fast(&priv->ct_zone_table,
                                             &zt->hash_node,
                                             nfp_zone_table_params);
                if (err)
                        goto err_zone_insert;
        }

        return zt;

err_zone_insert:
        rhashtable_destroy(&zt->nft_merge_tb);
err_nft_merge_tb_init:
        rhashtable_destroy(&zt->tc_merge_tb);
err_tc_merge_tb_init:
        kfree(zt);
        return ERR_PTR(err);
}

static struct
nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
                                         struct net_device *netdev,
                                         struct flow_cls_offload *flow,
                                         bool is_nft, struct netlink_ext_ack *extack)
{
        struct nf_flow_match *nft_match = NULL;
        struct nfp_fl_ct_flow_entry *entry;
        struct nfp_fl_ct_map_entry *map;
        struct flow_action_entry *act;
        int err, i;

        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
        if (!entry)
                return ERR_PTR(-ENOMEM);

        entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
        if (!entry->rule) {
                err = -ENOMEM;
                goto err_pre_ct_rule;
        }

        /* nft flows gets destroyed after callback return, so need
         * to do a full copy instead of just a reference.
         */
        if (is_nft) {
                nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
                if (!nft_match) {
                        err = -ENOMEM;
                        goto err_pre_ct_act;
                }
                memcpy(&nft_match->dissector, flow->rule->match.dissector,
                       sizeof(nft_match->dissector));
                memcpy(&nft_match->mask, flow->rule->match.mask,
                       sizeof(nft_match->mask));
                memcpy(&nft_match->key, flow->rule->match.key,
                       sizeof(nft_match->key));
                entry->rule->match.dissector = &nft_match->dissector;
                entry->rule->match.mask = &nft_match->mask;
                entry->rule->match.key = &nft_match->key;
        } else {
                entry->rule->match.dissector = flow->rule->match.dissector;
                entry->rule->match.mask = flow->rule->match.mask;
                entry->rule->match.key = flow->rule->match.key;
        }

        entry->zt = zt;
        entry->netdev = netdev;
        entry->cookie = flow->cookie;
        entry->chain_index = flow->common.chain_index;
        entry->tun_offset = NFP_FL_CT_NO_TUN;

        /* Copy over action data. Unfortunately we do not get a handle to the
         * original tcf_action data, and the flow objects gets destroyed, so we
         * cannot just save a pointer to this either, so need to copy over the
         * data unfortunately.
         */
        entry->rule->action.num_entries = flow->rule->action.num_entries;
        flow_action_for_each(i, act, &flow->rule->action) {
                struct flow_action_entry *new_act;

                new_act = &entry->rule->action.entries[i];
                memcpy(new_act, act, sizeof(struct flow_action_entry));
                /* Entunnel is a special case, need to allocate and copy
                 * tunnel info.
                 */
                if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
                        struct ip_tunnel_info *tun = act->tunnel;
                        size_t tun_size = sizeof(*tun) + tun->options_len;

                        new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
                        if (!new_act->tunnel) {
                                err = -ENOMEM;
                                goto err_pre_ct_tun_cp;
                        }
                        entry->tun_offset = i;
                }
        }

        INIT_LIST_HEAD(&entry->children);

        /* Now add a ct map entry to flower-priv */
        map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
                            nfp_ct_map_params, sizeof(*map));
        if (IS_ERR(map)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "offload error: ct map entry creation failed");
                err = -ENOMEM;
                goto err_ct_flow_insert;
        }
        map->cookie = flow->cookie;
        map->ct_entry = entry;
        err = rhashtable_insert_fast(&zt->priv->ct_map_table,
                                     &map->hash_node,
                                     nfp_ct_map_params);
        if (err) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "offload error: ct map entry table add failed");
                goto err_map_insert;
        }

        return entry;

err_map_insert:
        kfree(map);
err_ct_flow_insert:
        if (entry->tun_offset != NFP_FL_CT_NO_TUN)
                kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
err_pre_ct_tun_cp:
        kfree(nft_match);
err_pre_ct_act:
        kfree(entry->rule);
err_pre_ct_rule:
        kfree(entry);
        return ERR_PTR(err);
}

static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
{
        struct nfp_fl_ct_zone_entry *zt;
        int err;

        zt = m_entry->zt;

        /* Flow is in HW, need to delete */
        if (m_entry->tc_flower_cookie) {
                err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
                                            m_entry->netdev);
                if (err)
                        return;
        }

        WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
                                            &m_entry->hash_node,
                                            nfp_nft_ct_merge_params));
        zt->nft_merge_count--;
        list_del(&m_entry->tc_merge_list);
        list_del(&m_entry->nft_flow_list);

        kfree(m_entry);
}

static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
{
        struct nfp_fl_nft_tc_merge *m_entry, *tmp;

        /* These post entries are parts of two lists, one is a list of nft_entries
         * and the other is of from a list of tc_merge structures. Iterate
         * through the relevant list and cleanup the entries.
         */

        if (is_nft_flow) {
                /* Need to iterate through list of nft_flow entries*/
                struct nfp_fl_ct_flow_entry *ct_entry = entry;

                list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
                                         nft_flow_list) {
                        cleanup_nft_merge_entry(m_entry);
                }
        } else {
                /* Need to iterate through list of tc_merged_flow entries*/
                struct nfp_fl_ct_tc_merge *ct_entry = entry;

                list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
                                         tc_merge_list) {
                        cleanup_nft_merge_entry(m_entry);
                }
        }
}

static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
{
        struct nfp_fl_ct_zone_entry *zt;
        int err;

        zt = m_ent->zt;
        err = rhashtable_remove_fast(&zt->tc_merge_tb,
                                     &m_ent->hash_node,
                                     nfp_tc_ct_merge_params);
        if (err)
                pr_warn("WARNING: could not remove merge_entry from hashtable\n");
        zt->tc_merge_count--;
        list_del(&m_ent->post_ct_list);
        list_del(&m_ent->pre_ct_list);

        if (!list_empty(&m_ent->children))
                nfp_free_nft_merge_children(m_ent, false);
        kfree(m_ent);
}

static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
{
        struct nfp_fl_ct_tc_merge *m_ent, *tmp;

        switch (entry->type) {
        case CT_TYPE_PRE_CT:
                list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
                        nfp_del_tc_merge_entry(m_ent);
                }
                break;
        case CT_TYPE_POST_CT:
                list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
                        nfp_del_tc_merge_entry(m_ent);
                }
                break;
        default:
                break;
        }
}

void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
{
        list_del(&entry->list_node);

        if (!list_empty(&entry->children)) {
                if (entry->type == CT_TYPE_NFT)
                        nfp_free_nft_merge_children(entry, true);
                else
                        nfp_free_tc_merge_children(entry);
        }

        if (entry->tun_offset != NFP_FL_CT_NO_TUN)
                kfree(entry->rule->action.entries[entry->tun_offset].tunnel);

        if (entry->type == CT_TYPE_NFT) {
                struct nf_flow_match *nft_match;

                nft_match = container_of(entry->rule->match.dissector,
                                         struct nf_flow_match, dissector);
                kfree(nft_match);
        }

        kfree(entry->rule);
        kfree(entry);
}

static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
                                              enum flow_action_id act_id)
{
        struct flow_action_entry *act = NULL;
        int i;

        flow_action_for_each(i, act, &rule->action) {
                if (act->id == act_id)
                        return act;
        }
        return NULL;
}

static void
nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
                        struct nfp_fl_ct_zone_entry *zt_src,
                        struct nfp_fl_ct_zone_entry *zt_dst)
{
        struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
        struct list_head *ct_list;

        if (ct_entry1->type == CT_TYPE_PRE_CT)
                ct_list = &zt_src->post_ct_list;
        else if (ct_entry1->type == CT_TYPE_POST_CT)
                ct_list = &zt_src->pre_ct_list;
        else
                return;

        list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
                                 list_node) {
                nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
        }
}

static void
nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
                         struct nfp_fl_ct_zone_entry *zt)
{
        struct nfp_fl_ct_tc_merge *tc_merge_entry;
        struct rhashtable_iter iter;

        rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
        rhashtable_walk_start(&iter);
        while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
                if (IS_ERR(tc_merge_entry))
                        continue;
                rhashtable_walk_stop(&iter);
                nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
                rhashtable_walk_start(&iter);
        }
        rhashtable_walk_stop(&iter);
        rhashtable_walk_exit(&iter);
}

int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
                            struct net_device *netdev,
                            struct flow_cls_offload *flow,
                            struct netlink_ext_ack *extack)
{
        struct flow_action_entry *ct_act, *ct_goto;
        struct nfp_fl_ct_flow_entry *ct_entry;
        struct nfp_fl_ct_zone_entry *zt;
        int err;

        ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT);
        if (!ct_act) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "unsupported offload: Conntrack action empty in conntrack offload");
                return -EOPNOTSUPP;
        }

        ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
        if (!ct_goto) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "unsupported offload: Conntrack requires ACTION_GOTO");
                return -EOPNOTSUPP;
        }

        zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
        if (IS_ERR(zt)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "offload error: Could not create zone table entry");
                return PTR_ERR(zt);
        }

        if (!zt->nft) {
                zt->nft = ct_act->ct.flow_table;
                err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
                if (err) {
                        NL_SET_ERR_MSG_MOD(extack,
                                           "offload error: Could not register nft_callback");
                        return err;
                }
        }

        /* Add entry to pre_ct_list */
        ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
        if (IS_ERR(ct_entry))
                return PTR_ERR(ct_entry);
        ct_entry->type = CT_TYPE_PRE_CT;
        ct_entry->chain_index = ct_goto->chain_index;
        list_add(&ct_entry->list_node, &zt->pre_ct_list);
        zt->pre_ct_count++;

        nfp_ct_merge_tc_entries(ct_entry, zt, zt);

        /* Need to check and merge with tables in the wc_zone as well */
        if (priv->ct_zone_wc)
                nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);

        return 0;
}

int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
                             struct net_device *netdev,
                             struct flow_cls_offload *flow,
                             struct netlink_ext_ack *extack)
{
        struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
        struct nfp_fl_ct_flow_entry *ct_entry;
        struct nfp_fl_ct_zone_entry *zt;
        bool wildcarded = false;
        struct flow_match_ct ct;

        flow_rule_match_ct(rule, &ct);

        if (!ct.mask->ct_zone) {
                wildcarded = true;
        } else if (ct.mask->ct_zone != U16_MAX) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "unsupported offload: partially wildcarded ct_zone is not supported");
                return -EOPNOTSUPP;
        }

        zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
        if (IS_ERR(zt)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "offload error: Could not create zone table entry");
                return PTR_ERR(zt);
        }

        /* Add entry to post_ct_list */
        ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
        if (IS_ERR(ct_entry))
                return PTR_ERR(ct_entry);

        ct_entry->type = CT_TYPE_POST_CT;
        ct_entry->chain_index = flow->common.chain_index;
        list_add(&ct_entry->list_node, &zt->post_ct_list);
        zt->post_ct_count++;

        if (wildcarded) {
                /* Iterate through all zone tables if not empty, look for merges with
                 * pre_ct entries and merge them.
                 */
                struct rhashtable_iter iter;
                struct nfp_fl_ct_zone_entry *zone_table;

                rhashtable_walk_enter(&priv->ct_zone_table, &iter);
                rhashtable_walk_start(&iter);
                while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
                        if (IS_ERR(zone_table))
                                continue;
                        rhashtable_walk_stop(&iter);
                        nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
                        rhashtable_walk_start(&iter);
                }
                rhashtable_walk_stop(&iter);
                rhashtable_walk_exit(&iter);
        } else {
                nfp_ct_merge_tc_entries(ct_entry, zt, zt);
        }

        return 0;
}

static int
nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
{
        struct nfp_fl_ct_map_entry *ct_map_ent;
        struct nfp_fl_ct_flow_entry *ct_entry;
        struct netlink_ext_ack *extack = NULL;

        ASSERT_RTNL();

        extack = flow->common.extack;
        switch (flow->command) {
        case FLOW_CLS_REPLACE:
                /* Netfilter can request offload multiple times for the same
                 * flow - protect against adding duplicates.
                 */
                ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
                                                    nfp_ct_map_params);
                if (!ct_map_ent) {
                        ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
                        if (IS_ERR(ct_entry))
                                return PTR_ERR(ct_entry);
                        ct_entry->type = CT_TYPE_NFT;
                        list_add(&ct_entry->list_node, &zt->nft_flows_list);
                        zt->nft_flows_count++;
                        nfp_ct_merge_nft_with_tc(ct_entry, zt);
                }
                return 0;
        case FLOW_CLS_DESTROY:
                ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
                                                    nfp_ct_map_params);
                return nfp_fl_ct_del_flow(ct_map_ent);
        case FLOW_CLS_STATS:
                return 0;
        default:
                break;
        }
        return -EINVAL;
}

int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
{
        struct flow_cls_offload *flow = type_data;
        struct nfp_fl_ct_zone_entry *zt = cb_priv;
        int err = -EOPNOTSUPP;

        switch (type) {
        case TC_SETUP_CLSFLOWER:
                rtnl_lock();
                err = nfp_fl_ct_offload_nft_flow(zt, flow);
                rtnl_unlock();
                break;
        default:
                return -EOPNOTSUPP;
        }
        return err;
}

static void
nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
{
        struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
        struct nfp_fl_ct_map_entry *ct_map_ent;

        list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
                                 list_node) {
                ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
                                                    &nft_entry->cookie,
                                                    nfp_ct_map_params);
                nfp_fl_ct_del_flow(ct_map_ent);
        }
}

int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
{
        struct nfp_fl_ct_flow_entry *ct_entry;
        struct nfp_fl_ct_zone_entry *zt;
        struct rhashtable *m_table;

        if (!ct_map_ent)
                return -ENOENT;

        zt = ct_map_ent->ct_entry->zt;
        ct_entry = ct_map_ent->ct_entry;
        m_table = &zt->priv->ct_map_table;

        switch (ct_entry->type) {
        case CT_TYPE_PRE_CT:
                zt->pre_ct_count--;
                rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
                                       nfp_ct_map_params);
                nfp_fl_ct_clean_flow_entry(ct_entry);
                kfree(ct_map_ent);

                if (!zt->pre_ct_count) {
                        zt->nft = NULL;
                        nfp_fl_ct_clean_nft_entries(zt);
                }
                break;
        case CT_TYPE_POST_CT:
                zt->post_ct_count--;
                rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
                                       nfp_ct_map_params);
                nfp_fl_ct_clean_flow_entry(ct_entry);
                kfree(ct_map_ent);
                break;
        case CT_TYPE_NFT:
                zt->nft_flows_count--;
                rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
                                       nfp_ct_map_params);
                nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);
                kfree(ct_map_ent);
                break;
        default:
                break;
        }

        return 0;
}