linux/net/netfilter/nf_conntrack_core.c
<<
>>
Prefs
   1/* Connection state tracking for netfilter.  This is separated from,
   2   but required by, the NAT layer; it can also be used by an iptables
   3   extension. */
   4
   5/* (C) 1999-2001 Paul `Rusty' Russell
   6 * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   7 * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   8 * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
   9 *
  10 * This program is free software; you can redistribute it and/or modify
  11 * it under the terms of the GNU General Public License version 2 as
  12 * published by the Free Software Foundation.
  13 */
  14
  15#include <linux/types.h>
  16#include <linux/netfilter.h>
  17#include <linux/module.h>
  18#include <linux/sched.h>
  19#include <linux/skbuff.h>
  20#include <linux/proc_fs.h>
  21#include <linux/vmalloc.h>
  22#include <linux/stddef.h>
  23#include <linux/slab.h>
  24#include <linux/random.h>
  25#include <linux/jhash.h>
  26#include <linux/err.h>
  27#include <linux/percpu.h>
  28#include <linux/moduleparam.h>
  29#include <linux/notifier.h>
  30#include <linux/kernel.h>
  31#include <linux/netdevice.h>
  32#include <linux/socket.h>
  33#include <linux/mm.h>
  34#include <linux/nsproxy.h>
  35#include <linux/rculist_nulls.h>
  36
  37#include <net/netfilter/nf_conntrack.h>
  38#include <net/netfilter/nf_conntrack_l3proto.h>
  39#include <net/netfilter/nf_conntrack_l4proto.h>
  40#include <net/netfilter/nf_conntrack_expect.h>
  41#include <net/netfilter/nf_conntrack_helper.h>
  42#include <net/netfilter/nf_conntrack_core.h>
  43#include <net/netfilter/nf_conntrack_extend.h>
  44#include <net/netfilter/nf_conntrack_acct.h>
  45#include <net/netfilter/nf_conntrack_ecache.h>
  46#include <net/netfilter/nf_conntrack_zones.h>
  47#include <net/netfilter/nf_conntrack_timestamp.h>
  48#include <net/netfilter/nf_conntrack_timeout.h>
  49#include <net/netfilter/nf_conntrack_labels.h>
  50#include <net/netfilter/nf_nat.h>
  51#include <net/netfilter/nf_nat_core.h>
  52#include <net/netfilter/nf_nat_helper.h>
  53
  54#define NF_CONNTRACK_VERSION    "0.5.0"
  55
  56int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
  57                                      enum nf_nat_manip_type manip,
  58                                      const struct nlattr *attr) __read_mostly;
  59EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
  60
  61int (*nf_nat_seq_adjust_hook)(struct sk_buff *skb,
  62                              struct nf_conn *ct,
  63                              enum ip_conntrack_info ctinfo,
  64                              unsigned int protoff);
  65EXPORT_SYMBOL_GPL(nf_nat_seq_adjust_hook);
  66
  67DEFINE_SPINLOCK(nf_conntrack_lock);
  68EXPORT_SYMBOL_GPL(nf_conntrack_lock);
  69
  70unsigned int nf_conntrack_htable_size __read_mostly;
  71EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
  72
  73unsigned int nf_conntrack_max __read_mostly;
  74EXPORT_SYMBOL_GPL(nf_conntrack_max);
  75
  76DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
  77EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
  78
  79unsigned int nf_conntrack_hash_rnd __read_mostly;
  80EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
  81
  82static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
  83{
  84        unsigned int n;
  85
  86        /* The direction must be ignored, so we hash everything up to the
  87         * destination ports (which is a multiple of 4) and treat the last
  88         * three bytes manually.
  89         */
  90        n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
  91        return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
  92                      (((__force __u16)tuple->dst.u.all << 16) |
  93                      tuple->dst.protonum));
  94}
  95
  96static u32 __hash_bucket(u32 hash, unsigned int size)
  97{
  98        return ((u64)hash * size) >> 32;
  99}
 100
 101static u32 hash_bucket(u32 hash, const struct net *net)
 102{
 103        return __hash_bucket(hash, net->ct.htable_size);
 104}
 105
 106static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
 107                                  u16 zone, unsigned int size)
 108{
 109        return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
 110}
 111
 112static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
 113                                       const struct nf_conntrack_tuple *tuple)
 114{
 115        return __hash_conntrack(tuple, zone, net->ct.htable_size);
 116}
 117
 118bool
 119nf_ct_get_tuple(const struct sk_buff *skb,
 120                unsigned int nhoff,
 121                unsigned int dataoff,
 122                u_int16_t l3num,
 123                u_int8_t protonum,
 124                struct nf_conntrack_tuple *tuple,
 125                const struct nf_conntrack_l3proto *l3proto,
 126                const struct nf_conntrack_l4proto *l4proto)
 127{
 128        memset(tuple, 0, sizeof(*tuple));
 129
 130        tuple->src.l3num = l3num;
 131        if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
 132                return false;
 133
 134        tuple->dst.protonum = protonum;
 135        tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 136
 137        return l4proto->pkt_to_tuple(skb, dataoff, tuple);
 138}
 139EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
 140
 141bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 142                       u_int16_t l3num, struct nf_conntrack_tuple *tuple)
 143{
 144        struct nf_conntrack_l3proto *l3proto;
 145        struct nf_conntrack_l4proto *l4proto;
 146        unsigned int protoff;
 147        u_int8_t protonum;
 148        int ret;
 149
 150        rcu_read_lock();
 151
 152        l3proto = __nf_ct_l3proto_find(l3num);
 153        ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
 154        if (ret != NF_ACCEPT) {
 155                rcu_read_unlock();
 156                return false;
 157        }
 158
 159        l4proto = __nf_ct_l4proto_find(l3num, protonum);
 160
 161        ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
 162                              l3proto, l4proto);
 163
 164        rcu_read_unlock();
 165        return ret;
 166}
 167EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 168
 169bool
 170nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 171                   const struct nf_conntrack_tuple *orig,
 172                   const struct nf_conntrack_l3proto *l3proto,
 173                   const struct nf_conntrack_l4proto *l4proto)
 174{
 175        memset(inverse, 0, sizeof(*inverse));
 176
 177        inverse->src.l3num = orig->src.l3num;
 178        if (l3proto->invert_tuple(inverse, orig) == 0)
 179                return false;
 180
 181        inverse->dst.dir = !orig->dst.dir;
 182
 183        inverse->dst.protonum = orig->dst.protonum;
 184        return l4proto->invert_tuple(inverse, orig);
 185}
 186EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 187
 188static void
 189clean_from_lists(struct nf_conn *ct)
 190{
 191        pr_debug("clean_from_lists(%p)\n", ct);
 192        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 193        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 194
 195        /* Destroy all pending expectations */
 196        nf_ct_remove_expectations(ct);
 197}
 198
 199static void
 200destroy_conntrack(struct nf_conntrack *nfct)
 201{
 202        struct nf_conn *ct = (struct nf_conn *)nfct;
 203        struct net *net = nf_ct_net(ct);
 204        struct nf_conntrack_l4proto *l4proto;
 205
 206        pr_debug("destroy_conntrack(%p)\n", ct);
 207        NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
 208        NF_CT_ASSERT(!timer_pending(&ct->timeout));
 209
 210        /* To make sure we don't get any weird locking issues here:
 211         * destroy_conntrack() MUST NOT be called with a write lock
 212         * to nf_conntrack_lock!!! -HW */
 213        rcu_read_lock();
 214        l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 215        if (l4proto && l4proto->destroy)
 216                l4proto->destroy(ct);
 217
 218        rcu_read_unlock();
 219
 220        spin_lock_bh(&nf_conntrack_lock);
 221        /* Expectations will have been removed in clean_from_lists,
 222         * except TFTP can create an expectation on the first packet,
 223         * before connection is in the list, so we need to clean here,
 224         * too. */
 225        nf_ct_remove_expectations(ct);
 226
 227        /* We overload first tuple to link into unconfirmed or dying list.*/
 228        BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 229        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 230
 231        NF_CT_STAT_INC(net, delete);
 232        spin_unlock_bh(&nf_conntrack_lock);
 233
 234        if (ct->master)
 235                nf_ct_put(ct->master);
 236
 237        pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 238        nf_conntrack_free(ct);
 239}
 240
 241void nf_ct_delete_from_lists(struct nf_conn *ct)
 242{
 243        struct net *net = nf_ct_net(ct);
 244
 245        nf_ct_helper_destroy(ct);
 246        spin_lock_bh(&nf_conntrack_lock);
 247        /* Inside lock so preempt is disabled on module removal path.
 248         * Otherwise we can get spurious warnings. */
 249        NF_CT_STAT_INC(net, delete_list);
 250        clean_from_lists(ct);
 251        /* add this conntrack to the dying list */
 252        hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 253                             &net->ct.dying);
 254        spin_unlock_bh(&nf_conntrack_lock);
 255}
 256EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);
 257
 258static void death_by_event(unsigned long ul_conntrack)
 259{
 260        struct nf_conn *ct = (void *)ul_conntrack;
 261        struct net *net = nf_ct_net(ct);
 262        struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
 263
 264        BUG_ON(ecache == NULL);
 265
 266        if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
 267                /* bad luck, let's retry again */
 268                ecache->timeout.expires = jiffies +
 269                        (prandom_u32() % net->ct.sysctl_events_retry_timeout);
 270                add_timer(&ecache->timeout);
 271                return;
 272        }
 273        /* we've got the event delivered, now it's dying */
 274        set_bit(IPS_DYING_BIT, &ct->status);
 275        nf_ct_put(ct);
 276}
 277
 278void nf_ct_dying_timeout(struct nf_conn *ct)
 279{
 280        struct net *net = nf_ct_net(ct);
 281        struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
 282
 283        BUG_ON(ecache == NULL);
 284
 285        /* set a new timer to retry event delivery */
 286        setup_timer(&ecache->timeout, death_by_event, (unsigned long)ct);
 287        ecache->timeout.expires = jiffies +
 288                (prandom_u32() % net->ct.sysctl_events_retry_timeout);
 289        add_timer(&ecache->timeout);
 290}
 291EXPORT_SYMBOL_GPL(nf_ct_dying_timeout);
 292
 293static void death_by_timeout(unsigned long ul_conntrack)
 294{
 295        struct nf_conn *ct = (void *)ul_conntrack;
 296        struct nf_conn_tstamp *tstamp;
 297
 298        tstamp = nf_conn_tstamp_find(ct);
 299        if (tstamp && tstamp->stop == 0)
 300                tstamp->stop = ktime_to_ns(ktime_get_real());
 301
 302        if (!test_bit(IPS_DYING_BIT, &ct->status) &&
 303            unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
 304                /* destroy event was not delivered */
 305                nf_ct_delete_from_lists(ct);
 306                nf_ct_dying_timeout(ct);
 307                return;
 308        }
 309        set_bit(IPS_DYING_BIT, &ct->status);
 310        nf_ct_delete_from_lists(ct);
 311        nf_ct_put(ct);
 312}
 313
 314/*
 315 * Warning :
 316 * - Caller must take a reference on returned object
 317 *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 318 * OR
 319 * - Caller must lock nf_conntrack_lock before calling this function
 320 */
 321static struct nf_conntrack_tuple_hash *
 322____nf_conntrack_find(struct net *net, u16 zone,
 323                      const struct nf_conntrack_tuple *tuple, u32 hash)
 324{
 325        struct nf_conntrack_tuple_hash *h;
 326        struct hlist_nulls_node *n;
 327        unsigned int bucket = hash_bucket(hash, net);
 328
 329        /* Disable BHs the entire time since we normally need to disable them
 330         * at least once for the stats anyway.
 331         */
 332        local_bh_disable();
 333begin:
 334        hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
 335                if (nf_ct_tuple_equal(tuple, &h->tuple) &&
 336                    nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
 337                        NF_CT_STAT_INC(net, found);
 338                        local_bh_enable();
 339                        return h;
 340                }
 341                NF_CT_STAT_INC(net, searched);
 342        }
 343        /*
 344         * if the nulls value we got at the end of this lookup is
 345         * not the expected one, we must restart lookup.
 346         * We probably met an item that was moved to another chain.
 347         */
 348        if (get_nulls_value(n) != bucket) {
 349                NF_CT_STAT_INC(net, search_restart);
 350                goto begin;
 351        }
 352        local_bh_enable();
 353
 354        return NULL;
 355}
 356
 357struct nf_conntrack_tuple_hash *
 358__nf_conntrack_find(struct net *net, u16 zone,
 359                    const struct nf_conntrack_tuple *tuple)
 360{
 361        return ____nf_conntrack_find(net, zone, tuple,
 362                                     hash_conntrack_raw(tuple, zone));
 363}
 364EXPORT_SYMBOL_GPL(__nf_conntrack_find);
 365
 366/* Find a connection corresponding to a tuple. */
 367static struct nf_conntrack_tuple_hash *
 368__nf_conntrack_find_get(struct net *net, u16 zone,
 369                        const struct nf_conntrack_tuple *tuple, u32 hash)
 370{
 371        struct nf_conntrack_tuple_hash *h;
 372        struct nf_conn *ct;
 373
 374        rcu_read_lock();
 375begin:
 376        h = ____nf_conntrack_find(net, zone, tuple, hash);
 377        if (h) {
 378                ct = nf_ct_tuplehash_to_ctrack(h);
 379                if (unlikely(nf_ct_is_dying(ct) ||
 380                             !atomic_inc_not_zero(&ct->ct_general.use)))
 381                        h = NULL;
 382                else {
 383                        if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
 384                                     nf_ct_zone(ct) != zone)) {
 385                                nf_ct_put(ct);
 386                                goto begin;
 387                        }
 388                }
 389        }
 390        rcu_read_unlock();
 391
 392        return h;
 393}
 394
 395struct nf_conntrack_tuple_hash *
 396nf_conntrack_find_get(struct net *net, u16 zone,
 397                      const struct nf_conntrack_tuple *tuple)
 398{
 399        return __nf_conntrack_find_get(net, zone, tuple,
 400                                       hash_conntrack_raw(tuple, zone));
 401}
 402EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 403
 404static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 405                                       unsigned int hash,
 406                                       unsigned int repl_hash)
 407{
 408        struct net *net = nf_ct_net(ct);
 409
 410        hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 411                           &net->ct.hash[hash]);
 412        hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 413                           &net->ct.hash[repl_hash]);
 414}
 415
 416int
 417nf_conntrack_hash_check_insert(struct nf_conn *ct)
 418{
 419        struct net *net = nf_ct_net(ct);
 420        unsigned int hash, repl_hash;
 421        struct nf_conntrack_tuple_hash *h;
 422        struct hlist_nulls_node *n;
 423        u16 zone;
 424
 425        zone = nf_ct_zone(ct);
 426        hash = hash_conntrack(net, zone,
 427                              &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 428        repl_hash = hash_conntrack(net, zone,
 429                                   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 430
 431        spin_lock_bh(&nf_conntrack_lock);
 432
 433        /* See if there's one in the list already, including reverse */
 434        hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
 435                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 436                                      &h->tuple) &&
 437                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 438                        goto out;
 439        hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
 440                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 441                                      &h->tuple) &&
 442                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 443                        goto out;
 444
 445        add_timer(&ct->timeout);
 446        nf_conntrack_get(&ct->ct_general);
 447        __nf_conntrack_hash_insert(ct, hash, repl_hash);
 448        NF_CT_STAT_INC(net, insert);
 449        spin_unlock_bh(&nf_conntrack_lock);
 450
 451        return 0;
 452
 453out:
 454        NF_CT_STAT_INC(net, insert_failed);
 455        spin_unlock_bh(&nf_conntrack_lock);
 456        return -EEXIST;
 457}
 458EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 459
 460/* Confirm a connection given skb; places it in hash table */
 461int
 462__nf_conntrack_confirm(struct sk_buff *skb)
 463{
 464        unsigned int hash, repl_hash;
 465        struct nf_conntrack_tuple_hash *h;
 466        struct nf_conn *ct;
 467        struct nf_conn_help *help;
 468        struct nf_conn_tstamp *tstamp;
 469        struct hlist_nulls_node *n;
 470        enum ip_conntrack_info ctinfo;
 471        struct net *net;
 472        u16 zone;
 473
 474        ct = nf_ct_get(skb, &ctinfo);
 475        net = nf_ct_net(ct);
 476
 477        /* ipt_REJECT uses nf_conntrack_attach to attach related
 478           ICMP/TCP RST packets in other direction.  Actual packet
 479           which created connection will be IP_CT_NEW or for an
 480           expected connection, IP_CT_RELATED. */
 481        if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
 482                return NF_ACCEPT;
 483
 484        zone = nf_ct_zone(ct);
 485        /* reuse the hash saved before */
 486        hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
 487        hash = hash_bucket(hash, net);
 488        repl_hash = hash_conntrack(net, zone,
 489                                   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 490
 491        /* We're not in hash table, and we refuse to set up related
 492           connections for unconfirmed conns.  But packet copies and
 493           REJECT will give spurious warnings here. */
 494        /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
 495
 496        /* No external references means no one else could have
 497           confirmed us. */
 498        NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
 499        pr_debug("Confirming conntrack %p\n", ct);
 500
 501        spin_lock_bh(&nf_conntrack_lock);
 502
 503        /* We have to check the DYING flag inside the lock to prevent
 504           a race against nf_ct_get_next_corpse() possibly called from
 505           user context, else we insert an already 'dead' hash, blocking
 506           further use of that particular connection -JM */
 507
 508        if (unlikely(nf_ct_is_dying(ct))) {
 509                spin_unlock_bh(&nf_conntrack_lock);
 510                return NF_ACCEPT;
 511        }
 512
 513        /* See if there's one in the list already, including reverse:
 514           NAT could have grabbed it without realizing, since we're
 515           not in the hash.  If there is, we lost race. */
 516        hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
 517                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 518                                      &h->tuple) &&
 519                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 520                        goto out;
 521        hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
 522                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 523                                      &h->tuple) &&
 524                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 525                        goto out;
 526
 527        /* Remove from unconfirmed list */
 528        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 529
 530        /* Timer relative to confirmation time, not original
 531           setting time, otherwise we'd get timer wrap in
 532           weird delay cases. */
 533        ct->timeout.expires += jiffies;
 534        add_timer(&ct->timeout);
 535        atomic_inc(&ct->ct_general.use);
 536        ct->status |= IPS_CONFIRMED;
 537
 538        /* set conntrack timestamp, if enabled. */
 539        tstamp = nf_conn_tstamp_find(ct);
 540        if (tstamp) {
 541                if (skb->tstamp.tv64 == 0)
 542                        __net_timestamp(skb);
 543
 544                tstamp->start = ktime_to_ns(skb->tstamp);
 545        }
 546        /* Since the lookup is lockless, hash insertion must be done after
 547         * starting the timer and setting the CONFIRMED bit. The RCU barriers
 548         * guarantee that no other CPU can find the conntrack before the above
 549         * stores are visible.
 550         */
 551        __nf_conntrack_hash_insert(ct, hash, repl_hash);
 552        NF_CT_STAT_INC(net, insert);
 553        spin_unlock_bh(&nf_conntrack_lock);
 554
 555        help = nfct_help(ct);
 556        if (help && help->helper)
 557                nf_conntrack_event_cache(IPCT_HELPER, ct);
 558
 559        nf_conntrack_event_cache(master_ct(ct) ?
 560                                 IPCT_RELATED : IPCT_NEW, ct);
 561        return NF_ACCEPT;
 562
 563out:
 564        NF_CT_STAT_INC(net, insert_failed);
 565        spin_unlock_bh(&nf_conntrack_lock);
 566        return NF_DROP;
 567}
 568EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
 569
 570/* Returns true if a connection correspondings to the tuple (required
 571   for NAT). */
 572int
 573nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
 574                         const struct nf_conn *ignored_conntrack)
 575{
 576        struct net *net = nf_ct_net(ignored_conntrack);
 577        struct nf_conntrack_tuple_hash *h;
 578        struct hlist_nulls_node *n;
 579        struct nf_conn *ct;
 580        u16 zone = nf_ct_zone(ignored_conntrack);
 581        unsigned int hash = hash_conntrack(net, zone, tuple);
 582
 583        /* Disable BHs the entire time since we need to disable them at
 584         * least once for the stats anyway.
 585         */
 586        rcu_read_lock_bh();
 587        hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
 588                ct = nf_ct_tuplehash_to_ctrack(h);
 589                if (ct != ignored_conntrack &&
 590                    nf_ct_tuple_equal(tuple, &h->tuple) &&
 591                    nf_ct_zone(ct) == zone) {
 592                        NF_CT_STAT_INC(net, found);
 593                        rcu_read_unlock_bh();
 594                        return 1;
 595                }
 596                NF_CT_STAT_INC(net, searched);
 597        }
 598        rcu_read_unlock_bh();
 599
 600        return 0;
 601}
 602EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
 603
 604#define NF_CT_EVICTION_RANGE    8
 605
 606/* There's a small race here where we may free a just-assured
 607   connection.  Too bad: we're in trouble anyway. */
 608static noinline int early_drop(struct net *net, unsigned int hash)
 609{
 610        /* Use oldest entry, which is roughly LRU */
 611        struct nf_conntrack_tuple_hash *h;
 612        struct nf_conn *ct = NULL, *tmp;
 613        struct hlist_nulls_node *n;
 614        unsigned int i, cnt = 0;
 615        int dropped = 0;
 616
 617        rcu_read_lock();
 618        for (i = 0; i < net->ct.htable_size; i++) {
 619                hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
 620                                         hnnode) {
 621                        tmp = nf_ct_tuplehash_to_ctrack(h);
 622                        if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
 623                                ct = tmp;
 624                        cnt++;
 625                }
 626
 627                if (ct != NULL) {
 628                        if (likely(!nf_ct_is_dying(ct) &&
 629                                   atomic_inc_not_zero(&ct->ct_general.use)))
 630                                break;
 631                        else
 632                                ct = NULL;
 633                }
 634
 635                if (cnt >= NF_CT_EVICTION_RANGE)
 636                        break;
 637
 638                hash = (hash + 1) % net->ct.htable_size;
 639        }
 640        rcu_read_unlock();
 641
 642        if (!ct)
 643                return dropped;
 644
 645        if (del_timer(&ct->timeout)) {
 646                death_by_timeout((unsigned long)ct);
 647                /* Check if we indeed killed this entry. Reliable event
 648                   delivery may have inserted it into the dying list. */
 649                if (test_bit(IPS_DYING_BIT, &ct->status)) {
 650                        dropped = 1;
 651                        NF_CT_STAT_INC_ATOMIC(net, early_drop);
 652                }
 653        }
 654        nf_ct_put(ct);
 655        return dropped;
 656}
 657
 658void init_nf_conntrack_hash_rnd(void)
 659{
 660        unsigned int rand;
 661
 662        /*
 663         * Why not initialize nf_conntrack_rnd in a "init()" function ?
 664         * Because there isn't enough entropy when system initializing,
 665         * and we initialize it as late as possible.
 666         */
 667        do {
 668                get_random_bytes(&rand, sizeof(rand));
 669        } while (!rand);
 670        cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
 671}
 672
 673static struct nf_conn *
 674__nf_conntrack_alloc(struct net *net, u16 zone,
 675                     const struct nf_conntrack_tuple *orig,
 676                     const struct nf_conntrack_tuple *repl,
 677                     gfp_t gfp, u32 hash)
 678{
 679        struct nf_conn *ct;
 680
 681        if (unlikely(!nf_conntrack_hash_rnd)) {
 682                init_nf_conntrack_hash_rnd();
 683                /* recompute the hash as nf_conntrack_hash_rnd is initialized */
 684                hash = hash_conntrack_raw(orig, zone);
 685        }
 686
 687        /* We don't want any race condition at early drop stage */
 688        atomic_inc(&net->ct.count);
 689
 690        if (nf_conntrack_max &&
 691            unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
 692                if (!early_drop(net, hash_bucket(hash, net))) {
 693                        atomic_dec(&net->ct.count);
 694                        net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
 695                        return ERR_PTR(-ENOMEM);
 696                }
 697        }
 698
 699        /*
 700         * Do not use kmem_cache_zalloc(), as this cache uses
 701         * SLAB_DESTROY_BY_RCU.
 702         */
 703        ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
 704        if (ct == NULL) {
 705                atomic_dec(&net->ct.count);
 706                return ERR_PTR(-ENOMEM);
 707        }
 708        /*
 709         * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
 710         * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
 711         */
 712        memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
 713               offsetof(struct nf_conn, proto) -
 714               offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));
 715        spin_lock_init(&ct->lock);
 716        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
 717        ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
 718        ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
 719        /* save hash for reusing when confirming */
 720        *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
 721        /* Don't set timer yet: wait for confirmation */
 722        setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
 723        write_pnet(&ct->ct_net, net);
 724#ifdef CONFIG_NF_CONNTRACK_ZONES
 725        if (zone) {
 726                struct nf_conntrack_zone *nf_ct_zone;
 727
 728                nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
 729                if (!nf_ct_zone)
 730                        goto out_free;
 731                nf_ct_zone->id = zone;
 732        }
 733#endif
 734        /*
 735         * changes to lookup keys must be done before setting refcnt to 1
 736         */
 737        smp_wmb();
 738        atomic_set(&ct->ct_general.use, 1);
 739        return ct;
 740
 741#ifdef CONFIG_NF_CONNTRACK_ZONES
 742out_free:
 743        atomic_dec(&net->ct.count);
 744        kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
 745        return ERR_PTR(-ENOMEM);
 746#endif
 747}
 748
 749struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
 750                                   const struct nf_conntrack_tuple *orig,
 751                                   const struct nf_conntrack_tuple *repl,
 752                                   gfp_t gfp)
 753{
 754        return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
 755}
 756EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
 757
 758void nf_conntrack_free(struct nf_conn *ct)
 759{
 760        struct net *net = nf_ct_net(ct);
 761
 762        nf_ct_ext_destroy(ct);
 763        atomic_dec(&net->ct.count);
 764        nf_ct_ext_free(ct);
 765        kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
 766}
 767EXPORT_SYMBOL_GPL(nf_conntrack_free);
 768
 769
 770/* Allocate a new conntrack: we return -ENOMEM if classification
 771   failed due to stress.  Otherwise it really is unclassifiable. */
 772static struct nf_conntrack_tuple_hash *
 773init_conntrack(struct net *net, struct nf_conn *tmpl,
 774               const struct nf_conntrack_tuple *tuple,
 775               struct nf_conntrack_l3proto *l3proto,
 776               struct nf_conntrack_l4proto *l4proto,
 777               struct sk_buff *skb,
 778               unsigned int dataoff, u32 hash)
 779{
 780        struct nf_conn *ct;
 781        struct nf_conn_help *help;
 782        struct nf_conntrack_tuple repl_tuple;
 783        struct nf_conntrack_ecache *ecache;
 784        struct nf_conntrack_expect *exp;
 785        u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
 786        struct nf_conn_timeout *timeout_ext;
 787        unsigned int *timeouts;
 788
 789        if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
 790                pr_debug("Can't invert tuple.\n");
 791                return NULL;
 792        }
 793
 794        ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
 795                                  hash);
 796        if (IS_ERR(ct))
 797                return (struct nf_conntrack_tuple_hash *)ct;
 798
 799        timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
 800        if (timeout_ext)
 801                timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
 802        else
 803                timeouts = l4proto->get_timeouts(net);
 804
 805        if (!l4proto->new(ct, skb, dataoff, timeouts)) {
 806                nf_conntrack_free(ct);
 807                pr_debug("init conntrack: can't track with proto module\n");
 808                return NULL;
 809        }
 810
 811        if (timeout_ext)
 812                nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
 813
 814        nf_ct_acct_ext_add(ct, GFP_ATOMIC);
 815        nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
 816        nf_ct_labels_ext_add(ct);
 817
 818        ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
 819        nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
 820                                 ecache ? ecache->expmask : 0,
 821                             GFP_ATOMIC);
 822
 823        spin_lock_bh(&nf_conntrack_lock);
 824        exp = nf_ct_find_expectation(net, zone, tuple);
 825        if (exp) {
 826                pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
 827                         ct, exp);
 828                /* Welcome, Mr. Bond.  We've been expecting you... */
 829                __set_bit(IPS_EXPECTED_BIT, &ct->status);
 830                ct->master = exp->master;
 831                if (exp->helper) {
 832                        help = nf_ct_helper_ext_add(ct, exp->helper,
 833                                                    GFP_ATOMIC);
 834                        if (help)
 835                                rcu_assign_pointer(help->helper, exp->helper);
 836                }
 837
 838#ifdef CONFIG_NF_CONNTRACK_MARK
 839                ct->mark = exp->master->mark;
 840#endif
 841#ifdef CONFIG_NF_CONNTRACK_SECMARK
 842                ct->secmark = exp->master->secmark;
 843#endif
 844                nf_conntrack_get(&ct->master->ct_general);
 845                NF_CT_STAT_INC(net, expect_new);
 846        } else {
 847                __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
 848                NF_CT_STAT_INC(net, new);
 849        }
 850
 851        /* Overload tuple linked list to put us in unconfirmed list. */
 852        hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 853                       &net->ct.unconfirmed);
 854
 855        spin_unlock_bh(&nf_conntrack_lock);
 856
 857        if (exp) {
 858                if (exp->expectfn)
 859                        exp->expectfn(ct, exp);
 860                nf_ct_expect_put(exp);
 861        }
 862
 863        return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
 864}
 865
 866/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
 867static inline struct nf_conn *
 868resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
 869                  struct sk_buff *skb,
 870                  unsigned int dataoff,
 871                  u_int16_t l3num,
 872                  u_int8_t protonum,
 873                  struct nf_conntrack_l3proto *l3proto,
 874                  struct nf_conntrack_l4proto *l4proto,
 875                  int *set_reply,
 876                  enum ip_conntrack_info *ctinfo)
 877{
 878        struct nf_conntrack_tuple tuple;
 879        struct nf_conntrack_tuple_hash *h;
 880        struct nf_conn *ct;
 881        u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
 882        u32 hash;
 883
 884        if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
 885                             dataoff, l3num, protonum, &tuple, l3proto,
 886                             l4proto)) {
 887                pr_debug("resolve_normal_ct: Can't get tuple\n");
 888                return NULL;
 889        }
 890
 891        /* look for tuple match */
 892        hash = hash_conntrack_raw(&tuple, zone);
 893        h = __nf_conntrack_find_get(net, zone, &tuple, hash);
 894        if (!h) {
 895                h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
 896                                   skb, dataoff, hash);
 897                if (!h)
 898                        return NULL;
 899                if (IS_ERR(h))
 900                        return (void *)h;
 901        }
 902        ct = nf_ct_tuplehash_to_ctrack(h);
 903
 904        /* It exists; we have (non-exclusive) reference. */
 905        if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
 906                *ctinfo = IP_CT_ESTABLISHED_REPLY;
 907                /* Please set reply bit if this packet OK */
 908                *set_reply = 1;
 909        } else {
 910                /* Once we've had two way comms, always ESTABLISHED. */
 911                if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
 912                        pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
 913                        *ctinfo = IP_CT_ESTABLISHED;
 914                } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
 915                        pr_debug("nf_conntrack_in: related packet for %p\n",
 916                                 ct);
 917                        *ctinfo = IP_CT_RELATED;
 918                } else {
 919                        pr_debug("nf_conntrack_in: new packet for %p\n", ct);
 920                        *ctinfo = IP_CT_NEW;
 921                }
 922                *set_reply = 0;
 923        }
 924        skb->nfct = &ct->ct_general;
 925        skb->nfctinfo = *ctinfo;
 926        return ct;
 927}
 928
 929unsigned int
 930nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
 931                struct sk_buff *skb)
 932{
 933        struct nf_conn *ct, *tmpl = NULL;
 934        enum ip_conntrack_info ctinfo;
 935        struct nf_conntrack_l3proto *l3proto;
 936        struct nf_conntrack_l4proto *l4proto;
 937        unsigned int *timeouts;
 938        unsigned int dataoff;
 939        u_int8_t protonum;
 940        int set_reply = 0;
 941        int ret;
 942
 943        if (skb->nfct) {
 944                /* Previously seen (loopback or untracked)?  Ignore. */
 945                tmpl = (struct nf_conn *)skb->nfct;
 946                if (!nf_ct_is_template(tmpl)) {
 947                        NF_CT_STAT_INC_ATOMIC(net, ignore);
 948                        return NF_ACCEPT;
 949                }
 950                skb->nfct = NULL;
 951        }
 952
 953        /* rcu_read_lock()ed by nf_hook_slow */
 954        l3proto = __nf_ct_l3proto_find(pf);
 955        ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
 956                                   &dataoff, &protonum);
 957        if (ret <= 0) {
 958                pr_debug("not prepared to track yet or error occurred\n");
 959                NF_CT_STAT_INC_ATOMIC(net, error);
 960                NF_CT_STAT_INC_ATOMIC(net, invalid);
 961                ret = -ret;
 962                goto out;
 963        }
 964
 965        l4proto = __nf_ct_l4proto_find(pf, protonum);
 966
 967        /* It may be an special packet, error, unclean...
 968         * inverse of the return code tells to the netfilter
 969         * core what to do with the packet. */
 970        if (l4proto->error != NULL) {
 971                ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
 972                                     pf, hooknum);
 973                if (ret <= 0) {
 974                        NF_CT_STAT_INC_ATOMIC(net, error);
 975                        NF_CT_STAT_INC_ATOMIC(net, invalid);
 976                        ret = -ret;
 977                        goto out;
 978                }
 979                /* ICMP[v6] protocol trackers may assign one conntrack. */
 980                if (skb->nfct)
 981                        goto out;
 982        }
 983
 984        ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
 985                               l3proto, l4proto, &set_reply, &ctinfo);
 986        if (!ct) {
 987                /* Not valid part of a connection */
 988                NF_CT_STAT_INC_ATOMIC(net, invalid);
 989                ret = NF_ACCEPT;
 990                goto out;
 991        }
 992
 993        if (IS_ERR(ct)) {
 994                /* Too stressed to deal. */
 995                NF_CT_STAT_INC_ATOMIC(net, drop);
 996                ret = NF_DROP;
 997                goto out;
 998        }
 999
1000        NF_CT_ASSERT(skb->nfct);
1001
1002        /* Decide what timeout policy we want to apply to this flow. */
1003        timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1004
1005        ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1006        if (ret <= 0) {
1007                /* Invalid: inverse of the return code tells
1008                 * the netfilter core what to do */
1009                pr_debug("nf_conntrack_in: Can't track with proto module\n");
1010                nf_conntrack_put(skb->nfct);
1011                skb->nfct = NULL;
1012                NF_CT_STAT_INC_ATOMIC(net, invalid);
1013                if (ret == -NF_DROP)
1014                        NF_CT_STAT_INC_ATOMIC(net, drop);
1015                ret = -ret;
1016                goto out;
1017        }
1018
1019        if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1020                nf_conntrack_event_cache(IPCT_REPLY, ct);
1021out:
1022        if (tmpl) {
1023                /* Special case: we have to repeat this hook, assign the
1024                 * template again to this packet. We assume that this packet
1025                 * has no conntrack assigned. This is used by nf_ct_tcp. */
1026                if (ret == NF_REPEAT)
1027                        skb->nfct = (struct nf_conntrack *)tmpl;
1028                else
1029                        nf_ct_put(tmpl);
1030        }
1031
1032        return ret;
1033}
1034EXPORT_SYMBOL_GPL(nf_conntrack_in);
1035
1036bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1037                          const struct nf_conntrack_tuple *orig)
1038{
1039        bool ret;
1040
1041        rcu_read_lock();
1042        ret = nf_ct_invert_tuple(inverse, orig,
1043                                 __nf_ct_l3proto_find(orig->src.l3num),
1044                                 __nf_ct_l4proto_find(orig->src.l3num,
1045                                                      orig->dst.protonum));
1046        rcu_read_unlock();
1047        return ret;
1048}
1049EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1050
1051/* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1052   implicitly racy: see __nf_conntrack_confirm */
1053void nf_conntrack_alter_reply(struct nf_conn *ct,
1054                              const struct nf_conntrack_tuple *newreply)
1055{
1056        struct nf_conn_help *help = nfct_help(ct);
1057
1058        /* Should be unconfirmed, so not in hash table yet */
1059        NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1060
1061        pr_debug("Altering reply tuple of %p to ", ct);
1062        nf_ct_dump_tuple(newreply);
1063
1064        ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1065        if (ct->master || (help && !hlist_empty(&help->expectations)))
1066                return;
1067
1068        rcu_read_lock();
1069        __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1070        rcu_read_unlock();
1071}
1072EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1073
1074/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1075void __nf_ct_refresh_acct(struct nf_conn *ct,
1076                          enum ip_conntrack_info ctinfo,
1077                          const struct sk_buff *skb,
1078                          unsigned long extra_jiffies,
1079                          int do_acct)
1080{
1081        NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1082        NF_CT_ASSERT(skb);
1083
1084        /* Only update if this is not a fixed timeout */
1085        if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1086                goto acct;
1087
1088        /* If not in hash table, timer will not be active yet */
1089        if (!nf_ct_is_confirmed(ct)) {
1090                ct->timeout.expires = extra_jiffies;
1091        } else {
1092                unsigned long newtime = jiffies + extra_jiffies;
1093
1094                /* Only update the timeout if the new timeout is at least
1095                   HZ jiffies from the old timeout. Need del_timer for race
1096                   avoidance (may already be dying). */
1097                if (newtime - ct->timeout.expires >= HZ)
1098                        mod_timer_pending(&ct->timeout, newtime);
1099        }
1100
1101acct:
1102        if (do_acct) {
1103                struct nf_conn_counter *acct;
1104
1105                acct = nf_conn_acct_find(ct);
1106                if (acct) {
1107                        atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1108                        atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);
1109                }
1110        }
1111}
1112EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1113
1114bool __nf_ct_kill_acct(struct nf_conn *ct,
1115                       enum ip_conntrack_info ctinfo,
1116                       const struct sk_buff *skb,
1117                       int do_acct)
1118{
1119        if (do_acct) {
1120                struct nf_conn_counter *acct;
1121
1122                acct = nf_conn_acct_find(ct);
1123                if (acct) {
1124                        atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1125                        atomic64_add(skb->len - skb_network_offset(skb),
1126                                     &acct[CTINFO2DIR(ctinfo)].bytes);
1127                }
1128        }
1129
1130        if (del_timer(&ct->timeout)) {
1131                ct->timeout.function((unsigned long)ct);
1132                return true;
1133        }
1134        return false;
1135}
1136EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
1137
1138#ifdef CONFIG_NF_CONNTRACK_ZONES
1139static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
1140        .len    = sizeof(struct nf_conntrack_zone),
1141        .align  = __alignof__(struct nf_conntrack_zone),
1142        .id     = NF_CT_EXT_ZONE,
1143};
1144#endif
1145
1146#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1147
1148#include <linux/netfilter/nfnetlink.h>
1149#include <linux/netfilter/nfnetlink_conntrack.h>
1150#include <linux/mutex.h>
1151
1152/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1153 * in ip_conntrack_core, since we don't want the protocols to autoload
1154 * or depend on ctnetlink */
1155int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1156                               const struct nf_conntrack_tuple *tuple)
1157{
1158        if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1159            nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1160                goto nla_put_failure;
1161        return 0;
1162
1163nla_put_failure:
1164        return -1;
1165}
1166EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1167
1168const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1169        [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1170        [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1171};
1172EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1173
1174int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1175                               struct nf_conntrack_tuple *t)
1176{
1177        if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1178                return -EINVAL;
1179
1180        t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1181        t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1182
1183        return 0;
1184}
1185EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1186
1187int nf_ct_port_nlattr_tuple_size(void)
1188{
1189        return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1190}
1191EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1192#endif
1193
1194/* Used by ipt_REJECT and ip6t_REJECT. */
1195static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
1196{
1197        struct nf_conn *ct;
1198        enum ip_conntrack_info ctinfo;
1199
1200        /* This ICMP is in reverse direction to the packet which caused it */
1201        ct = nf_ct_get(skb, &ctinfo);
1202        if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1203                ctinfo = IP_CT_RELATED_REPLY;
1204        else
1205                ctinfo = IP_CT_RELATED;
1206
1207        /* Attach to new skbuff, and increment count */
1208        nskb->nfct = &ct->ct_general;
1209        nskb->nfctinfo = ctinfo;
1210        nf_conntrack_get(nskb->nfct);
1211}
1212
1213/* Bring out ya dead! */
1214static struct nf_conn *
1215get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1216                void *data, unsigned int *bucket)
1217{
1218        struct nf_conntrack_tuple_hash *h;
1219        struct nf_conn *ct;
1220        struct hlist_nulls_node *n;
1221
1222        spin_lock_bh(&nf_conntrack_lock);
1223        for (; *bucket < net->ct.htable_size; (*bucket)++) {
1224                hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
1225                        if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1226                                continue;
1227                        ct = nf_ct_tuplehash_to_ctrack(h);
1228                        if (iter(ct, data))
1229                                goto found;
1230                }
1231        }
1232        hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
1233                ct = nf_ct_tuplehash_to_ctrack(h);
1234                if (iter(ct, data))
1235                        set_bit(IPS_DYING_BIT, &ct->status);
1236        }
1237        spin_unlock_bh(&nf_conntrack_lock);
1238        return NULL;
1239found:
1240        atomic_inc(&ct->ct_general.use);
1241        spin_unlock_bh(&nf_conntrack_lock);
1242        return ct;
1243}
1244
1245void nf_ct_iterate_cleanup(struct net *net,
1246                           int (*iter)(struct nf_conn *i, void *data),
1247                           void *data)
1248{
1249        struct nf_conn *ct;
1250        unsigned int bucket = 0;
1251
1252        while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1253                /* Time to push up daises... */
1254                if (del_timer(&ct->timeout))
1255                        death_by_timeout((unsigned long)ct);
1256                /* ... else the timer will get him soon. */
1257
1258                nf_ct_put(ct);
1259        }
1260}
1261EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1262
1263struct __nf_ct_flush_report {
1264        u32 portid;
1265        int report;
1266};
1267
1268static int kill_report(struct nf_conn *i, void *data)
1269{
1270        struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
1271        struct nf_conn_tstamp *tstamp;
1272
1273        tstamp = nf_conn_tstamp_find(i);
1274        if (tstamp && tstamp->stop == 0)
1275                tstamp->stop = ktime_to_ns(ktime_get_real());
1276
1277        /* If we fail to deliver the event, death_by_timeout() will retry */
1278        if (nf_conntrack_event_report(IPCT_DESTROY, i,
1279                                      fr->portid, fr->report) < 0)
1280                return 1;
1281
1282        /* Avoid the delivery of the destroy event in death_by_timeout(). */
1283        set_bit(IPS_DYING_BIT, &i->status);
1284        return 1;
1285}
1286
1287static int kill_all(struct nf_conn *i, void *data)
1288{
1289        return 1;
1290}
1291
1292void nf_ct_free_hashtable(void *hash, unsigned int size)
1293{
1294        if (is_vmalloc_addr(hash))
1295                vfree(hash);
1296        else
1297                free_pages((unsigned long)hash,
1298                           get_order(sizeof(struct hlist_head) * size));
1299}
1300EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1301
1302void nf_conntrack_flush_report(struct net *net, u32 portid, int report)
1303{
1304        struct __nf_ct_flush_report fr = {
1305                .portid = portid,
1306                .report = report,
1307        };
1308        nf_ct_iterate_cleanup(net, kill_report, &fr);
1309}
1310EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);
1311
1312static void nf_ct_release_dying_list(struct net *net)
1313{
1314        struct nf_conntrack_tuple_hash *h;
1315        struct nf_conn *ct;
1316        struct hlist_nulls_node *n;
1317
1318        spin_lock_bh(&nf_conntrack_lock);
1319        hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
1320                ct = nf_ct_tuplehash_to_ctrack(h);
1321                /* never fails to remove them, no listeners at this point */
1322                nf_ct_kill(ct);
1323        }
1324        spin_unlock_bh(&nf_conntrack_lock);
1325}
1326
1327static int untrack_refs(void)
1328{
1329        int cnt = 0, cpu;
1330
1331        for_each_possible_cpu(cpu) {
1332                struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1333
1334                cnt += atomic_read(&ct->ct_general.use) - 1;
1335        }
1336        return cnt;
1337}
1338
1339void nf_conntrack_cleanup_start(void)
1340{
1341        RCU_INIT_POINTER(ip_ct_attach, NULL);
1342}
1343
1344void nf_conntrack_cleanup_end(void)
1345{
1346        RCU_INIT_POINTER(nf_ct_destroy, NULL);
1347        while (untrack_refs() > 0)
1348                schedule();
1349
1350#ifdef CONFIG_NF_CONNTRACK_ZONES
1351        nf_ct_extend_unregister(&nf_ct_zone_extend);
1352#endif
1353        nf_conntrack_proto_fini();
1354        nf_conntrack_labels_fini();
1355        nf_conntrack_helper_fini();
1356        nf_conntrack_timeout_fini();
1357        nf_conntrack_ecache_fini();
1358        nf_conntrack_tstamp_fini();
1359        nf_conntrack_acct_fini();
1360        nf_conntrack_expect_fini();
1361}
1362
1363/*
1364 * Mishearing the voices in his head, our hero wonders how he's
1365 * supposed to kill the mall.
1366 */
1367void nf_conntrack_cleanup_net(struct net *net)
1368{
1369        LIST_HEAD(single);
1370
1371        list_add(&net->exit_list, &single);
1372        nf_conntrack_cleanup_net_list(&single);
1373}
1374
1375void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
1376{
1377        int busy;
1378        struct net *net;
1379
1380        /*
1381         * This makes sure all current packets have passed through
1382         *  netfilter framework.  Roll on, two-stage module
1383         *  delete...
1384         */
1385        synchronize_net();
1386i_see_dead_people:
1387        busy = 0;
1388        list_for_each_entry(net, net_exit_list, exit_list) {
1389                nf_ct_iterate_cleanup(net, kill_all, NULL);
1390                nf_ct_release_dying_list(net);
1391                if (atomic_read(&net->ct.count) != 0)
1392                        busy = 1;
1393        }
1394        if (busy) {
1395                schedule();
1396                goto i_see_dead_people;
1397        }
1398
1399        list_for_each_entry(net, net_exit_list, exit_list) {
1400                nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1401                nf_conntrack_proto_pernet_fini(net);
1402                nf_conntrack_helper_pernet_fini(net);
1403                nf_conntrack_ecache_pernet_fini(net);
1404                nf_conntrack_tstamp_pernet_fini(net);
1405                nf_conntrack_acct_pernet_fini(net);
1406                nf_conntrack_expect_pernet_fini(net);
1407                kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1408                kfree(net->ct.slabname);
1409                free_percpu(net->ct.stat);
1410        }
1411}
1412
1413void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1414{
1415        struct hlist_nulls_head *hash;
1416        unsigned int nr_slots, i;
1417        size_t sz;
1418
1419        BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1420        nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1421        sz = nr_slots * sizeof(struct hlist_nulls_head);
1422        hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1423                                        get_order(sz));
1424        if (!hash) {
1425                printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
1426                hash = vzalloc(sz);
1427        }
1428
1429        if (hash && nulls)
1430                for (i = 0; i < nr_slots; i++)
1431                        INIT_HLIST_NULLS_HEAD(&hash[i], i);
1432
1433        return hash;
1434}
1435EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1436
1437int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1438{
1439        int i, bucket, rc;
1440        unsigned int hashsize, old_size;
1441        struct hlist_nulls_head *hash, *old_hash;
1442        struct nf_conntrack_tuple_hash *h;
1443        struct nf_conn *ct;
1444
1445        if (current->nsproxy->net_ns != &init_net)
1446                return -EOPNOTSUPP;
1447
1448        /* On boot, we can set this without any fancy locking. */
1449        if (!nf_conntrack_htable_size)
1450                return param_set_uint(val, kp);
1451
1452        rc = kstrtouint(val, 0, &hashsize);
1453        if (rc)
1454                return rc;
1455        if (!hashsize)
1456                return -EINVAL;
1457
1458        hash = nf_ct_alloc_hashtable(&hashsize, 1);
1459        if (!hash)
1460                return -ENOMEM;
1461
1462        /* Lookups in the old hash might happen in parallel, which means we
1463         * might get false negatives during connection lookup. New connections
1464         * created because of a false negative won't make it into the hash
1465         * though since that required taking the lock.
1466         */
1467        spin_lock_bh(&nf_conntrack_lock);
1468        for (i = 0; i < init_net.ct.htable_size; i++) {
1469                while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
1470                        h = hlist_nulls_entry(init_net.ct.hash[i].first,
1471                                        struct nf_conntrack_tuple_hash, hnnode);
1472                        ct = nf_ct_tuplehash_to_ctrack(h);
1473                        hlist_nulls_del_rcu(&h->hnnode);
1474                        bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
1475                                                  hashsize);
1476                        hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1477                }
1478        }
1479        old_size = init_net.ct.htable_size;
1480        old_hash = init_net.ct.hash;
1481
1482        init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
1483        init_net.ct.hash = hash;
1484        spin_unlock_bh(&nf_conntrack_lock);
1485
1486        nf_ct_free_hashtable(old_hash, old_size);
1487        return 0;
1488}
1489EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1490
1491module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1492                  &nf_conntrack_htable_size, 0600);
1493
1494void nf_ct_untracked_status_or(unsigned long bits)
1495{
1496        int cpu;
1497
1498        for_each_possible_cpu(cpu)
1499                per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1500}
1501EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1502
1503int nf_conntrack_init_start(void)
1504{
1505        int max_factor = 8;
1506        int ret, cpu;
1507
1508        /* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
1509         * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
1510        if (!nf_conntrack_htable_size) {
1511                nf_conntrack_htable_size
1512                        = (((totalram_pages << PAGE_SHIFT) / 16384)
1513                           / sizeof(struct hlist_head));
1514                if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1515                        nf_conntrack_htable_size = 16384;
1516                if (nf_conntrack_htable_size < 32)
1517                        nf_conntrack_htable_size = 32;
1518
1519                /* Use a max. factor of four by default to get the same max as
1520                 * with the old struct list_heads. When a table size is given
1521                 * we use the old value of 8 to avoid reducing the max.
1522                 * entries. */
1523                max_factor = 4;
1524        }
1525        nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1526
1527        printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1528               NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1529               nf_conntrack_max);
1530
1531        ret = nf_conntrack_expect_init();
1532        if (ret < 0)
1533                goto err_expect;
1534
1535        ret = nf_conntrack_acct_init();
1536        if (ret < 0)
1537                goto err_acct;
1538
1539        ret = nf_conntrack_tstamp_init();
1540        if (ret < 0)
1541                goto err_tstamp;
1542
1543        ret = nf_conntrack_ecache_init();
1544        if (ret < 0)
1545                goto err_ecache;
1546
1547        ret = nf_conntrack_timeout_init();
1548        if (ret < 0)
1549                goto err_timeout;
1550
1551        ret = nf_conntrack_helper_init();
1552        if (ret < 0)
1553                goto err_helper;
1554
1555        ret = nf_conntrack_labels_init();
1556        if (ret < 0)
1557                goto err_labels;
1558
1559#ifdef CONFIG_NF_CONNTRACK_ZONES
1560        ret = nf_ct_extend_register(&nf_ct_zone_extend);
1561        if (ret < 0)
1562                goto err_extend;
1563#endif
1564        ret = nf_conntrack_proto_init();
1565        if (ret < 0)
1566                goto err_proto;
1567
1568        /* Set up fake conntrack: to never be deleted, not in any hashes */
1569        for_each_possible_cpu(cpu) {
1570                struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1571                write_pnet(&ct->ct_net, &init_net);
1572                atomic_set(&ct->ct_general.use, 1);
1573        }
1574        /*  - and look it like as a confirmed connection */
1575        nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
1576        return 0;
1577
1578err_proto:
1579#ifdef CONFIG_NF_CONNTRACK_ZONES
1580        nf_ct_extend_unregister(&nf_ct_zone_extend);
1581err_extend:
1582#endif
1583        nf_conntrack_labels_fini();
1584err_labels:
1585        nf_conntrack_helper_fini();
1586err_helper:
1587        nf_conntrack_timeout_fini();
1588err_timeout:
1589        nf_conntrack_ecache_fini();
1590err_ecache:
1591        nf_conntrack_tstamp_fini();
1592err_tstamp:
1593        nf_conntrack_acct_fini();
1594err_acct:
1595        nf_conntrack_expect_fini();
1596err_expect:
1597        return ret;
1598}
1599
1600void nf_conntrack_init_end(void)
1601{
1602        /* For use by REJECT target */
1603        RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1604        RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1605
1606        /* Howto get NAT offsets */
1607        RCU_INIT_POINTER(nf_ct_nat_offset, NULL);
1608}
1609
1610/*
1611 * We need to use special "null" values, not used in hash table
1612 */
1613#define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
1614#define DYING_NULLS_VAL         ((1<<30)+1)
1615#define TEMPLATE_NULLS_VAL      ((1<<30)+2)
1616
1617int nf_conntrack_init_net(struct net *net)
1618{
1619        int ret;
1620
1621        atomic_set(&net->ct.count, 0);
1622        INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
1623        INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
1624        INIT_HLIST_NULLS_HEAD(&net->ct.tmpl, TEMPLATE_NULLS_VAL);
1625        net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1626        if (!net->ct.stat) {
1627                ret = -ENOMEM;
1628                goto err_stat;
1629        }
1630
1631        net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
1632        if (!net->ct.slabname) {
1633                ret = -ENOMEM;
1634                goto err_slabname;
1635        }
1636
1637        net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
1638                                                        sizeof(struct nf_conn), 0,
1639                                                        SLAB_DESTROY_BY_RCU, NULL);
1640        if (!net->ct.nf_conntrack_cachep) {
1641                printk(KERN_ERR "Unable to create nf_conn slab cache\n");
1642                ret = -ENOMEM;
1643                goto err_cache;
1644        }
1645
1646        net->ct.htable_size = nf_conntrack_htable_size;
1647        net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
1648        if (!net->ct.hash) {
1649                ret = -ENOMEM;
1650                printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
1651                goto err_hash;
1652        }
1653        ret = nf_conntrack_expect_pernet_init(net);
1654        if (ret < 0)
1655                goto err_expect;
1656        ret = nf_conntrack_acct_pernet_init(net);
1657        if (ret < 0)
1658                goto err_acct;
1659        ret = nf_conntrack_tstamp_pernet_init(net);
1660        if (ret < 0)
1661                goto err_tstamp;
1662        ret = nf_conntrack_ecache_pernet_init(net);
1663        if (ret < 0)
1664                goto err_ecache;
1665        ret = nf_conntrack_helper_pernet_init(net);
1666        if (ret < 0)
1667                goto err_helper;
1668        ret = nf_conntrack_proto_pernet_init(net);
1669        if (ret < 0)
1670                goto err_proto;
1671        return 0;
1672
1673err_proto:
1674        nf_conntrack_helper_pernet_fini(net);
1675err_helper:
1676        nf_conntrack_ecache_pernet_fini(net);
1677err_ecache:
1678        nf_conntrack_tstamp_pernet_fini(net);
1679err_tstamp:
1680        nf_conntrack_acct_pernet_fini(net);
1681err_acct:
1682        nf_conntrack_expect_pernet_fini(net);
1683err_expect:
1684        nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1685err_hash:
1686        kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1687err_cache:
1688        kfree(net->ct.slabname);
1689err_slabname:
1690        free_percpu(net->ct.stat);
1691err_stat:
1692        return ret;
1693}
1694
1695s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
1696                        enum ip_conntrack_dir dir,
1697                        u32 seq);
1698EXPORT_SYMBOL_GPL(nf_ct_nat_offset);
1699
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.