linux/net/netfilter/nf_conntrack_core.c
<<
>>
Prefs
   1/* Connection state tracking for netfilter.  This is separated from,
   2   but required by, the NAT layer; it can also be used by an iptables
   3   extension. */
   4
   5/* (C) 1999-2001 Paul `Rusty' Russell
   6 * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   7 * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as
  11 * published by the Free Software Foundation.
  12 */
  13
  14#include <linux/types.h>
  15#include <linux/netfilter.h>
  16#include <linux/module.h>
  17#include <linux/sched.h>
  18#include <linux/skbuff.h>
  19#include <linux/proc_fs.h>
  20#include <linux/vmalloc.h>
  21#include <linux/stddef.h>
  22#include <linux/slab.h>
  23#include <linux/random.h>
  24#include <linux/jhash.h>
  25#include <linux/err.h>
  26#include <linux/percpu.h>
  27#include <linux/moduleparam.h>
  28#include <linux/notifier.h>
  29#include <linux/kernel.h>
  30#include <linux/netdevice.h>
  31#include <linux/socket.h>
  32#include <linux/mm.h>
  33#include <linux/nsproxy.h>
  34#include <linux/rculist_nulls.h>
  35
  36#include <net/netfilter/nf_conntrack.h>
  37#include <net/netfilter/nf_conntrack_l3proto.h>
  38#include <net/netfilter/nf_conntrack_l4proto.h>
  39#include <net/netfilter/nf_conntrack_expect.h>
  40#include <net/netfilter/nf_conntrack_helper.h>
  41#include <net/netfilter/nf_conntrack_core.h>
  42#include <net/netfilter/nf_conntrack_extend.h>
  43#include <net/netfilter/nf_conntrack_acct.h>
  44#include <net/netfilter/nf_conntrack_ecache.h>
  45#include <net/netfilter/nf_conntrack_zones.h>
  46#include <net/netfilter/nf_conntrack_timestamp.h>
  47#include <net/netfilter/nf_conntrack_timeout.h>
  48#include <net/netfilter/nf_nat.h>
  49#include <net/netfilter/nf_nat_core.h>
  50
  51#define NF_CONNTRACK_VERSION    "0.5.0"
  52
  53int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
  54                                      enum nf_nat_manip_type manip,
  55                                      const struct nlattr *attr) __read_mostly;
  56EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
  57
  58int (*nf_nat_seq_adjust_hook)(struct sk_buff *skb,
  59                              struct nf_conn *ct,
  60                              enum ip_conntrack_info ctinfo,
  61                              unsigned int protoff);
  62EXPORT_SYMBOL_GPL(nf_nat_seq_adjust_hook);
  63
  64DEFINE_SPINLOCK(nf_conntrack_lock);
  65EXPORT_SYMBOL_GPL(nf_conntrack_lock);
  66
  67unsigned int nf_conntrack_htable_size __read_mostly;
  68EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
  69
  70unsigned int nf_conntrack_max __read_mostly;
  71EXPORT_SYMBOL_GPL(nf_conntrack_max);
  72
  73DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
  74EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
  75
  76unsigned int nf_conntrack_hash_rnd __read_mostly;
  77EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
  78
  79static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
  80{
  81        unsigned int n;
  82
  83        /* The direction must be ignored, so we hash everything up to the
  84         * destination ports (which is a multiple of 4) and treat the last
  85         * three bytes manually.
  86         */
  87        n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
  88        return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
  89                      (((__force __u16)tuple->dst.u.all << 16) |
  90                      tuple->dst.protonum));
  91}
  92
  93static u32 __hash_bucket(u32 hash, unsigned int size)
  94{
  95        return ((u64)hash * size) >> 32;
  96}
  97
  98static u32 hash_bucket(u32 hash, const struct net *net)
  99{
 100        return __hash_bucket(hash, net->ct.htable_size);
 101}
 102
 103static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
 104                                  u16 zone, unsigned int size)
 105{
 106        return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
 107}
 108
 109static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
 110                                       const struct nf_conntrack_tuple *tuple)
 111{
 112        return __hash_conntrack(tuple, zone, net->ct.htable_size);
 113}
 114
 115bool
 116nf_ct_get_tuple(const struct sk_buff *skb,
 117                unsigned int nhoff,
 118                unsigned int dataoff,
 119                u_int16_t l3num,
 120                u_int8_t protonum,
 121                struct nf_conntrack_tuple *tuple,
 122                const struct nf_conntrack_l3proto *l3proto,
 123                const struct nf_conntrack_l4proto *l4proto)
 124{
 125        memset(tuple, 0, sizeof(*tuple));
 126
 127        tuple->src.l3num = l3num;
 128        if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
 129                return false;
 130
 131        tuple->dst.protonum = protonum;
 132        tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 133
 134        return l4proto->pkt_to_tuple(skb, dataoff, tuple);
 135}
 136EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
 137
 138bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 139                       u_int16_t l3num, struct nf_conntrack_tuple *tuple)
 140{
 141        struct nf_conntrack_l3proto *l3proto;
 142        struct nf_conntrack_l4proto *l4proto;
 143        unsigned int protoff;
 144        u_int8_t protonum;
 145        int ret;
 146
 147        rcu_read_lock();
 148
 149        l3proto = __nf_ct_l3proto_find(l3num);
 150        ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
 151        if (ret != NF_ACCEPT) {
 152                rcu_read_unlock();
 153                return false;
 154        }
 155
 156        l4proto = __nf_ct_l4proto_find(l3num, protonum);
 157
 158        ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
 159                              l3proto, l4proto);
 160
 161        rcu_read_unlock();
 162        return ret;
 163}
 164EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 165
 166bool
 167nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 168                   const struct nf_conntrack_tuple *orig,
 169                   const struct nf_conntrack_l3proto *l3proto,
 170                   const struct nf_conntrack_l4proto *l4proto)
 171{
 172        memset(inverse, 0, sizeof(*inverse));
 173
 174        inverse->src.l3num = orig->src.l3num;
 175        if (l3proto->invert_tuple(inverse, orig) == 0)
 176                return false;
 177
 178        inverse->dst.dir = !orig->dst.dir;
 179
 180        inverse->dst.protonum = orig->dst.protonum;
 181        return l4proto->invert_tuple(inverse, orig);
 182}
 183EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 184
 185static void
 186clean_from_lists(struct nf_conn *ct)
 187{
 188        pr_debug("clean_from_lists(%p)\n", ct);
 189        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 190        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 191
 192        /* Destroy all pending expectations */
 193        nf_ct_remove_expectations(ct);
 194}
 195
 196static void
 197destroy_conntrack(struct nf_conntrack *nfct)
 198{
 199        struct nf_conn *ct = (struct nf_conn *)nfct;
 200        struct net *net = nf_ct_net(ct);
 201        struct nf_conntrack_l4proto *l4proto;
 202
 203        pr_debug("destroy_conntrack(%p)\n", ct);
 204        NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
 205        NF_CT_ASSERT(!timer_pending(&ct->timeout));
 206
 207        /* To make sure we don't get any weird locking issues here:
 208         * destroy_conntrack() MUST NOT be called with a write lock
 209         * to nf_conntrack_lock!!! -HW */
 210        rcu_read_lock();
 211        l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 212        if (l4proto && l4proto->destroy)
 213                l4proto->destroy(ct);
 214
 215        rcu_read_unlock();
 216
 217        spin_lock_bh(&nf_conntrack_lock);
 218        /* Expectations will have been removed in clean_from_lists,
 219         * except TFTP can create an expectation on the first packet,
 220         * before connection is in the list, so we need to clean here,
 221         * too. */
 222        nf_ct_remove_expectations(ct);
 223
 224        /* We overload first tuple to link into unconfirmed list. */
 225        if (!nf_ct_is_confirmed(ct)) {
 226                BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 227                hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 228        }
 229
 230        NF_CT_STAT_INC(net, delete);
 231        spin_unlock_bh(&nf_conntrack_lock);
 232
 233        if (ct->master)
 234                nf_ct_put(ct->master);
 235
 236        pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 237        nf_conntrack_free(ct);
 238}
 239
 240void nf_ct_delete_from_lists(struct nf_conn *ct)
 241{
 242        struct net *net = nf_ct_net(ct);
 243
 244        nf_ct_helper_destroy(ct);
 245        spin_lock_bh(&nf_conntrack_lock);
 246        /* Inside lock so preempt is disabled on module removal path.
 247         * Otherwise we can get spurious warnings. */
 248        NF_CT_STAT_INC(net, delete_list);
 249        clean_from_lists(ct);
 250        spin_unlock_bh(&nf_conntrack_lock);
 251}
 252EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);
 253
 254static void death_by_event(unsigned long ul_conntrack)
 255{
 256        struct nf_conn *ct = (void *)ul_conntrack;
 257        struct net *net = nf_ct_net(ct);
 258        struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
 259
 260        BUG_ON(ecache == NULL);
 261
 262        if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
 263                /* bad luck, let's retry again */
 264                ecache->timeout.expires = jiffies +
 265                        (random32() % net->ct.sysctl_events_retry_timeout);
 266                add_timer(&ecache->timeout);
 267                return;
 268        }
 269        /* we've got the event delivered, now it's dying */
 270        set_bit(IPS_DYING_BIT, &ct->status);
 271        spin_lock(&nf_conntrack_lock);
 272        hlist_nulls_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 273        spin_unlock(&nf_conntrack_lock);
 274        nf_ct_put(ct);
 275}
 276
 277void nf_ct_insert_dying_list(struct nf_conn *ct)
 278{
 279        struct net *net = nf_ct_net(ct);
 280        struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
 281
 282        BUG_ON(ecache == NULL);
 283
 284        /* add this conntrack to the dying list */
 285        spin_lock_bh(&nf_conntrack_lock);
 286        hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 287                             &net->ct.dying);
 288        spin_unlock_bh(&nf_conntrack_lock);
 289        /* set a new timer to retry event delivery */
 290        setup_timer(&ecache->timeout, death_by_event, (unsigned long)ct);
 291        ecache->timeout.expires = jiffies +
 292                (random32() % net->ct.sysctl_events_retry_timeout);
 293        add_timer(&ecache->timeout);
 294}
 295EXPORT_SYMBOL_GPL(nf_ct_insert_dying_list);
 296
 297static void death_by_timeout(unsigned long ul_conntrack)
 298{
 299        struct nf_conn *ct = (void *)ul_conntrack;
 300        struct nf_conn_tstamp *tstamp;
 301
 302        tstamp = nf_conn_tstamp_find(ct);
 303        if (tstamp && tstamp->stop == 0)
 304                tstamp->stop = ktime_to_ns(ktime_get_real());
 305
 306        if (!test_bit(IPS_DYING_BIT, &ct->status) &&
 307            unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
 308                /* destroy event was not delivered */
 309                nf_ct_delete_from_lists(ct);
 310                nf_ct_insert_dying_list(ct);
 311                return;
 312        }
 313        set_bit(IPS_DYING_BIT, &ct->status);
 314        nf_ct_delete_from_lists(ct);
 315        nf_ct_put(ct);
 316}
 317
 318/*
 319 * Warning :
 320 * - Caller must take a reference on returned object
 321 *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 322 * OR
 323 * - Caller must lock nf_conntrack_lock before calling this function
 324 */
 325static struct nf_conntrack_tuple_hash *
 326____nf_conntrack_find(struct net *net, u16 zone,
 327                      const struct nf_conntrack_tuple *tuple, u32 hash)
 328{
 329        struct nf_conntrack_tuple_hash *h;
 330        struct hlist_nulls_node *n;
 331        unsigned int bucket = hash_bucket(hash, net);
 332
 333        /* Disable BHs the entire time since we normally need to disable them
 334         * at least once for the stats anyway.
 335         */
 336        local_bh_disable();
 337begin:
 338        hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
 339                if (nf_ct_tuple_equal(tuple, &h->tuple) &&
 340                    nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
 341                        NF_CT_STAT_INC(net, found);
 342                        local_bh_enable();
 343                        return h;
 344                }
 345                NF_CT_STAT_INC(net, searched);
 346        }
 347        /*
 348         * if the nulls value we got at the end of this lookup is
 349         * not the expected one, we must restart lookup.
 350         * We probably met an item that was moved to another chain.
 351         */
 352        if (get_nulls_value(n) != bucket) {
 353                NF_CT_STAT_INC(net, search_restart);
 354                goto begin;
 355        }
 356        local_bh_enable();
 357
 358        return NULL;
 359}
 360
 361struct nf_conntrack_tuple_hash *
 362__nf_conntrack_find(struct net *net, u16 zone,
 363                    const struct nf_conntrack_tuple *tuple)
 364{
 365        return ____nf_conntrack_find(net, zone, tuple,
 366                                     hash_conntrack_raw(tuple, zone));
 367}
 368EXPORT_SYMBOL_GPL(__nf_conntrack_find);
 369
 370/* Find a connection corresponding to a tuple. */
 371static struct nf_conntrack_tuple_hash *
 372__nf_conntrack_find_get(struct net *net, u16 zone,
 373                        const struct nf_conntrack_tuple *tuple, u32 hash)
 374{
 375        struct nf_conntrack_tuple_hash *h;
 376        struct nf_conn *ct;
 377
 378        rcu_read_lock();
 379begin:
 380        h = ____nf_conntrack_find(net, zone, tuple, hash);
 381        if (h) {
 382                ct = nf_ct_tuplehash_to_ctrack(h);
 383                if (unlikely(nf_ct_is_dying(ct) ||
 384                             !atomic_inc_not_zero(&ct->ct_general.use)))
 385                        h = NULL;
 386                else {
 387                        if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
 388                                     nf_ct_zone(ct) != zone)) {
 389                                nf_ct_put(ct);
 390                                goto begin;
 391                        }
 392                }
 393        }
 394        rcu_read_unlock();
 395
 396        return h;
 397}
 398
 399struct nf_conntrack_tuple_hash *
 400nf_conntrack_find_get(struct net *net, u16 zone,
 401                      const struct nf_conntrack_tuple *tuple)
 402{
 403        return __nf_conntrack_find_get(net, zone, tuple,
 404                                       hash_conntrack_raw(tuple, zone));
 405}
 406EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 407
 408static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 409                                       unsigned int hash,
 410                                       unsigned int repl_hash)
 411{
 412        struct net *net = nf_ct_net(ct);
 413
 414        hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 415                           &net->ct.hash[hash]);
 416        hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 417                           &net->ct.hash[repl_hash]);
 418}
 419
 420int
 421nf_conntrack_hash_check_insert(struct nf_conn *ct)
 422{
 423        struct net *net = nf_ct_net(ct);
 424        unsigned int hash, repl_hash;
 425        struct nf_conntrack_tuple_hash *h;
 426        struct hlist_nulls_node *n;
 427        u16 zone;
 428
 429        zone = nf_ct_zone(ct);
 430        hash = hash_conntrack(net, zone,
 431                              &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 432        repl_hash = hash_conntrack(net, zone,
 433                                   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 434
 435        spin_lock_bh(&nf_conntrack_lock);
 436
 437        /* See if there's one in the list already, including reverse */
 438        hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
 439                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 440                                      &h->tuple) &&
 441                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 442                        goto out;
 443        hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
 444                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 445                                      &h->tuple) &&
 446                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 447                        goto out;
 448
 449        add_timer(&ct->timeout);
 450        nf_conntrack_get(&ct->ct_general);
 451        __nf_conntrack_hash_insert(ct, hash, repl_hash);
 452        NF_CT_STAT_INC(net, insert);
 453        spin_unlock_bh(&nf_conntrack_lock);
 454
 455        return 0;
 456
 457out:
 458        NF_CT_STAT_INC(net, insert_failed);
 459        spin_unlock_bh(&nf_conntrack_lock);
 460        return -EEXIST;
 461}
 462EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 463
 464/* Confirm a connection given skb; places it in hash table */
 465int
 466__nf_conntrack_confirm(struct sk_buff *skb)
 467{
 468        unsigned int hash, repl_hash;
 469        struct nf_conntrack_tuple_hash *h;
 470        struct nf_conn *ct;
 471        struct nf_conn_help *help;
 472        struct nf_conn_tstamp *tstamp;
 473        struct hlist_nulls_node *n;
 474        enum ip_conntrack_info ctinfo;
 475        struct net *net;
 476        u16 zone;
 477
 478        ct = nf_ct_get(skb, &ctinfo);
 479        net = nf_ct_net(ct);
 480
 481        /* ipt_REJECT uses nf_conntrack_attach to attach related
 482           ICMP/TCP RST packets in other direction.  Actual packet
 483           which created connection will be IP_CT_NEW or for an
 484           expected connection, IP_CT_RELATED. */
 485        if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
 486                return NF_ACCEPT;
 487
 488        zone = nf_ct_zone(ct);
 489        /* reuse the hash saved before */
 490        hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
 491        hash = hash_bucket(hash, net);
 492        repl_hash = hash_conntrack(net, zone,
 493                                   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 494
 495        /* We're not in hash table, and we refuse to set up related
 496           connections for unconfirmed conns.  But packet copies and
 497           REJECT will give spurious warnings here. */
 498        /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
 499
 500        /* No external references means no one else could have
 501           confirmed us. */
 502        NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
 503        pr_debug("Confirming conntrack %p\n", ct);
 504
 505        spin_lock_bh(&nf_conntrack_lock);
 506
 507        /* We have to check the DYING flag inside the lock to prevent
 508           a race against nf_ct_get_next_corpse() possibly called from
 509           user context, else we insert an already 'dead' hash, blocking
 510           further use of that particular connection -JM */
 511
 512        if (unlikely(nf_ct_is_dying(ct))) {
 513                spin_unlock_bh(&nf_conntrack_lock);
 514                return NF_ACCEPT;
 515        }
 516
 517        /* See if there's one in the list already, including reverse:
 518           NAT could have grabbed it without realizing, since we're
 519           not in the hash.  If there is, we lost race. */
 520        hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
 521                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 522                                      &h->tuple) &&
 523                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 524                        goto out;
 525        hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
 526                if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 527                                      &h->tuple) &&
 528                    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 529                        goto out;
 530
 531        /* Remove from unconfirmed list */
 532        hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 533
 534        /* Timer relative to confirmation time, not original
 535           setting time, otherwise we'd get timer wrap in
 536           weird delay cases. */
 537        ct->timeout.expires += jiffies;
 538        add_timer(&ct->timeout);
 539        atomic_inc(&ct->ct_general.use);
 540        ct->status |= IPS_CONFIRMED;
 541
 542        /* set conntrack timestamp, if enabled. */
 543        tstamp = nf_conn_tstamp_find(ct);
 544        if (tstamp) {
 545                if (skb->tstamp.tv64 == 0)
 546                        __net_timestamp(skb);
 547
 548                tstamp->start = ktime_to_ns(skb->tstamp);
 549        }
 550        /* Since the lookup is lockless, hash insertion must be done after
 551         * starting the timer and setting the CONFIRMED bit. The RCU barriers
 552         * guarantee that no other CPU can find the conntrack before the above
 553         * stores are visible.
 554         */
 555        __nf_conntrack_hash_insert(ct, hash, repl_hash);
 556        NF_CT_STAT_INC(net, insert);
 557        spin_unlock_bh(&nf_conntrack_lock);
 558
 559        help = nfct_help(ct);
 560        if (help && help->helper)
 561                nf_conntrack_event_cache(IPCT_HELPER, ct);
 562
 563        nf_conntrack_event_cache(master_ct(ct) ?
 564                                 IPCT_RELATED : IPCT_NEW, ct);
 565        return NF_ACCEPT;
 566
 567out:
 568        NF_CT_STAT_INC(net, insert_failed);
 569        spin_unlock_bh(&nf_conntrack_lock);
 570        return NF_DROP;
 571}
 572EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
 573
 574/* Returns true if a connection correspondings to the tuple (required
 575   for NAT). */
 576int
 577nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
 578                         const struct nf_conn *ignored_conntrack)
 579{
 580        struct net *net = nf_ct_net(ignored_conntrack);
 581        struct nf_conntrack_tuple_hash *h;
 582        struct hlist_nulls_node *n;
 583        struct nf_conn *ct;
 584        u16 zone = nf_ct_zone(ignored_conntrack);
 585        unsigned int hash = hash_conntrack(net, zone, tuple);
 586
 587        /* Disable BHs the entire time since we need to disable them at
 588         * least once for the stats anyway.
 589         */
 590        rcu_read_lock_bh();
 591        hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
 592                ct = nf_ct_tuplehash_to_ctrack(h);
 593                if (ct != ignored_conntrack &&
 594                    nf_ct_tuple_equal(tuple, &h->tuple) &&
 595                    nf_ct_zone(ct) == zone) {
 596                        NF_CT_STAT_INC(net, found);
 597                        rcu_read_unlock_bh();
 598                        return 1;
 599                }
 600                NF_CT_STAT_INC(net, searched);
 601        }
 602        rcu_read_unlock_bh();
 603
 604        return 0;
 605}
 606EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
 607
 608#define NF_CT_EVICTION_RANGE    8
 609
 610/* There's a small race here where we may free a just-assured
 611   connection.  Too bad: we're in trouble anyway. */
 612static noinline int early_drop(struct net *net, unsigned int hash)
 613{
 614        /* Use oldest entry, which is roughly LRU */
 615        struct nf_conntrack_tuple_hash *h;
 616        struct nf_conn *ct = NULL, *tmp;
 617        struct hlist_nulls_node *n;
 618        unsigned int i, cnt = 0;
 619        int dropped = 0;
 620
 621        rcu_read_lock();
 622        for (i = 0; i < net->ct.htable_size; i++) {
 623                hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
 624                                         hnnode) {
 625                        tmp = nf_ct_tuplehash_to_ctrack(h);
 626                        if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
 627                                ct = tmp;
 628                        cnt++;
 629                }
 630
 631                if (ct != NULL) {
 632                        if (likely(!nf_ct_is_dying(ct) &&
 633                                   atomic_inc_not_zero(&ct->ct_general.use)))
 634                                break;
 635                        else
 636                                ct = NULL;
 637                }
 638
 639                if (cnt >= NF_CT_EVICTION_RANGE)
 640                        break;
 641
 642                hash = (hash + 1) % net->ct.htable_size;
 643        }
 644        rcu_read_unlock();
 645
 646        if (!ct)
 647                return dropped;
 648
 649        if (del_timer(&ct->timeout)) {
 650                death_by_timeout((unsigned long)ct);
 651                /* Check if we indeed killed this entry. Reliable event
 652                   delivery may have inserted it into the dying list. */
 653                if (test_bit(IPS_DYING_BIT, &ct->status)) {
 654                        dropped = 1;
 655                        NF_CT_STAT_INC_ATOMIC(net, early_drop);
 656                }
 657        }
 658        nf_ct_put(ct);
 659        return dropped;
 660}
 661
 662void init_nf_conntrack_hash_rnd(void)
 663{
 664        unsigned int rand;
 665
 666        /*
 667         * Why not initialize nf_conntrack_rnd in a "init()" function ?
 668         * Because there isn't enough entropy when system initializing,
 669         * and we initialize it as late as possible.
 670         */
 671        do {
 672                get_random_bytes(&rand, sizeof(rand));
 673        } while (!rand);
 674        cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
 675}
 676
 677static struct nf_conn *
 678__nf_conntrack_alloc(struct net *net, u16 zone,
 679                     const struct nf_conntrack_tuple *orig,
 680                     const struct nf_conntrack_tuple *repl,
 681                     gfp_t gfp, u32 hash)
 682{
 683        struct nf_conn *ct;
 684
 685        if (unlikely(!nf_conntrack_hash_rnd)) {
 686                init_nf_conntrack_hash_rnd();
 687                /* recompute the hash as nf_conntrack_hash_rnd is initialized */
 688                hash = hash_conntrack_raw(orig, zone);
 689        }
 690
 691        /* We don't want any race condition at early drop stage */
 692        atomic_inc(&net->ct.count);
 693
 694        if (nf_conntrack_max &&
 695            unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
 696                if (!early_drop(net, hash_bucket(hash, net))) {
 697                        atomic_dec(&net->ct.count);
 698                        net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
 699                        return ERR_PTR(-ENOMEM);
 700                }
 701        }
 702
 703        /*
 704         * Do not use kmem_cache_zalloc(), as this cache uses
 705         * SLAB_DESTROY_BY_RCU.
 706         */
 707        ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
 708        if (ct == NULL) {
 709                atomic_dec(&net->ct.count);
 710                return ERR_PTR(-ENOMEM);
 711        }
 712        /*
 713         * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
 714         * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
 715         */
 716        memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
 717               offsetof(struct nf_conn, proto) -
 718               offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));
 719        spin_lock_init(&ct->lock);
 720        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
 721        ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
 722        ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
 723        /* save hash for reusing when confirming */
 724        *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
 725        /* Don't set timer yet: wait for confirmation */
 726        setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
 727        write_pnet(&ct->ct_net, net);
 728#ifdef CONFIG_NF_CONNTRACK_ZONES
 729        if (zone) {
 730                struct nf_conntrack_zone *nf_ct_zone;
 731
 732                nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
 733                if (!nf_ct_zone)
 734                        goto out_free;
 735                nf_ct_zone->id = zone;
 736        }
 737#endif
 738        /*
 739         * changes to lookup keys must be done before setting refcnt to 1
 740         */
 741        smp_wmb();
 742        atomic_set(&ct->ct_general.use, 1);
 743        return ct;
 744
 745#ifdef CONFIG_NF_CONNTRACK_ZONES
 746out_free:
 747        atomic_dec(&net->ct.count);
 748        kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
 749        return ERR_PTR(-ENOMEM);
 750#endif
 751}
 752
 753struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
 754                                   const struct nf_conntrack_tuple *orig,
 755                                   const struct nf_conntrack_tuple *repl,
 756                                   gfp_t gfp)
 757{
 758        return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
 759}
 760EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
 761
 762void nf_conntrack_free(struct nf_conn *ct)
 763{
 764        struct net *net = nf_ct_net(ct);
 765
 766        nf_ct_ext_destroy(ct);
 767        atomic_dec(&net->ct.count);
 768        nf_ct_ext_free(ct);
 769        kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
 770}
 771EXPORT_SYMBOL_GPL(nf_conntrack_free);
 772
 773/* Allocate a new conntrack: we return -ENOMEM if classification
 774   failed due to stress.  Otherwise it really is unclassifiable. */
 775static struct nf_conntrack_tuple_hash *
 776init_conntrack(struct net *net, struct nf_conn *tmpl,
 777               const struct nf_conntrack_tuple *tuple,
 778               struct nf_conntrack_l3proto *l3proto,
 779               struct nf_conntrack_l4proto *l4proto,
 780               struct sk_buff *skb,
 781               unsigned int dataoff, u32 hash)
 782{
 783        struct nf_conn *ct;
 784        struct nf_conn_help *help;
 785        struct nf_conntrack_tuple repl_tuple;
 786        struct nf_conntrack_ecache *ecache;
 787        struct nf_conntrack_expect *exp;
 788        u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
 789        struct nf_conn_timeout *timeout_ext;
 790        unsigned int *timeouts;
 791
 792        if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
 793                pr_debug("Can't invert tuple.\n");
 794                return NULL;
 795        }
 796
 797        ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
 798                                  hash);
 799        if (IS_ERR(ct))
 800                return (struct nf_conntrack_tuple_hash *)ct;
 801
 802        timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
 803        if (timeout_ext)
 804                timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
 805        else
 806                timeouts = l4proto->get_timeouts(net);
 807
 808        if (!l4proto->new(ct, skb, dataoff, timeouts)) {
 809                nf_conntrack_free(ct);
 810                pr_debug("init conntrack: can't track with proto module\n");
 811                return NULL;
 812        }
 813
 814        if (timeout_ext)
 815                nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
 816
 817        nf_ct_acct_ext_add(ct, GFP_ATOMIC);
 818        nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
 819
 820        ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
 821        nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
 822                                 ecache ? ecache->expmask : 0,
 823                             GFP_ATOMIC);
 824
 825        spin_lock_bh(&nf_conntrack_lock);
 826        exp = nf_ct_find_expectation(net, zone, tuple);
 827        if (exp) {
 828                pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
 829                         ct, exp);
 830                /* Welcome, Mr. Bond.  We've been expecting you... */
 831                __set_bit(IPS_EXPECTED_BIT, &ct->status);
 832                ct->master = exp->master;
 833                if (exp->helper) {
 834                        help = nf_ct_helper_ext_add(ct, exp->helper,
 835                                                    GFP_ATOMIC);
 836                        if (help)
 837                                rcu_assign_pointer(help->helper, exp->helper);
 838                }
 839
 840#ifdef CONFIG_NF_CONNTRACK_MARK
 841                ct->mark = exp->master->mark;
 842#endif
 843#ifdef CONFIG_NF_CONNTRACK_SECMARK
 844                ct->secmark = exp->master->secmark;
 845#endif
 846                nf_conntrack_get(&ct->master->ct_general);
 847                NF_CT_STAT_INC(net, expect_new);
 848        } else {
 849                __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
 850                NF_CT_STAT_INC(net, new);
 851        }
 852
 853        /* Overload tuple linked list to put us in unconfirmed list. */
 854        hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 855                       &net->ct.unconfirmed);
 856
 857        spin_unlock_bh(&nf_conntrack_lock);
 858
 859        if (exp) {
 860                if (exp->expectfn)
 861                        exp->expectfn(ct, exp);
 862                nf_ct_expect_put(exp);
 863        }
 864
 865        return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
 866}
 867
 868/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
 869static inline struct nf_conn *
 870resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
 871                  struct sk_buff *skb,
 872                  unsigned int dataoff,
 873                  u_int16_t l3num,
 874                  u_int8_t protonum,
 875                  struct nf_conntrack_l3proto *l3proto,
 876                  struct nf_conntrack_l4proto *l4proto,
 877                  int *set_reply,
 878                  enum ip_conntrack_info *ctinfo)
 879{
 880        struct nf_conntrack_tuple tuple;
 881        struct nf_conntrack_tuple_hash *h;
 882        struct nf_conn *ct;
 883        u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
 884        u32 hash;
 885
 886        if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
 887                             dataoff, l3num, protonum, &tuple, l3proto,
 888                             l4proto)) {
 889                pr_debug("resolve_normal_ct: Can't get tuple\n");
 890                return NULL;
 891        }
 892
 893        /* look for tuple match */
 894        hash = hash_conntrack_raw(&tuple, zone);
 895        h = __nf_conntrack_find_get(net, zone, &tuple, hash);
 896        if (!h) {
 897                h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
 898                                   skb, dataoff, hash);
 899                if (!h)
 900                        return NULL;
 901                if (IS_ERR(h))
 902                        return (void *)h;
 903        }
 904        ct = nf_ct_tuplehash_to_ctrack(h);
 905
 906        /* It exists; we have (non-exclusive) reference. */
 907        if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
 908                *ctinfo = IP_CT_ESTABLISHED_REPLY;
 909                /* Please set reply bit if this packet OK */
 910                *set_reply = 1;
 911        } else {
 912                /* Once we've had two way comms, always ESTABLISHED. */
 913                if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
 914                        pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
 915                        *ctinfo = IP_CT_ESTABLISHED;
 916                } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
 917                        pr_debug("nf_conntrack_in: related packet for %p\n",
 918                                 ct);
 919                        *ctinfo = IP_CT_RELATED;
 920                } else {
 921                        pr_debug("nf_conntrack_in: new packet for %p\n", ct);
 922                        *ctinfo = IP_CT_NEW;
 923                }
 924                *set_reply = 0;
 925        }
 926        skb->nfct = &ct->ct_general;
 927        skb->nfctinfo = *ctinfo;
 928        return ct;
 929}
 930
 931unsigned int
 932nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
 933                struct sk_buff *skb)
 934{
 935        struct nf_conn *ct, *tmpl = NULL;
 936        enum ip_conntrack_info ctinfo;
 937        struct nf_conntrack_l3proto *l3proto;
 938        struct nf_conntrack_l4proto *l4proto;
 939        unsigned int *timeouts;
 940        unsigned int dataoff;
 941        u_int8_t protonum;
 942        int set_reply = 0;
 943        int ret;
 944
 945        if (skb->nfct) {
 946                /* Previously seen (loopback or untracked)?  Ignore. */
 947                tmpl = (struct nf_conn *)skb->nfct;
 948                if (!nf_ct_is_template(tmpl)) {
 949                        NF_CT_STAT_INC_ATOMIC(net, ignore);
 950                        return NF_ACCEPT;
 951                }
 952                skb->nfct = NULL;
 953        }
 954
 955        /* rcu_read_lock()ed by nf_hook_slow */
 956        l3proto = __nf_ct_l3proto_find(pf);
 957        ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
 958                                   &dataoff, &protonum);
 959        if (ret <= 0) {
 960                pr_debug("not prepared to track yet or error occurred\n");
 961                NF_CT_STAT_INC_ATOMIC(net, error);
 962                NF_CT_STAT_INC_ATOMIC(net, invalid);
 963                ret = -ret;
 964                goto out;
 965        }
 966
 967        l4proto = __nf_ct_l4proto_find(pf, protonum);
 968
 969        /* It may be an special packet, error, unclean...
 970         * inverse of the return code tells to the netfilter
 971         * core what to do with the packet. */
 972        if (l4proto->error != NULL) {
 973                ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
 974                                     pf, hooknum);
 975                if (ret <= 0) {
 976                        NF_CT_STAT_INC_ATOMIC(net, error);
 977                        NF_CT_STAT_INC_ATOMIC(net, invalid);
 978                        ret = -ret;
 979                        goto out;
 980                }
 981                /* ICMP[v6] protocol trackers may assign one conntrack. */
 982                if (skb->nfct)
 983                        goto out;
 984        }
 985
 986        ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
 987                               l3proto, l4proto, &set_reply, &ctinfo);
 988        if (!ct) {
 989                /* Not valid part of a connection */
 990                NF_CT_STAT_INC_ATOMIC(net, invalid);
 991                ret = NF_ACCEPT;
 992                goto out;
 993        }
 994
 995        if (IS_ERR(ct)) {
 996                /* Too stressed to deal. */
 997                NF_CT_STAT_INC_ATOMIC(net, drop);
 998                ret = NF_DROP;
 999                goto out;
1000        }
1001
1002        NF_CT_ASSERT(skb->nfct);
1003
1004        /* Decide what timeout policy we want to apply to this flow. */
1005        timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1006
1007        ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1008        if (ret <= 0) {
1009                /* Invalid: inverse of the return code tells
1010                 * the netfilter core what to do */
1011                pr_debug("nf_conntrack_in: Can't track with proto module\n");
1012                nf_conntrack_put(skb->nfct);
1013                skb->nfct = NULL;
1014                NF_CT_STAT_INC_ATOMIC(net, invalid);
1015                if (ret == -NF_DROP)
1016                        NF_CT_STAT_INC_ATOMIC(net, drop);
1017                ret = -ret;
1018                goto out;
1019        }
1020
1021        if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1022                nf_conntrack_event_cache(IPCT_REPLY, ct);
1023out:
1024        if (tmpl) {
1025                /* Special case: we have to repeat this hook, assign the
1026                 * template again to this packet. We assume that this packet
1027                 * has no conntrack assigned. This is used by nf_ct_tcp. */
1028                if (ret == NF_REPEAT)
1029                        skb->nfct = (struct nf_conntrack *)tmpl;
1030                else
1031                        nf_ct_put(tmpl);
1032        }
1033
1034        return ret;
1035}
1036EXPORT_SYMBOL_GPL(nf_conntrack_in);
1037
1038bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1039                          const struct nf_conntrack_tuple *orig)
1040{
1041        bool ret;
1042
1043        rcu_read_lock();
1044        ret = nf_ct_invert_tuple(inverse, orig,
1045                                 __nf_ct_l3proto_find(orig->src.l3num),
1046                                 __nf_ct_l4proto_find(orig->src.l3num,
1047                                                      orig->dst.protonum));
1048        rcu_read_unlock();
1049        return ret;
1050}
1051EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1052
1053/* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1054   implicitly racy: see __nf_conntrack_confirm */
1055void nf_conntrack_alter_reply(struct nf_conn *ct,
1056                              const struct nf_conntrack_tuple *newreply)
1057{
1058        struct nf_conn_help *help = nfct_help(ct);
1059
1060        /* Should be unconfirmed, so not in hash table yet */
1061        NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1062
1063        pr_debug("Altering reply tuple of %p to ", ct);
1064        nf_ct_dump_tuple(newreply);
1065
1066        ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1067        if (ct->master || (help && !hlist_empty(&help->expectations)))
1068                return;
1069
1070        rcu_read_lock();
1071        __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1072        rcu_read_unlock();
1073}
1074EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1075
1076/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1077void __nf_ct_refresh_acct(struct nf_conn *ct,
1078                          enum ip_conntrack_info ctinfo,
1079                          const struct sk_buff *skb,
1080                          unsigned long extra_jiffies,
1081                          int do_acct)
1082{
1083        NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1084        NF_CT_ASSERT(skb);
1085
1086        /* Only update if this is not a fixed timeout */
1087        if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1088                goto acct;
1089
1090        /* If not in hash table, timer will not be active yet */
1091        if (!nf_ct_is_confirmed(ct)) {
1092                ct->timeout.expires = extra_jiffies;
1093        } else {
1094                unsigned long newtime = jiffies + extra_jiffies;
1095
1096                /* Only update the timeout if the new timeout is at least
1097                   HZ jiffies from the old timeout. Need del_timer for race
1098                   avoidance (may already be dying). */
1099                if (newtime - ct->timeout.expires >= HZ)
1100                        mod_timer_pending(&ct->timeout, newtime);
1101        }
1102
1103acct:
1104        if (do_acct) {
1105                struct nf_conn_counter *acct;
1106
1107                acct = nf_conn_acct_find(ct);
1108                if (acct) {
1109                        atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1110                        atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);
1111                }
1112        }
1113}
1114EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1115
1116bool __nf_ct_kill_acct(struct nf_conn *ct,
1117                       enum ip_conntrack_info ctinfo,
1118                       const struct sk_buff *skb,
1119                       int do_acct)
1120{
1121        if (do_acct) {
1122                struct nf_conn_counter *acct;
1123
1124                acct = nf_conn_acct_find(ct);
1125                if (acct) {
1126                        atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1127                        atomic64_add(skb->len - skb_network_offset(skb),
1128                                     &acct[CTINFO2DIR(ctinfo)].bytes);
1129                }
1130        }
1131
1132        if (del_timer(&ct->timeout)) {
1133                ct->timeout.function((unsigned long)ct);
1134                return true;
1135        }
1136        return false;
1137}
1138EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
1139
1140#ifdef CONFIG_NF_CONNTRACK_ZONES
1141static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
1142        .len    = sizeof(struct nf_conntrack_zone),
1143        .align  = __alignof__(struct nf_conntrack_zone),
1144        .id     = NF_CT_EXT_ZONE,
1145};
1146#endif
1147
1148#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1149
1150#include <linux/netfilter/nfnetlink.h>
1151#include <linux/netfilter/nfnetlink_conntrack.h>
1152#include <linux/mutex.h>
1153
1154/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1155 * in ip_conntrack_core, since we don't want the protocols to autoload
1156 * or depend on ctnetlink */
1157int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1158                               const struct nf_conntrack_tuple *tuple)
1159{
1160        if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1161            nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1162                goto nla_put_failure;
1163        return 0;
1164
1165nla_put_failure:
1166        return -1;
1167}
1168EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1169
1170const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1171        [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1172        [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1173};
1174EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1175
1176int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1177                               struct nf_conntrack_tuple *t)
1178{
1179        if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1180                return -EINVAL;
1181
1182        t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1183        t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1184
1185        return 0;
1186}
1187EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1188
1189int nf_ct_port_nlattr_tuple_size(void)
1190{
1191        return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1192}
1193EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1194#endif
1195
1196/* Used by ipt_REJECT and ip6t_REJECT. */
1197static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
1198{
1199        struct nf_conn *ct;
1200        enum ip_conntrack_info ctinfo;
1201
1202        /* This ICMP is in reverse direction to the packet which caused it */
1203        ct = nf_ct_get(skb, &ctinfo);
1204        if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1205                ctinfo = IP_CT_RELATED_REPLY;
1206        else
1207                ctinfo = IP_CT_RELATED;
1208
1209        /* Attach to new skbuff, and increment count */
1210        nskb->nfct = &ct->ct_general;
1211        nskb->nfctinfo = ctinfo;
1212        nf_conntrack_get(nskb->nfct);
1213}
1214
1215/* Bring out ya dead! */
1216static struct nf_conn *
1217get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1218                void *data, unsigned int *bucket)
1219{
1220        struct nf_conntrack_tuple_hash *h;
1221        struct nf_conn *ct;
1222        struct hlist_nulls_node *n;
1223
1224        spin_lock_bh(&nf_conntrack_lock);
1225        for (; *bucket < net->ct.htable_size; (*bucket)++) {
1226                hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
1227                        if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1228                                continue;
1229                        ct = nf_ct_tuplehash_to_ctrack(h);
1230                        if (iter(ct, data))
1231                                goto found;
1232                }
1233        }
1234        hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
1235                ct = nf_ct_tuplehash_to_ctrack(h);
1236                if (iter(ct, data))
1237                        set_bit(IPS_DYING_BIT, &ct->status);
1238        }
1239        spin_unlock_bh(&nf_conntrack_lock);
1240        return NULL;
1241found:
1242        atomic_inc(&ct->ct_general.use);
1243        spin_unlock_bh(&nf_conntrack_lock);
1244        return ct;
1245}
1246
1247void nf_ct_iterate_cleanup(struct net *net,
1248                           int (*iter)(struct nf_conn *i, void *data),
1249                           void *data)
1250{
1251        struct nf_conn *ct;
1252        unsigned int bucket = 0;
1253
1254        while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1255                /* Time to push up daises... */
1256                if (del_timer(&ct->timeout))
1257                        death_by_timeout((unsigned long)ct);
1258                /* ... else the timer will get him soon. */
1259
1260                nf_ct_put(ct);
1261        }
1262}
1263EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1264
1265struct __nf_ct_flush_report {
1266        u32 pid;
1267        int report;
1268};
1269
1270static int kill_report(struct nf_conn *i, void *data)
1271{
1272        struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
1273        struct nf_conn_tstamp *tstamp;
1274
1275        tstamp = nf_conn_tstamp_find(i);
1276        if (tstamp && tstamp->stop == 0)
1277                tstamp->stop = ktime_to_ns(ktime_get_real());
1278
1279        /* If we fail to deliver the event, death_by_timeout() will retry */
1280        if (nf_conntrack_event_report(IPCT_DESTROY, i,
1281                                      fr->pid, fr->report) < 0)
1282                return 1;
1283
1284        /* Avoid the delivery of the destroy event in death_by_timeout(). */
1285        set_bit(IPS_DYING_BIT, &i->status);
1286        return 1;
1287}
1288
1289static int kill_all(struct nf_conn *i, void *data)
1290{
1291        return 1;
1292}
1293
1294void nf_ct_free_hashtable(void *hash, unsigned int size)
1295{
1296        if (is_vmalloc_addr(hash))
1297                vfree(hash);
1298        else
1299                free_pages((unsigned long)hash,
1300                           get_order(sizeof(struct hlist_head) * size));
1301}
1302EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1303
1304void nf_conntrack_flush_report(struct net *net, u32 pid, int report)
1305{
1306        struct __nf_ct_flush_report fr = {
1307                .pid    = pid,
1308                .report = report,
1309        };
1310        nf_ct_iterate_cleanup(net, kill_report, &fr);
1311}
1312EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);
1313
1314static void nf_ct_release_dying_list(struct net *net)
1315{
1316        struct nf_conntrack_tuple_hash *h;
1317        struct nf_conn *ct;
1318        struct hlist_nulls_node *n;
1319
1320        spin_lock_bh(&nf_conntrack_lock);
1321        hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
1322                ct = nf_ct_tuplehash_to_ctrack(h);
1323                /* never fails to remove them, no listeners at this point */
1324                nf_ct_kill(ct);
1325        }
1326        spin_unlock_bh(&nf_conntrack_lock);
1327}
1328
1329static int untrack_refs(void)
1330{
1331        int cnt = 0, cpu;
1332
1333        for_each_possible_cpu(cpu) {
1334                struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1335
1336                cnt += atomic_read(&ct->ct_general.use) - 1;
1337        }
1338        return cnt;
1339}
1340
1341static void nf_conntrack_cleanup_init_net(void)
1342{
1343        while (untrack_refs() > 0)
1344                schedule();
1345
1346#ifdef CONFIG_NF_CONNTRACK_ZONES
1347        nf_ct_extend_unregister(&nf_ct_zone_extend);
1348#endif
1349}
1350
1351static void nf_conntrack_cleanup_net(struct net *net)
1352{
1353 i_see_dead_people:
1354        nf_ct_iterate_cleanup(net, kill_all, NULL);
1355        nf_ct_release_dying_list(net);
1356        if (atomic_read(&net->ct.count) != 0) {
1357                schedule();
1358                goto i_see_dead_people;
1359        }
1360
1361        nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1362        nf_conntrack_helper_fini(net);
1363        nf_conntrack_timeout_fini(net);
1364        nf_conntrack_ecache_fini(net);
1365        nf_conntrack_tstamp_fini(net);
1366        nf_conntrack_acct_fini(net);
1367        nf_conntrack_expect_fini(net);
1368        kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1369        kfree(net->ct.slabname);
1370        free_percpu(net->ct.stat);
1371}
1372
1373/* Mishearing the voices in his head, our hero wonders how he's
1374   supposed to kill the mall. */
1375void nf_conntrack_cleanup(struct net *net)
1376{
1377        if (net_eq(net, &init_net))
1378                RCU_INIT_POINTER(ip_ct_attach, NULL);
1379
1380        /* This makes sure all current packets have passed through
1381           netfilter framework.  Roll on, two-stage module
1382           delete... */
1383        synchronize_net();
1384        nf_conntrack_proto_fini(net);
1385        nf_conntrack_cleanup_net(net);
1386
1387        if (net_eq(net, &init_net)) {
1388                RCU_INIT_POINTER(nf_ct_destroy, NULL);
1389                nf_conntrack_cleanup_init_net();
1390        }
1391}
1392
1393void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1394{
1395        struct hlist_nulls_head *hash;
1396        unsigned int nr_slots, i;
1397        size_t sz;
1398
1399        BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1400        nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1401        sz = nr_slots * sizeof(struct hlist_nulls_head);
1402        hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1403                                        get_order(sz));
1404        if (!hash) {
1405                printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
1406                hash = vzalloc(sz);
1407        }
1408
1409        if (hash && nulls)
1410                for (i = 0; i < nr_slots; i++)
1411                        INIT_HLIST_NULLS_HEAD(&hash[i], i);
1412
1413        return hash;
1414}
1415EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1416
1417int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1418{
1419        int i, bucket;
1420        unsigned int hashsize, old_size;
1421        struct hlist_nulls_head *hash, *old_hash;
1422        struct nf_conntrack_tuple_hash *h;
1423        struct nf_conn *ct;
1424
1425        if (current->nsproxy->net_ns != &init_net)
1426                return -EOPNOTSUPP;
1427
1428        /* On boot, we can set this without any fancy locking. */
1429        if (!nf_conntrack_htable_size)
1430                return param_set_uint(val, kp);
1431
1432        hashsize = simple_strtoul(val, NULL, 0);
1433        if (!hashsize)
1434                return -EINVAL;
1435
1436        hash = nf_ct_alloc_hashtable(&hashsize, 1);
1437        if (!hash)
1438                return -ENOMEM;
1439
1440        /* Lookups in the old hash might happen in parallel, which means we
1441         * might get false negatives during connection lookup. New connections
1442         * created because of a false negative won't make it into the hash
1443         * though since that required taking the lock.
1444         */
1445        spin_lock_bh(&nf_conntrack_lock);
1446        for (i = 0; i < init_net.ct.htable_size; i++) {
1447                while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
1448                        h = hlist_nulls_entry(init_net.ct.hash[i].first,
1449                                        struct nf_conntrack_tuple_hash, hnnode);
1450                        ct = nf_ct_tuplehash_to_ctrack(h);
1451                        hlist_nulls_del_rcu(&h->hnnode);
1452                        bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
1453                                                  hashsize);
1454                        hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1455                }
1456        }
1457        old_size = init_net.ct.htable_size;
1458        old_hash = init_net.ct.hash;
1459
1460        init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
1461        init_net.ct.hash = hash;
1462        spin_unlock_bh(&nf_conntrack_lock);
1463
1464        nf_ct_free_hashtable(old_hash, old_size);
1465        return 0;
1466}
1467EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1468
1469module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1470                  &nf_conntrack_htable_size, 0600);
1471
1472void nf_ct_untracked_status_or(unsigned long bits)
1473{
1474        int cpu;
1475
1476        for_each_possible_cpu(cpu)
1477                per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1478}
1479EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1480
1481static int nf_conntrack_init_init_net(void)
1482{
1483        int max_factor = 8;
1484        int ret, cpu;
1485
1486        /* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
1487         * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
1488        if (!nf_conntrack_htable_size) {
1489                nf_conntrack_htable_size
1490                        = (((totalram_pages << PAGE_SHIFT) / 16384)
1491                           / sizeof(struct hlist_head));
1492                if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1493                        nf_conntrack_htable_size = 16384;
1494                if (nf_conntrack_htable_size < 32)
1495                        nf_conntrack_htable_size = 32;
1496
1497                /* Use a max. factor of four by default to get the same max as
1498                 * with the old struct list_heads. When a table size is given
1499                 * we use the old value of 8 to avoid reducing the max.
1500                 * entries. */
1501                max_factor = 4;
1502        }
1503        nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1504
1505        printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1506               NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1507               nf_conntrack_max);
1508#ifdef CONFIG_NF_CONNTRACK_ZONES
1509        ret = nf_ct_extend_register(&nf_ct_zone_extend);
1510        if (ret < 0)
1511                goto err_extend;
1512#endif
1513        /* Set up fake conntrack: to never be deleted, not in any hashes */
1514        for_each_possible_cpu(cpu) {
1515                struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1516                write_pnet(&ct->ct_net, &init_net);
1517                atomic_set(&ct->ct_general.use, 1);
1518        }
1519        /*  - and look it like as a confirmed connection */
1520        nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
1521        return 0;
1522
1523#ifdef CONFIG_NF_CONNTRACK_ZONES
1524err_extend:
1525#endif
1526        return ret;
1527}
1528
1529/*
1530 * We need to use special "null" values, not used in hash table
1531 */
1532#define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
1533#define DYING_NULLS_VAL         ((1<<30)+1)
1534
1535static int nf_conntrack_init_net(struct net *net)
1536{
1537        int ret;
1538
1539        atomic_set(&net->ct.count, 0);
1540        INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
1541        INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
1542        net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1543        if (!net->ct.stat) {
1544                ret = -ENOMEM;
1545                goto err_stat;
1546        }
1547
1548        net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
1549        if (!net->ct.slabname) {
1550                ret = -ENOMEM;
1551                goto err_slabname;
1552        }
1553
1554        net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
1555                                                        sizeof(struct nf_conn), 0,
1556                                                        SLAB_DESTROY_BY_RCU, NULL);
1557        if (!net->ct.nf_conntrack_cachep) {
1558                printk(KERN_ERR "Unable to create nf_conn slab cache\n");
1559                ret = -ENOMEM;
1560                goto err_cache;
1561        }
1562
1563        net->ct.htable_size = nf_conntrack_htable_size;
1564        net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
1565        if (!net->ct.hash) {
1566                ret = -ENOMEM;
1567                printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
1568                goto err_hash;
1569        }
1570        ret = nf_conntrack_expect_init(net);
1571        if (ret < 0)
1572                goto err_expect;
1573        ret = nf_conntrack_acct_init(net);
1574        if (ret < 0)
1575                goto err_acct;
1576        ret = nf_conntrack_tstamp_init(net);
1577        if (ret < 0)
1578                goto err_tstamp;
1579        ret = nf_conntrack_ecache_init(net);
1580        if (ret < 0)
1581                goto err_ecache;
1582        ret = nf_conntrack_timeout_init(net);
1583        if (ret < 0)
1584                goto err_timeout;
1585        ret = nf_conntrack_helper_init(net);
1586        if (ret < 0)
1587                goto err_helper;
1588        return 0;
1589err_helper:
1590        nf_conntrack_timeout_fini(net);
1591err_timeout:
1592        nf_conntrack_ecache_fini(net);
1593err_ecache:
1594        nf_conntrack_tstamp_fini(net);
1595err_tstamp:
1596        nf_conntrack_acct_fini(net);
1597err_acct:
1598        nf_conntrack_expect_fini(net);
1599err_expect:
1600        nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1601err_hash:
1602        kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1603err_cache:
1604        kfree(net->ct.slabname);
1605err_slabname:
1606        free_percpu(net->ct.stat);
1607err_stat:
1608        return ret;
1609}
1610
1611s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
1612                        enum ip_conntrack_dir dir,
1613                        u32 seq);
1614EXPORT_SYMBOL_GPL(nf_ct_nat_offset);
1615
1616int nf_conntrack_init(struct net *net)
1617{
1618        int ret;
1619
1620        if (net_eq(net, &init_net)) {
1621                ret = nf_conntrack_init_init_net();
1622                if (ret < 0)
1623                        goto out_init_net;
1624        }
1625        ret = nf_conntrack_proto_init(net);
1626        if (ret < 0)
1627                goto out_proto;
1628        ret = nf_conntrack_init_net(net);
1629        if (ret < 0)
1630                goto out_net;
1631
1632        if (net_eq(net, &init_net)) {
1633                /* For use by REJECT target */
1634                RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1635                RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1636
1637                /* Howto get NAT offsets */
1638                RCU_INIT_POINTER(nf_ct_nat_offset, NULL);
1639        }
1640        return 0;
1641
1642out_net:
1643        nf_conntrack_proto_fini(net);
1644out_proto:
1645        if (net_eq(net, &init_net))
1646                nf_conntrack_cleanup_init_net();
1647out_init_net:
1648        return ret;
1649}
1650
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.