linux/net/core/dev.c
<<
>>
Prefs
   1/*
   2 *      NET3    Protocol independent device support routines.
   3 *
   4 *              This program is free software; you can redistribute it and/or
   5 *              modify it under the terms of the GNU General Public License
   6 *              as published by the Free Software Foundation; either version
   7 *              2 of the License, or (at your option) any later version.
   8 *
   9 *      Derived from the non IP parts of dev.c 1.0.19
  10 *              Authors:        Ross Biro
  11 *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12 *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
  13 *
  14 *      Additional Authors:
  15 *              Florian la Roche <rzsfl@rz.uni-sb.de>
  16 *              Alan Cox <gw4pts@gw4pts.ampr.org>
  17 *              David Hinds <dahinds@users.sourceforge.net>
  18 *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  19 *              Adam Sulmicki <adam@cfar.umd.edu>
  20 *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  21 *
  22 *      Changes:
  23 *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  24 *                                      to 2 if register_netdev gets called
  25 *                                      before net_dev_init & also removed a
  26 *                                      few lines of code in the process.
  27 *              Alan Cox        :       device private ioctl copies fields back.
  28 *              Alan Cox        :       Transmit queue code does relevant
  29 *                                      stunts to keep the queue safe.
  30 *              Alan Cox        :       Fixed double lock.
  31 *              Alan Cox        :       Fixed promisc NULL pointer trap
  32 *              ????????        :       Support the full private ioctl range
  33 *              Alan Cox        :       Moved ioctl permission check into
  34 *                                      drivers
  35 *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
  36 *              Alan Cox        :       100 backlog just doesn't cut it when
  37 *                                      you start doing multicast video 8)
  38 *              Alan Cox        :       Rewrote net_bh and list manager.
  39 *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
  40 *              Alan Cox        :       Took out transmit every packet pass
  41 *                                      Saved a few bytes in the ioctl handler
  42 *              Alan Cox        :       Network driver sets packet type before
  43 *                                      calling netif_rx. Saves a function
  44 *                                      call a packet.
  45 *              Alan Cox        :       Hashed net_bh()
  46 *              Richard Kooijman:       Timestamp fixes.
  47 *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
  48 *              Alan Cox        :       Device lock protection.
  49 *              Alan Cox        :       Fixed nasty side effect of device close
  50 *                                      changes.
  51 *              Rudi Cilibrasi  :       Pass the right thing to
  52 *                                      set_mac_address()
  53 *              Dave Miller     :       32bit quantity for the device lock to
  54 *                                      make it work out on a Sparc.
  55 *              Bjorn Ekwall    :       Added KERNELD hack.
  56 *              Alan Cox        :       Cleaned up the backlog initialise.
  57 *              Craig Metz      :       SIOCGIFCONF fix if space for under
  58 *                                      1 device.
  59 *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
  60 *                                      is no device open function.
  61 *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
  62 *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
  63 *              Cyrus Durgin    :       Cleaned for KMOD
  64 *              Adam Sulmicki   :       Bug Fix : Network Device Unload
  65 *                                      A network device unload needs to purge
  66 *                                      the backlog queue.
  67 *      Paul Rusty Russell      :       SIOCSIFNAME
  68 *              Pekka Riikonen  :       Netdev boot-time settings code
  69 *              Andrew Morton   :       Make unregister_netdevice wait
  70 *                                      indefinitely on dev->refcnt
  71 *              J Hadi Salim    :       - Backlog queue sampling
  72 *                                      - netif_rx() feedback
  73 */
  74
  75#include <asm/uaccess.h>
  76#include <asm/system.h>
  77#include <linux/bitops.h>
  78#include <linux/capability.h>
  79#include <linux/cpu.h>
  80#include <linux/types.h>
  81#include <linux/kernel.h>
  82#include <linux/hash.h>
  83#include <linux/slab.h>
  84#include <linux/sched.h>
  85#include <linux/mutex.h>
  86#include <linux/string.h>
  87#include <linux/mm.h>
  88#include <linux/socket.h>
  89#include <linux/sockios.h>
  90#include <linux/errno.h>
  91#include <linux/interrupt.h>
  92#include <linux/if_ether.h>
  93#include <linux/netdevice.h>
  94#include <linux/etherdevice.h>
  95#include <linux/ethtool.h>
  96#include <linux/notifier.h>
  97#include <linux/skbuff.h>
  98#include <net/net_namespace.h>
  99#include <net/sock.h>
 100#include <linux/rtnetlink.h>
 101#include <linux/proc_fs.h>
 102#include <linux/seq_file.h>
 103#include <linux/stat.h>
 104#include <linux/if_bridge.h>
 105#include <linux/if_macvlan.h>
 106#include <net/dst.h>
 107#include <net/pkt_sched.h>
 108#include <net/checksum.h>
 109#include <net/xfrm.h>
 110#include <linux/highmem.h>
 111#include <linux/init.h>
 112#include <linux/kmod.h>
 113#include <linux/module.h>
 114#include <linux/netpoll.h>
 115#include <linux/rcupdate.h>
 116#include <linux/delay.h>
 117#include <net/wext.h>
 118#include <net/iw_handler.h>
 119#include <asm/current.h>
 120#include <linux/audit.h>
 121#include <linux/dmaengine.h>
 122#include <linux/err.h>
 123#include <linux/ctype.h>
 124#include <linux/if_arp.h>
 125#include <linux/if_vlan.h>
 126#include <linux/ip.h>
 127#include <net/ip.h>
 128#include <linux/ipv6.h>
 129#include <linux/in.h>
 130#include <linux/jhash.h>
 131#include <linux/random.h>
 132#include <trace/events/napi.h>
 133#include <linux/pci.h>
 134
 135#include "net-sysfs.h"
 136
 137/* Instead of increasing this, you should create a hash table. */
 138#define MAX_GRO_SKBS 8
 139
 140/* This should be increased if a protocol with a bigger head is added. */
 141#define GRO_MAX_HEAD (MAX_HEADER + 128)
 142
 143/*
 144 *      The list of packet types we will receive (as opposed to discard)
 145 *      and the routines to invoke.
 146 *
 147 *      Why 16. Because with 16 the only overlap we get on a hash of the
 148 *      low nibble of the protocol value is RARP/SNAP/X.25.
 149 *
 150 *      NOTE:  That is no longer true with the addition of VLAN tags.  Not
 151 *             sure which should go first, but I bet it won't make much
 152 *             difference if we are running VLANs.  The good news is that
 153 *             this protocol won't be in the list unless compiled in, so
 154 *             the average user (w/out VLANs) will not be adversely affected.
 155 *             --BLG
 156 *
 157 *              0800    IP
 158 *              8100    802.1Q VLAN
 159 *              0001    802.3
 160 *              0002    AX.25
 161 *              0004    802.2
 162 *              8035    RARP
 163 *              0005    SNAP
 164 *              0805    X.25
 165 *              0806    ARP
 166 *              8137    IPX
 167 *              0009    Localtalk
 168 *              86DD    IPv6
 169 */
 170
 171#define PTYPE_HASH_SIZE (16)
 172#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
 173
 174static DEFINE_SPINLOCK(ptype_lock);
 175static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 176static struct list_head ptype_all __read_mostly;        /* Taps */
 177
 178/*
 179 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
 180 * semaphore.
 181 *
 182 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
 183 *
 184 * Writers must hold the rtnl semaphore while they loop through the
 185 * dev_base_head list, and hold dev_base_lock for writing when they do the
 186 * actual updates.  This allows pure readers to access the list even
 187 * while a writer is preparing to update it.
 188 *
 189 * To put it another way, dev_base_lock is held for writing only to
 190 * protect against pure readers; the rtnl semaphore provides the
 191 * protection against other writers.
 192 *
 193 * See, for example usages, register_netdevice() and
 194 * unregister_netdevice(), which must be called with the rtnl
 195 * semaphore held.
 196 */
 197DEFINE_RWLOCK(dev_base_lock);
 198EXPORT_SYMBOL(dev_base_lock);
 199
 200static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 201{
 202        unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
 203        return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 204}
 205
 206static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 207{
 208        return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 209}
 210
 211static inline void rps_lock(struct softnet_data *sd)
 212{
 213#ifdef CONFIG_RPS
 214        spin_lock(&sd->input_pkt_queue.lock);
 215#endif
 216}
 217
 218static inline void rps_unlock(struct softnet_data *sd)
 219{
 220#ifdef CONFIG_RPS
 221        spin_unlock(&sd->input_pkt_queue.lock);
 222#endif
 223}
 224
 225/* Device list insertion */
 226static int list_netdevice(struct net_device *dev)
 227{
 228        struct net *net = dev_net(dev);
 229
 230        ASSERT_RTNL();
 231
 232        write_lock_bh(&dev_base_lock);
 233        list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 234        hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
 235        hlist_add_head_rcu(&dev->index_hlist,
 236                           dev_index_hash(net, dev->ifindex));
 237        write_unlock_bh(&dev_base_lock);
 238        return 0;
 239}
 240
 241/* Device list removal
 242 * caller must respect a RCU grace period before freeing/reusing dev
 243 */
 244static void unlist_netdevice(struct net_device *dev)
 245{
 246        ASSERT_RTNL();
 247
 248        /* Unlink dev from the device chain */
 249        write_lock_bh(&dev_base_lock);
 250        list_del_rcu(&dev->dev_list);
 251        hlist_del_rcu(&dev->name_hlist);
 252        hlist_del_rcu(&dev->index_hlist);
 253        write_unlock_bh(&dev_base_lock);
 254}
 255
 256/*
 257 *      Our notifier list
 258 */
 259
 260static RAW_NOTIFIER_HEAD(netdev_chain);
 261
 262/*
 263 *      Device drivers call our routines to queue packets here. We empty the
 264 *      queue in the local softnet handler.
 265 */
 266
 267DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 268EXPORT_PER_CPU_SYMBOL(softnet_data);
 269
 270#ifdef CONFIG_LOCKDEP
 271/*
 272 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
 273 * according to dev->type
 274 */
 275static const unsigned short netdev_lock_type[] =
 276        {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 277         ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 278         ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 279         ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 280         ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 281         ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 282         ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 283         ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 284         ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 285         ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 286         ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 287         ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 288         ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
 289         ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
 290         ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
 291         ARPHRD_VOID, ARPHRD_NONE};
 292
 293static const char *const netdev_lock_name[] =
 294        {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 295         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 296         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 297         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 298         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 299         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 300         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 301         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 302         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 303         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 304         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 305         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 306         "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
 307         "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
 308         "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
 309         "_xmit_VOID", "_xmit_NONE"};
 310
 311static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 312static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 313
 314static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 315{
 316        int i;
 317
 318        for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 319                if (netdev_lock_type[i] == dev_type)
 320                        return i;
 321        /* the last key is used by default */
 322        return ARRAY_SIZE(netdev_lock_type) - 1;
 323}
 324
 325static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 326                                                 unsigned short dev_type)
 327{
 328        int i;
 329
 330        i = netdev_lock_pos(dev_type);
 331        lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 332                                   netdev_lock_name[i]);
 333}
 334
 335static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 336{
 337        int i;
 338
 339        i = netdev_lock_pos(dev->type);
 340        lockdep_set_class_and_name(&dev->addr_list_lock,
 341                                   &netdev_addr_lock_key[i],
 342                                   netdev_lock_name[i]);
 343}
 344#else
 345static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 346                                                 unsigned short dev_type)
 347{
 348}
 349static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 350{
 351}
 352#endif
 353
 354/*******************************************************************************
 355
 356                Protocol management and registration routines
 357
 358*******************************************************************************/
 359
 360/*
 361 *      Add a protocol ID to the list. Now that the input handler is
 362 *      smarter we can dispense with all the messy stuff that used to be
 363 *      here.
 364 *
 365 *      BEWARE!!! Protocol handlers, mangling input packets,
 366 *      MUST BE last in hash buckets and checking protocol handlers
 367 *      MUST start from promiscuous ptype_all chain in net_bh.
 368 *      It is true now, do not change it.
 369 *      Explanation follows: if protocol handler, mangling packet, will
 370 *      be the first on list, it is not able to sense, that packet
 371 *      is cloned and should be copied-on-write, so that it will
 372 *      change it and subsequent readers will get broken packet.
 373 *                                                      --ANK (980803)
 374 */
 375
 376/**
 377 *      dev_add_pack - add packet handler
 378 *      @pt: packet type declaration
 379 *
 380 *      Add a protocol handler to the networking stack. The passed &packet_type
 381 *      is linked into kernel lists and may not be freed until it has been
 382 *      removed from the kernel lists.
 383 *
 384 *      This call does not sleep therefore it can not
 385 *      guarantee all CPU's that are in middle of receiving packets
 386 *      will see the new packet type (until the next received packet).
 387 */
 388
 389void dev_add_pack(struct packet_type *pt)
 390{
 391        int hash;
 392
 393        spin_lock_bh(&ptype_lock);
 394        if (pt->type == htons(ETH_P_ALL))
 395                list_add_rcu(&pt->list, &ptype_all);
 396        else {
 397                hash = ntohs(pt->type) & PTYPE_HASH_MASK;
 398                list_add_rcu(&pt->list, &ptype_base[hash]);
 399        }
 400        spin_unlock_bh(&ptype_lock);
 401}
 402EXPORT_SYMBOL(dev_add_pack);
 403
 404/**
 405 *      __dev_remove_pack        - remove packet handler
 406 *      @pt: packet type declaration
 407 *
 408 *      Remove a protocol handler that was previously added to the kernel
 409 *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 410 *      from the kernel lists and can be freed or reused once this function
 411 *      returns.
 412 *
 413 *      The packet type might still be in use by receivers
 414 *      and must not be freed until after all the CPU's have gone
 415 *      through a quiescent state.
 416 */
 417void __dev_remove_pack(struct packet_type *pt)
 418{
 419        struct list_head *head;
 420        struct packet_type *pt1;
 421
 422        spin_lock_bh(&ptype_lock);
 423
 424        if (pt->type == htons(ETH_P_ALL))
 425                head = &ptype_all;
 426        else
 427                head = &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 428
 429        list_for_each_entry(pt1, head, list) {
 430                if (pt == pt1) {
 431                        list_del_rcu(&pt->list);
 432                        goto out;
 433                }
 434        }
 435
 436        printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
 437out:
 438        spin_unlock_bh(&ptype_lock);
 439}
 440EXPORT_SYMBOL(__dev_remove_pack);
 441
 442/**
 443 *      dev_remove_pack  - remove packet handler
 444 *      @pt: packet type declaration
 445 *
 446 *      Remove a protocol handler that was previously added to the kernel
 447 *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 448 *      from the kernel lists and can be freed or reused once this function
 449 *      returns.
 450 *
 451 *      This call sleeps to guarantee that no CPU is looking at the packet
 452 *      type after return.
 453 */
 454void dev_remove_pack(struct packet_type *pt)
 455{
 456        __dev_remove_pack(pt);
 457
 458        synchronize_net();
 459}
 460EXPORT_SYMBOL(dev_remove_pack);
 461
 462/******************************************************************************
 463
 464                      Device Boot-time Settings Routines
 465
 466*******************************************************************************/
 467
 468/* Boot time configuration table */
 469static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
 470
 471/**
 472 *      netdev_boot_setup_add   - add new setup entry
 473 *      @name: name of the device
 474 *      @map: configured settings for the device
 475 *
 476 *      Adds new setup entry to the dev_boot_setup list.  The function
 477 *      returns 0 on error and 1 on success.  This is a generic routine to
 478 *      all netdevices.
 479 */
 480static int netdev_boot_setup_add(char *name, struct ifmap *map)
 481{
 482        struct netdev_boot_setup *s;
 483        int i;
 484
 485        s = dev_boot_setup;
 486        for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 487                if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
 488                        memset(s[i].name, 0, sizeof(s[i].name));
 489                        strlcpy(s[i].name, name, IFNAMSIZ);
 490                        memcpy(&s[i].map, map, sizeof(s[i].map));
 491                        break;
 492                }
 493        }
 494
 495        return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
 496}
 497
 498/**
 499 *      netdev_boot_setup_check - check boot time settings
 500 *      @dev: the netdevice
 501 *
 502 *      Check boot time settings for the device.
 503 *      The found settings are set for the device to be used
 504 *      later in the device probing.
 505 *      Returns 0 if no settings found, 1 if they are.
 506 */
 507int netdev_boot_setup_check(struct net_device *dev)
 508{
 509        struct netdev_boot_setup *s = dev_boot_setup;
 510        int i;
 511
 512        for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 513                if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
 514                    !strcmp(dev->name, s[i].name)) {
 515                        dev->irq        = s[i].map.irq;
 516                        dev->base_addr  = s[i].map.base_addr;
 517                        dev->mem_start  = s[i].map.mem_start;
 518                        dev->mem_end    = s[i].map.mem_end;
 519                        return 1;
 520                }
 521        }
 522        return 0;
 523}
 524EXPORT_SYMBOL(netdev_boot_setup_check);
 525
 526
 527/**
 528 *      netdev_boot_base        - get address from boot time settings
 529 *      @prefix: prefix for network device
 530 *      @unit: id for network device
 531 *
 532 *      Check boot time settings for the base address of device.
 533 *      The found settings are set for the device to be used
 534 *      later in the device probing.
 535 *      Returns 0 if no settings found.
 536 */
 537unsigned long netdev_boot_base(const char *prefix, int unit)
 538{
 539        const struct netdev_boot_setup *s = dev_boot_setup;
 540        char name[IFNAMSIZ];
 541        int i;
 542
 543        sprintf(name, "%s%d", prefix, unit);
 544
 545        /*
 546         * If device already registered then return base of 1
 547         * to indicate not to probe for this interface
 548         */
 549        if (__dev_get_by_name(&init_net, name))
 550                return 1;
 551
 552        for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
 553                if (!strcmp(name, s[i].name))
 554                        return s[i].map.base_addr;
 555        return 0;
 556}
 557
 558/*
 559 * Saves at boot time configured settings for any netdevice.
 560 */
 561int __init netdev_boot_setup(char *str)
 562{
 563        int ints[5];
 564        struct ifmap map;
 565
 566        str = get_options(str, ARRAY_SIZE(ints), ints);
 567        if (!str || !*str)
 568                return 0;
 569
 570        /* Save settings */
 571        memset(&map, 0, sizeof(map));
 572        if (ints[0] > 0)
 573                map.irq = ints[1];
 574        if (ints[0] > 1)
 575                map.base_addr = ints[2];
 576        if (ints[0] > 2)
 577                map.mem_start = ints[3];
 578        if (ints[0] > 3)
 579                map.mem_end = ints[4];
 580
 581        /* Add new entry to the list */
 582        return netdev_boot_setup_add(str, &map);
 583}
 584
 585__setup("netdev=", netdev_boot_setup);
 586
 587/*******************************************************************************
 588
 589                            Device Interface Subroutines
 590
 591*******************************************************************************/
 592
 593/**
 594 *      __dev_get_by_name       - find a device by its name
 595 *      @net: the applicable net namespace
 596 *      @name: name to find
 597 *
 598 *      Find an interface by name. Must be called under RTNL semaphore
 599 *      or @dev_base_lock. If the name is found a pointer to the device
 600 *      is returned. If the name is not found then %NULL is returned. The
 601 *      reference counters are not incremented so the caller must be
 602 *      careful with locks.
 603 */
 604
 605struct net_device *__dev_get_by_name(struct net *net, const char *name)
 606{
 607        struct hlist_node *p;
 608        struct net_device *dev;
 609        struct hlist_head *head = dev_name_hash(net, name);
 610
 611        hlist_for_each_entry(dev, p, head, name_hlist)
 612                if (!strncmp(dev->name, name, IFNAMSIZ))
 613                        return dev;
 614
 615        return NULL;
 616}
 617EXPORT_SYMBOL(__dev_get_by_name);
 618
 619/**
 620 *      dev_get_by_name_rcu     - find a device by its name
 621 *      @net: the applicable net namespace
 622 *      @name: name to find
 623 *
 624 *      Find an interface by name.
 625 *      If the name is found a pointer to the device is returned.
 626 *      If the name is not found then %NULL is returned.
 627 *      The reference counters are not incremented so the caller must be
 628 *      careful with locks. The caller must hold RCU lock.
 629 */
 630
 631struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 632{
 633        struct hlist_node *p;
 634        struct net_device *dev;
 635        struct hlist_head *head = dev_name_hash(net, name);
 636
 637        hlist_for_each_entry_rcu(dev, p, head, name_hlist)
 638                if (!strncmp(dev->name, name, IFNAMSIZ))
 639                        return dev;
 640
 641        return NULL;
 642}
 643EXPORT_SYMBOL(dev_get_by_name_rcu);
 644
 645/**
 646 *      dev_get_by_name         - find a device by its name
 647 *      @net: the applicable net namespace
 648 *      @name: name to find
 649 *
 650 *      Find an interface by name. This can be called from any
 651 *      context and does its own locking. The returned handle has
 652 *      the usage count incremented and the caller must use dev_put() to
 653 *      release it when it is no longer needed. %NULL is returned if no
 654 *      matching device is found.
 655 */
 656
 657struct net_device *dev_get_by_name(struct net *net, const char *name)
 658{
 659        struct net_device *dev;
 660
 661        rcu_read_lock();
 662        dev = dev_get_by_name_rcu(net, name);
 663        if (dev)
 664                dev_hold(dev);
 665        rcu_read_unlock();
 666        return dev;
 667}
 668EXPORT_SYMBOL(dev_get_by_name);
 669
 670/**
 671 *      __dev_get_by_index - find a device by its ifindex
 672 *      @net: the applicable net namespace
 673 *      @ifindex: index of device
 674 *
 675 *      Search for an interface by index. Returns %NULL if the device
 676 *      is not found or a pointer to the device. The device has not
 677 *      had its reference counter increased so the caller must be careful
 678 *      about locking. The caller must hold either the RTNL semaphore
 679 *      or @dev_base_lock.
 680 */
 681
 682struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 683{
 684        struct hlist_node *p;
 685        struct net_device *dev;
 686        struct hlist_head *head = dev_index_hash(net, ifindex);
 687
 688        hlist_for_each_entry(dev, p, head, index_hlist)
 689                if (dev->ifindex == ifindex)
 690                        return dev;
 691
 692        return NULL;
 693}
 694EXPORT_SYMBOL(__dev_get_by_index);
 695
 696/**
 697 *      dev_get_by_index_rcu - find a device by its ifindex
 698 *      @net: the applicable net namespace
 699 *      @ifindex: index of device
 700 *
 701 *      Search for an interface by index. Returns %NULL if the device
 702 *      is not found or a pointer to the device. The device has not
 703 *      had its reference counter increased so the caller must be careful
 704 *      about locking. The caller must hold RCU lock.
 705 */
 706
 707struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 708{
 709        struct hlist_node *p;
 710        struct net_device *dev;
 711        struct hlist_head *head = dev_index_hash(net, ifindex);
 712
 713        hlist_for_each_entry_rcu(dev, p, head, index_hlist)
 714                if (dev->ifindex == ifindex)
 715                        return dev;
 716
 717        return NULL;
 718}
 719EXPORT_SYMBOL(dev_get_by_index_rcu);
 720
 721
 722/**
 723 *      dev_get_by_index - find a device by its ifindex
 724 *      @net: the applicable net namespace
 725 *      @ifindex: index of device
 726 *
 727 *      Search for an interface by index. Returns NULL if the device
 728 *      is not found or a pointer to the device. The device returned has
 729 *      had a reference added and the pointer is safe until the user calls
 730 *      dev_put to indicate they have finished with it.
 731 */
 732
 733struct net_device *dev_get_by_index(struct net *net, int ifindex)
 734{
 735        struct net_device *dev;
 736
 737        rcu_read_lock();
 738        dev = dev_get_by_index_rcu(net, ifindex);
 739        if (dev)
 740                dev_hold(dev);
 741        rcu_read_unlock();
 742        return dev;
 743}
 744EXPORT_SYMBOL(dev_get_by_index);
 745
 746/**
 747 *      dev_getbyhwaddr - find a device by its hardware address
 748 *      @net: the applicable net namespace
 749 *      @type: media type of device
 750 *      @ha: hardware address
 751 *
 752 *      Search for an interface by MAC address. Returns NULL if the device
 753 *      is not found or a pointer to the device. The caller must hold the
 754 *      rtnl semaphore. The returned device has not had its ref count increased
 755 *      and the caller must therefore be careful about locking
 756 *
 757 *      BUGS:
 758 *      If the API was consistent this would be __dev_get_by_hwaddr
 759 */
 760
 761struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
 762{
 763        struct net_device *dev;
 764
 765        ASSERT_RTNL();
 766
 767        for_each_netdev(net, dev)
 768                if (dev->type == type &&
 769                    !memcmp(dev->dev_addr, ha, dev->addr_len))
 770                        return dev;
 771
 772        return NULL;
 773}
 774EXPORT_SYMBOL(dev_getbyhwaddr);
 775
 776struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
 777{
 778        struct net_device *dev;
 779
 780        ASSERT_RTNL();
 781        for_each_netdev(net, dev)
 782                if (dev->type == type)
 783                        return dev;
 784
 785        return NULL;
 786}
 787EXPORT_SYMBOL(__dev_getfirstbyhwtype);
 788
 789struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 790{
 791        struct net_device *dev, *ret = NULL;
 792
 793        rcu_read_lock();
 794        for_each_netdev_rcu(net, dev)
 795                if (dev->type == type) {
 796                        dev_hold(dev);
 797                        ret = dev;
 798                        break;
 799                }
 800        rcu_read_unlock();
 801        return ret;
 802}
 803EXPORT_SYMBOL(dev_getfirstbyhwtype);
 804
 805/**
 806 *      dev_get_by_flags - find any device with given flags
 807 *      @net: the applicable net namespace
 808 *      @if_flags: IFF_* values
 809 *      @mask: bitmask of bits in if_flags to check
 810 *
 811 *      Search for any interface with the given flags. Returns NULL if a device
 812 *      is not found or a pointer to the device. The device returned has
 813 *      had a reference added and the pointer is safe until the user calls
 814 *      dev_put to indicate they have finished with it.
 815 */
 816
 817struct net_device *dev_get_by_flags(struct net *net, unsigned short if_flags,
 818                                    unsigned short mask)
 819{
 820        struct net_device *dev, *ret;
 821
 822        ret = NULL;
 823        rcu_read_lock();
 824        for_each_netdev_rcu(net, dev) {
 825                if (((dev->flags ^ if_flags) & mask) == 0) {
 826                        dev_hold(dev);
 827                        ret = dev;
 828                        break;
 829                }
 830        }
 831        rcu_read_unlock();
 832        return ret;
 833}
 834EXPORT_SYMBOL(dev_get_by_flags);
 835
 836/**
 837 *      dev_valid_name - check if name is okay for network device
 838 *      @name: name string
 839 *
 840 *      Network device names need to be valid file names to
 841 *      to allow sysfs to work.  We also disallow any kind of
 842 *      whitespace.
 843 */
 844int dev_valid_name(const char *name)
 845{
 846        if (*name == '\0')
 847                return 0;
 848        if (strlen(name) >= IFNAMSIZ)
 849                return 0;
 850        if (!strcmp(name, ".") || !strcmp(name, ".."))
 851                return 0;
 852
 853        while (*name) {
 854                if (*name == '/' || isspace(*name))
 855                        return 0;
 856                name++;
 857        }
 858        return 1;
 859}
 860EXPORT_SYMBOL(dev_valid_name);
 861
 862/**
 863 *      __dev_alloc_name - allocate a name for a device
 864 *      @net: network namespace to allocate the device name in
 865 *      @name: name format string
 866 *      @buf:  scratch buffer and result name string
 867 *
 868 *      Passed a format string - eg "lt%d" it will try and find a suitable
 869 *      id. It scans list of devices to build up a free map, then chooses
 870 *      the first empty slot. The caller must hold the dev_base or rtnl lock
 871 *      while allocating the name and adding the device in order to avoid
 872 *      duplicates.
 873 *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
 874 *      Returns the number of the unit assigned or a negative errno code.
 875 */
 876
 877static int __dev_alloc_name(struct net *net, const char *name, char *buf)
 878{
 879        int i = 0;
 880        const char *p;
 881        const int max_netdevices = 8*PAGE_SIZE;
 882        unsigned long *inuse;
 883        struct net_device *d;
 884
 885        p = strnchr(name, IFNAMSIZ-1, '%');
 886        if (p) {
 887                /*
 888                 * Verify the string as this thing may have come from
 889                 * the user.  There must be either one "%d" and no other "%"
 890                 * characters.
 891                 */
 892                if (p[1] != 'd' || strchr(p + 2, '%'))
 893                        return -EINVAL;
 894
 895                /* Use one page as a bit array of possible slots */
 896                inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
 897                if (!inuse)
 898                        return -ENOMEM;
 899
 900                for_each_netdev(net, d) {
 901                        if (!sscanf(d->name, name, &i))
 902                                continue;
 903                        if (i < 0 || i >= max_netdevices)
 904                                continue;
 905
 906                        /*  avoid cases where sscanf is not exact inverse of printf */
 907                        snprintf(buf, IFNAMSIZ, name, i);
 908                        if (!strncmp(buf, d->name, IFNAMSIZ))
 909                                set_bit(i, inuse);
 910                }
 911
 912                i = find_first_zero_bit(inuse, max_netdevices);
 913                free_page((unsigned long) inuse);
 914        }
 915
 916        if (buf != name)
 917                snprintf(buf, IFNAMSIZ, name, i);
 918        if (!__dev_get_by_name(net, buf))
 919                return i;
 920
 921        /* It is possible to run out of possible slots
 922         * when the name is long and there isn't enough space left
 923         * for the digits, or if all bits are used.
 924         */
 925        return -ENFILE;
 926}
 927
 928/**
 929 *      dev_alloc_name - allocate a name for a device
 930 *      @dev: device
 931 *      @name: name format string
 932 *
 933 *      Passed a format string - eg "lt%d" it will try and find a suitable
 934 *      id. It scans list of devices to build up a free map, then chooses
 935 *      the first empty slot. The caller must hold the dev_base or rtnl lock
 936 *      while allocating the name and adding the device in order to avoid
 937 *      duplicates.
 938 *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
 939 *      Returns the number of the unit assigned or a negative errno code.
 940 */
 941
 942int dev_alloc_name(struct net_device *dev, const char *name)
 943{
 944        char buf[IFNAMSIZ];
 945        struct net *net;
 946        int ret;
 947
 948        BUG_ON(!dev_net(dev));
 949        net = dev_net(dev);
 950        ret = __dev_alloc_name(net, name, buf);
 951        if (ret >= 0)
 952                strlcpy(dev->name, buf, IFNAMSIZ);
 953        return ret;
 954}
 955EXPORT_SYMBOL(dev_alloc_name);
 956
 957static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
 958{
 959        struct net *net;
 960
 961        BUG_ON(!dev_net(dev));
 962        net = dev_net(dev);
 963
 964        if (!dev_valid_name(name))
 965                return -EINVAL;
 966
 967        if (fmt && strchr(name, '%'))
 968                return dev_alloc_name(dev, name);
 969        else if (__dev_get_by_name(net, name))
 970                return -EEXIST;
 971        else if (dev->name != name)
 972                strlcpy(dev->name, name, IFNAMSIZ);
 973
 974        return 0;
 975}
 976
 977/**
 978 *      dev_change_name - change name of a device
 979 *      @dev: device
 980 *      @newname: name (or format string) must be at least IFNAMSIZ
 981 *
 982 *      Change name of a device, can pass format strings "eth%d".
 983 *      for wildcarding.
 984 */
 985int dev_change_name(struct net_device *dev, const char *newname)
 986{
 987        char oldname[IFNAMSIZ];
 988        int err = 0;
 989        int ret;
 990        struct net *net;
 991
 992        ASSERT_RTNL();
 993        BUG_ON(!dev_net(dev));
 994
 995        net = dev_net(dev);
 996        if (dev->flags & IFF_UP)
 997                return -EBUSY;
 998
 999        if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
1000                return 0;
1001
1002        memcpy(oldname, dev->name, IFNAMSIZ);
1003
1004        err = dev_get_valid_name(dev, newname, 1);
1005        if (err < 0)
1006                return err;
1007
1008rollback:
1009        ret = device_rename(&dev->dev, dev->name);
1010        if (ret) {
1011                memcpy(dev->name, oldname, IFNAMSIZ);
1012                return ret;
1013        }
1014
1015        write_lock_bh(&dev_base_lock);
1016        hlist_del(&dev->name_hlist);
1017        write_unlock_bh(&dev_base_lock);
1018
1019        synchronize_rcu();
1020
1021        write_lock_bh(&dev_base_lock);
1022        hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1023        write_unlock_bh(&dev_base_lock);
1024
1025        ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1026        ret = notifier_to_errno(ret);
1027
1028        if (ret) {
1029                /* err >= 0 after dev_alloc_name() or stores the first errno */
1030                if (err >= 0) {
1031                        err = ret;
1032                        memcpy(dev->name, oldname, IFNAMSIZ);
1033                        goto rollback;
1034                } else {
1035                        printk(KERN_ERR
1036                               "%s: name change rollback failed: %d.\n",
1037                               dev->name, ret);
1038                }
1039        }
1040
1041        return err;
1042}
1043
1044/**
1045 *      dev_set_alias - change ifalias of a device
1046 *      @dev: device
1047 *      @alias: name up to IFALIASZ
1048 *      @len: limit of bytes to copy from info
1049 *
1050 *      Set ifalias for a device,
1051 */
1052int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1053{
1054        ASSERT_RTNL();
1055
1056        if (len >= IFALIASZ)
1057                return -EINVAL;
1058
1059        if (!len) {
1060                if (dev->ifalias) {
1061                        kfree(dev->ifalias);
1062                        dev->ifalias = NULL;
1063                }
1064                return 0;
1065        }
1066
1067        dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1068        if (!dev->ifalias)
1069                return -ENOMEM;
1070
1071        strlcpy(dev->ifalias, alias, len+1);
1072        return len;
1073}
1074
1075
1076/**
1077 *      netdev_features_change - device changes features
1078 *      @dev: device to cause notification
1079 *
1080 *      Called to indicate a device has changed features.
1081 */
1082void netdev_features_change(struct net_device *dev)
1083{
1084        call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1085}
1086EXPORT_SYMBOL(netdev_features_change);
1087
1088/**
1089 *      netdev_state_change - device changes state
1090 *      @dev: device to cause notification
1091 *
1092 *      Called to indicate a device has changed state. This function calls
1093 *      the notifier chains for netdev_chain and sends a NEWLINK message
1094 *      to the routing socket.
1095 */
1096void netdev_state_change(struct net_device *dev)
1097{
1098        if (dev->flags & IFF_UP) {
1099                call_netdevice_notifiers(NETDEV_CHANGE, dev);
1100                rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1101        }
1102}
1103EXPORT_SYMBOL(netdev_state_change);
1104
1105int netdev_bonding_change(struct net_device *dev, unsigned long event)
1106{
1107        return call_netdevice_notifiers(event, dev);
1108}
1109EXPORT_SYMBOL(netdev_bonding_change);
1110
1111/**
1112 *      dev_load        - load a network module
1113 *      @net: the applicable net namespace
1114 *      @name: name of interface
1115 *
1116 *      If a network interface is not present and the process has suitable
1117 *      privileges this function loads the module. If module loading is not
1118 *      available in this kernel then it becomes a nop.
1119 */
1120
1121void dev_load(struct net *net, const char *name)
1122{
1123        struct net_device *dev;
1124        int no_module;
1125
1126        rcu_read_lock();
1127        dev = dev_get_by_name_rcu(net, name);
1128        rcu_read_unlock();
1129        no_module = !dev;
1130        if (no_module && capable(CAP_NET_ADMIN))
1131                no_module = request_module("netdev-%s", name);
1132        if (no_module && capable(CAP_SYS_MODULE)) {
1133                if (!request_module("%s", name))
1134                        pr_err("Loading kernel module for a network device "
1135"with CAP_SYS_MODULE (deprecated).  Use CAP_NET_ADMIN and alias netdev-%s "
1136"instead\n", name);
1137        }
1138}
1139EXPORT_SYMBOL(dev_load);
1140
1141static int __dev_open(struct net_device *dev)
1142{
1143        const struct net_device_ops *ops = dev->netdev_ops;
1144        int ret;
1145
1146        ASSERT_RTNL();
1147
1148        /*
1149         *      Is it even present?
1150         */
1151        if (!netif_device_present(dev))
1152                return -ENODEV;
1153
1154        ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1155        ret = notifier_to_errno(ret);
1156        if (ret)
1157                return ret;
1158
1159        /*
1160         *      Call device private open method
1161         */
1162        set_bit(__LINK_STATE_START, &dev->state);
1163
1164        if (ops->ndo_validate_addr)
1165                ret = ops->ndo_validate_addr(dev);
1166
1167        if (!ret && ops->ndo_open)
1168                ret = ops->ndo_open(dev);
1169
1170        /*
1171         *      If it went open OK then:
1172         */
1173
1174        if (ret)
1175                clear_bit(__LINK_STATE_START, &dev->state);
1176        else {
1177                /*
1178                 *      Set the flags.
1179                 */
1180                dev->flags |= IFF_UP;
1181
1182                /*
1183                 *      Enable NET_DMA
1184                 */
1185                net_dmaengine_get();
1186
1187                /*
1188                 *      Initialize multicasting status
1189                 */
1190                dev_set_rx_mode(dev);
1191
1192                /*
1193                 *      Wakeup transmit queue engine
1194                 */
1195                dev_activate(dev);
1196        }
1197
1198        return ret;
1199}
1200
1201/**
1202 *      dev_open        - prepare an interface for use.
1203 *      @dev:   device to open
1204 *
1205 *      Takes a device from down to up state. The device's private open
1206 *      function is invoked and then the multicast lists are loaded. Finally
1207 *      the device is moved into the up state and a %NETDEV_UP message is
1208 *      sent to the netdev notifier chain.
1209 *
1210 *      Calling this function on an active interface is a nop. On a failure
1211 *      a negative errno code is returned.
1212 */
1213int dev_open(struct net_device *dev)
1214{
1215        int ret;
1216
1217        /*
1218         *      Is it already up?
1219         */
1220        if (dev->flags & IFF_UP)
1221                return 0;
1222
1223        /*
1224         *      Open device
1225         */
1226        ret = __dev_open(dev);
1227        if (ret < 0)
1228                return ret;
1229
1230        /*
1231         *      ... and announce new interface.
1232         */
1233        rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1234        call_netdevice_notifiers(NETDEV_UP, dev);
1235
1236        return ret;
1237}
1238EXPORT_SYMBOL(dev_open);
1239
1240static int __dev_close(struct net_device *dev)
1241{
1242        const struct net_device_ops *ops = dev->netdev_ops;
1243
1244        ASSERT_RTNL();
1245        might_sleep();
1246
1247        /*
1248         *      Tell people we are going down, so that they can
1249         *      prepare to death, when device is still operating.
1250         */
1251        call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1252
1253        clear_bit(__LINK_STATE_START, &dev->state);
1254
1255        /* Synchronize to scheduled poll. We cannot touch poll list,
1256         * it can be even on different cpu. So just clear netif_running().
1257         *
1258         * dev->stop() will invoke napi_disable() on all of it's
1259         * napi_struct instances on this device.
1260         */
1261        smp_mb__after_clear_bit(); /* Commit netif_running(). */
1262
1263        dev_deactivate(dev);
1264
1265        /*
1266         *      Call the device specific close. This cannot fail.
1267         *      Only if device is UP
1268         *
1269         *      We allow it to be called even after a DETACH hot-plug
1270         *      event.
1271         */
1272        if (ops->ndo_stop)
1273                ops->ndo_stop(dev);
1274
1275        /*
1276         *      Device is now down.
1277         */
1278
1279        dev->flags &= ~IFF_UP;
1280
1281        /*
1282         *      Shutdown NET_DMA
1283         */
1284        net_dmaengine_put();
1285
1286        return 0;
1287}
1288
1289/**
1290 *      dev_close - shutdown an interface.
1291 *      @dev: device to shutdown
1292 *
1293 *      This function moves an active device into down state. A
1294 *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1295 *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1296 *      chain.
1297 */
1298int dev_close(struct net_device *dev)
1299{
1300        if (!(dev->flags & IFF_UP))
1301                return 0;
1302
1303        __dev_close(dev);
1304
1305        /*
1306         * Tell people we are down
1307         */
1308        rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1309        call_netdevice_notifiers(NETDEV_DOWN, dev);
1310
1311        return 0;
1312}
1313EXPORT_SYMBOL(dev_close);
1314
1315
1316/**
1317 *      dev_disable_lro - disable Large Receive Offload on a device
1318 *      @dev: device
1319 *
1320 *      Disable Large Receive Offload (LRO) on a net device.  Must be
1321 *      called under RTNL.  This is needed if received packets may be
1322 *      forwarded to another interface.
1323 */
1324void dev_disable_lro(struct net_device *dev)
1325{
1326        if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1327            dev->ethtool_ops->set_flags) {
1328                u32 flags = dev->ethtool_ops->get_flags(dev);
1329                if (flags & ETH_FLAG_LRO) {
1330                        flags &= ~ETH_FLAG_LRO;
1331                        dev->ethtool_ops->set_flags(dev, flags);
1332                }
1333        }
1334        WARN_ON(dev->features & NETIF_F_LRO);
1335}
1336EXPORT_SYMBOL(dev_disable_lro);
1337
1338
1339static int dev_boot_phase = 1;
1340
1341/*
1342 *      Device change register/unregister. These are not inline or static
1343 *      as we export them to the world.
1344 */
1345
1346/**
1347 *      register_netdevice_notifier - register a network notifier block
1348 *      @nb: notifier
1349 *
1350 *      Register a notifier to be called when network device events occur.
1351 *      The notifier passed is linked into the kernel structures and must
1352 *      not be reused until it has been unregistered. A negative errno code
1353 *      is returned on a failure.
1354 *
1355 *      When registered all registration and up events are replayed
1356 *      to the new notifier to allow device to have a race free
1357 *      view of the network device list.
1358 */
1359
1360int register_netdevice_notifier(struct notifier_block *nb)
1361{
1362        struct net_device *dev;
1363        struct net_device *last;
1364        struct net *net;
1365        int err;
1366
1367        rtnl_lock();
1368        err = raw_notifier_chain_register(&netdev_chain, nb);
1369        if (err)
1370                goto unlock;
1371        if (dev_boot_phase)
1372                goto unlock;
1373        for_each_net(net) {
1374                for_each_netdev(net, dev) {
1375                        err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1376                        err = notifier_to_errno(err);
1377                        if (err)
1378                                goto rollback;
1379
1380                        if (!(dev->flags & IFF_UP))
1381                                continue;
1382
1383                        nb->notifier_call(nb, NETDEV_UP, dev);
1384                }
1385        }
1386
1387unlock:
1388        rtnl_unlock();
1389        return err;
1390
1391rollback:
1392        last = dev;
1393        for_each_net(net) {
1394                for_each_netdev(net, dev) {
1395                        if (dev == last)
1396                                break;
1397
1398                        if (dev->flags & IFF_UP) {
1399                                nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1400                                nb->notifier_call(nb, NETDEV_DOWN, dev);
1401                        }
1402                        nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1403                        nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1404                }
1405        }
1406
1407        raw_notifier_chain_unregister(&netdev_chain, nb);
1408        goto unlock;
1409}
1410EXPORT_SYMBOL(register_netdevice_notifier);
1411
1412/**
1413 *      unregister_netdevice_notifier - unregister a network notifier block
1414 *      @nb: notifier
1415 *
1416 *      Unregister a notifier previously registered by
1417 *      register_netdevice_notifier(). The notifier is unlinked into the
1418 *      kernel structures and may then be reused. A negative errno code
1419 *      is returned on a failure.
1420 */
1421
1422int unregister_netdevice_notifier(struct notifier_block *nb)
1423{
1424        int err;
1425
1426        rtnl_lock();
1427        err = raw_notifier_chain_unregister(&netdev_chain, nb);
1428        rtnl_unlock();
1429        return err;
1430}
1431EXPORT_SYMBOL(unregister_netdevice_notifier);
1432
1433/**
1434 *      call_netdevice_notifiers - call all network notifier blocks
1435 *      @val: value passed unmodified to notifier function
1436 *      @dev: net_device pointer passed unmodified to notifier function
1437 *
1438 *      Call all network notifier blocks.  Parameters and return value
1439 *      are as for raw_notifier_call_chain().
1440 */
1441
1442int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1443{
1444        ASSERT_RTNL();
1445        return raw_notifier_call_chain(&netdev_chain, val, dev);
1446}
1447
1448/* When > 0 there are consumers of rx skb time stamps */
1449static atomic_t netstamp_needed = ATOMIC_INIT(0);
1450
1451void net_enable_timestamp(void)
1452{
1453        atomic_inc(&netstamp_needed);
1454}
1455EXPORT_SYMBOL(net_enable_timestamp);
1456
1457void net_disable_timestamp(void)
1458{
1459        atomic_dec(&netstamp_needed);
1460}
1461EXPORT_SYMBOL(net_disable_timestamp);
1462
1463static inline void net_timestamp_set(struct sk_buff *skb)
1464{
1465        if (atomic_read(&netstamp_needed))
1466                __net_timestamp(skb);
1467        else
1468                skb->tstamp.tv64 = 0;
1469}
1470
1471static inline void net_timestamp_check(struct sk_buff *skb)
1472{
1473        if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1474                __net_timestamp(skb);
1475}
1476
1477/**
1478 * dev_forward_skb - loopback an skb to another netif
1479 *
1480 * @dev: destination network device
1481 * @skb: buffer to forward
1482 *
1483 * return values:
1484 *      NET_RX_SUCCESS  (no congestion)
1485 *      NET_RX_DROP     (packet was dropped, but freed)
1486 *
1487 * dev_forward_skb can be used for injecting an skb from the
1488 * start_xmit function of one device into the receive queue
1489 * of another device.
1490 *
1491 * The receiving device may be in another namespace, so
1492 * we have to clear all information in the skb that could
1493 * impact namespace isolation.
1494 */
1495int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1496{
1497        skb_orphan(skb);
1498        nf_reset(skb);
1499
1500        if (!(dev->flags & IFF_UP) ||
1501            (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN))) {
1502                kfree_skb(skb);
1503                return NET_RX_DROP;
1504        }
1505        skb_set_dev(skb, dev);
1506        skb->tstamp.tv64 = 0;
1507        skb->pkt_type = PACKET_HOST;
1508        skb->protocol = eth_type_trans(skb, dev);
1509        return netif_rx(skb);
1510}
1511EXPORT_SYMBOL_GPL(dev_forward_skb);
1512
1513/*
1514 *      Support routine. Sends outgoing frames to any network
1515 *      taps currently in use.
1516 */
1517
1518static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1519{
1520        struct packet_type *ptype;
1521
1522#ifdef CONFIG_NET_CLS_ACT
1523        if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
1524                net_timestamp_set(skb);
1525#else
1526        net_timestamp_set(skb);
1527#endif
1528
1529        rcu_read_lock();
1530        list_for_each_entry_rcu(ptype, &ptype_all, list) {
1531                /* Never send packets back to the socket
1532                 * they originated from - MvS (miquels@drinkel.ow.org)
1533                 */
1534                if ((ptype->dev == dev || !ptype->dev) &&
1535                    (ptype->af_packet_priv == NULL ||
1536                     (struct sock *)ptype->af_packet_priv != skb->sk)) {
1537                        struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
1538                        if (!skb2)
1539                                break;
1540
1541                        /* skb->nh should be correctly
1542                           set by sender, so that the second statement is
1543                           just protection against buggy protocols.
1544                         */
1545                        skb_reset_mac_header(skb2);
1546
1547                        if (skb_network_header(skb2) < skb2->data ||
1548                            skb2->network_header > skb2->tail) {
1549                                if (net_ratelimit())
1550                                        printk(KERN_CRIT "protocol %04x is "
1551                                               "buggy, dev %s\n",
1552                                               skb2->protocol, dev->name);
1553                                skb_reset_network_header(skb2);
1554                        }
1555
1556                        skb2->transport_header = skb2->network_header;
1557                        skb2->pkt_type = PACKET_OUTGOING;
1558                        ptype->func(skb2, skb->dev, ptype, skb->dev);
1559                }
1560        }
1561        rcu_read_unlock();
1562}
1563
1564/*
1565 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1566 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1567 */
1568void netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1569{
1570        unsigned int real_num = dev->real_num_tx_queues;
1571
1572        if (unlikely(txq > dev->num_tx_queues))
1573                ;
1574        else if (txq > real_num)
1575                dev->real_num_tx_queues = txq;
1576        else if (txq < real_num) {
1577                dev->real_num_tx_queues = txq;
1578                qdisc_reset_all_tx_gt(dev, txq);
1579        }
1580}
1581EXPORT_SYMBOL(netif_set_real_num_tx_queues);
1582
1583static inline void __netif_reschedule(struct Qdisc *q)
1584{
1585        struct softnet_data *sd;
1586        unsigned long flags;
1587
1588        local_irq_save(flags);
1589        sd = &__get_cpu_var(softnet_data);
1590        q->next_sched = NULL;
1591        *sd->output_queue_tailp = q;
1592        sd->output_queue_tailp = &q->next_sched;
1593        raise_softirq_irqoff(NET_TX_SOFTIRQ);
1594        local_irq_restore(flags);
1595}
1596
1597void __netif_schedule(struct Qdisc *q)
1598{
1599        if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1600                __netif_reschedule(q);
1601}
1602EXPORT_SYMBOL(__netif_schedule);
1603
1604void dev_kfree_skb_irq(struct sk_buff *skb)
1605{
1606        if (atomic_dec_and_test(&skb->users)) {
1607                struct softnet_data *sd;
1608                unsigned long flags;
1609
1610                local_irq_save(flags);
1611                sd = &__get_cpu_var(softnet_data);
1612                skb->next = sd->completion_queue;
1613                sd->completion_queue = skb;
1614                raise_softirq_irqoff(NET_TX_SOFTIRQ);
1615                local_irq_restore(flags);
1616        }
1617}
1618EXPORT_SYMBOL(dev_kfree_skb_irq);
1619
1620void dev_kfree_skb_any(struct sk_buff *skb)
1621{
1622        if (in_irq() || irqs_disabled())
1623                dev_kfree_skb_irq(skb);
1624        else
1625                dev_kfree_skb(skb);
1626}
1627EXPORT_SYMBOL(dev_kfree_skb_any);
1628
1629
1630/**
1631 * netif_device_detach - mark device as removed
1632 * @dev: network device
1633 *
1634 * Mark device as removed from system and therefore no longer available.
1635 */
1636void netif_device_detach(struct net_device *dev)
1637{
1638        if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1639            netif_running(dev)) {
1640                netif_tx_stop_all_queues(dev);
1641        }
1642}
1643EXPORT_SYMBOL(netif_device_detach);
1644
1645/**
1646 * netif_device_attach - mark device as attached
1647 * @dev: network device
1648 *
1649 * Mark device as attached from system and restart if needed.
1650 */
1651void netif_device_attach(struct net_device *dev)
1652{
1653        if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1654            netif_running(dev)) {
1655                netif_tx_wake_all_queues(dev);
1656                __netdev_watchdog_up(dev);
1657        }
1658}
1659EXPORT_SYMBOL(netif_device_attach);
1660
1661static bool can_checksum_protocol(unsigned long features, __be16 protocol)
1662{
1663        return ((features & NETIF_F_NO_CSUM) ||
1664                ((features & NETIF_F_V4_CSUM) &&
1665                 protocol == htons(ETH_P_IP)) ||
1666                ((features & NETIF_F_V6_CSUM) &&
1667                 protocol == htons(ETH_P_IPV6)) ||
1668                ((features & NETIF_F_FCOE_CRC) &&
1669                 protocol == htons(ETH_P_FCOE)));
1670}
1671
1672static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
1673{
1674        if (can_checksum_protocol(dev->features, skb->protocol))
1675                return true;
1676
1677        if (skb->protocol == htons(ETH_P_8021Q)) {
1678                struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1679                if (can_checksum_protocol(dev->features & dev->vlan_features,
1680                                          veh->h_vlan_encapsulated_proto))
1681                        return true;
1682        }
1683
1684        return false;
1685}
1686
1687/**
1688 * skb_dev_set -- assign a new device to a buffer
1689 * @skb: buffer for the new device
1690 * @dev: network device
1691 *
1692 * If an skb is owned by a device already, we have to reset
1693 * all data private to the namespace a device belongs to
1694 * before assigning it a new device.
1695 */
1696#ifdef CONFIG_NET_NS
1697void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1698{
1699        skb_dst_drop(skb);
1700        if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1701                secpath_reset(skb);
1702                nf_reset(skb);
1703                skb_init_secmark(skb);
1704                skb->mark = 0;
1705                skb->priority = 0;
1706                skb->nf_trace = 0;
1707                skb->ipvs_property = 0;
1708#ifdef CONFIG_NET_SCHED
1709                skb->tc_index = 0;
1710#endif
1711        }
1712        skb->dev = dev;
1713}
1714EXPORT_SYMBOL(skb_set_dev);
1715#endif /* CONFIG_NET_NS */
1716
1717/*
1718 * Invalidate hardware checksum when packet is to be mangled, and
1719 * complete checksum manually on outgoing path.
1720 */
1721int skb_checksum_help(struct sk_buff *skb)
1722{
1723        __wsum csum;
1724        int ret = 0, offset;
1725
1726        if (skb->ip_summed == CHECKSUM_COMPLETE)
1727                goto out_set_summed;
1728
1729        if (unlikely(skb_shinfo(skb)->gso_size)) {
1730                /* Let GSO fix up the checksum. */
1731                goto out_set_summed;
1732        }
1733
1734        offset = skb->csum_start - skb_headroom(skb);
1735        BUG_ON(offset >= skb_headlen(skb));
1736        csum = skb_checksum(skb, offset, skb->len - offset, 0);
1737
1738        offset += skb->csum_offset;
1739        BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1740
1741        if (skb_cloned(skb) &&
1742            !skb_clone_writable(skb, offset + sizeof(__sum16))) {
1743                ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1744                if (ret)
1745                        goto out;
1746        }
1747
1748        *(__sum16 *)(skb->data + offset) = csum_fold(csum);
1749out_set_summed:
1750        skb->ip_summed = CHECKSUM_NONE;
1751out:
1752        return ret;
1753}
1754EXPORT_SYMBOL(skb_checksum_help);
1755
1756/**
1757 *      skb_gso_segment - Perform segmentation on skb.
1758 *      @skb: buffer to segment
1759 *      @features: features for the output path (see dev->features)
1760 *
1761 *      This function segments the given skb and returns a list of segments.
1762 *
1763 *      It may return NULL if the skb requires no segmentation.  This is
1764 *      only possible when GSO is used for verifying header integrity.
1765 */
1766struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1767{
1768        struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1769        struct packet_type *ptype;
1770        __be16 type = skb->protocol;
1771        int err;
1772
1773        skb_reset_mac_header(skb);
1774        skb->mac_len = skb->network_header - skb->mac_header;
1775        __skb_pull(skb, skb->mac_len);
1776
1777        if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1778                struct net_device *dev = skb->dev;
1779                struct ethtool_drvinfo info = {};
1780
1781                if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1782                        dev->ethtool_ops->get_drvinfo(dev, &info);
1783
1784                WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
1785                        "ip_summed=%d",
1786                     info.driver, dev ? dev->features : 0L,
1787                     skb->sk ? skb->sk->sk_route_caps : 0L,
1788                     skb->len, skb->data_len, skb->ip_summed);
1789
1790                if (skb_header_cloned(skb) &&
1791                    (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1792                        return ERR_PTR(err);
1793        }
1794
1795        rcu_read_lock();
1796        list_for_each_entry_rcu(ptype,
1797                        &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
1798                if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
1799                        if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1800                                err = ptype->gso_send_check(skb);
1801                                segs = ERR_PTR(err);
1802                                if (err || skb_gso_ok(skb, features))
1803                                        break;
1804                                __skb_push(skb, (skb->data -
1805                                                 skb_network_header(skb)));
1806                        }
1807                        segs = ptype->gso_segment(skb, features);
1808                        break;
1809                }
1810        }
1811        rcu_read_unlock();
1812
1813        __skb_push(skb, skb->data - skb_mac_header(skb));
1814
1815        return segs;
1816}
1817EXPORT_SYMBOL(skb_gso_segment);
1818
1819/* Take action when hardware reception checksum errors are detected. */
1820#ifdef CONFIG_BUG
1821void netdev_rx_csum_fault(struct net_device *dev)
1822{
1823        if (net_ratelimit()) {
1824                printk(KERN_ERR "%s: hw csum failure.\n",
1825                        dev ? dev->name : "<unknown>");
1826                dump_stack();
1827        }
1828}
1829EXPORT_SYMBOL(netdev_rx_csum_fault);
1830#endif
1831
1832/* Actually, we should eliminate this check as soon as we know, that:
1833 * 1. IOMMU is present and allows to map all the memory.
1834 * 2. No high memory really exists on this machine.
1835 */
1836
1837static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1838{
1839#ifdef CONFIG_HIGHMEM
1840        int i;
1841        if (!(dev->features & NETIF_F_HIGHDMA)) {
1842                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1843                        if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1844                                return 1;
1845        }
1846
1847        if (PCI_DMA_BUS_IS_PHYS) {
1848                struct device *pdev = dev->dev.parent;
1849
1850                if (!pdev)
1851                        return 0;
1852                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1853                        dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1854                        if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1855                                return 1;
1856                }
1857        }
1858#endif
1859        return 0;
1860}
1861
1862struct dev_gso_cb {
1863        void (*destructor)(struct sk_buff *skb);
1864};
1865
1866#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1867
1868static void dev_gso_skb_destructor(struct sk_buff *skb)
1869{
1870        struct dev_gso_cb *cb;
1871
1872        do {
1873                struct sk_buff *nskb = skb->next;
1874
1875                skb->next = nskb->next;
1876                nskb->next = NULL;
1877                kfree_skb(nskb);
1878        } while (skb->next);
1879
1880        cb = DEV_GSO_CB(skb);
1881        if (cb->destructor)
1882                cb->destructor(skb);
1883}
1884
1885/**
1886 *      dev_gso_segment - Perform emulated hardware segmentation on skb.
1887 *      @skb: buffer to segment
1888 *
1889 *      This function segments the given skb and stores the list of segments
1890 *      in skb->next.
1891 */
1892static int dev_gso_segment(struct sk_buff *skb)
1893{
1894        struct net_device *dev = skb->dev;
1895        struct sk_buff *segs;
1896        int features = dev->features & ~(illegal_highdma(dev, skb) ?
1897                                         NETIF_F_SG : 0);
1898
1899        segs = skb_gso_segment(skb, features);
1900
1901        /* Verifying header integrity only. */
1902        if (!segs)
1903                return 0;
1904
1905        if (IS_ERR(segs))
1906                return PTR_ERR(segs);
1907
1908        skb->next = segs;
1909        DEV_GSO_CB(skb)->destructor = skb->destructor;
1910        skb->destructor = dev_gso_skb_destructor;
1911
1912        return 0;
1913}
1914
1915/*
1916 * Try to orphan skb early, right before transmission by the device.
1917 * We cannot orphan skb if tx timestamp is requested, since
1918 * drivers need to call skb_tstamp_tx() to send the timestamp.
1919 */
1920static inline void skb_orphan_try(struct sk_buff *skb)
1921{
1922        struct sock *sk = skb->sk;
1923
1924        if (sk && !skb_tx(skb)->flags) {
1925                /* skb_tx_hash() wont be able to get sk.
1926                 * We copy sk_hash into skb->rxhash
1927                 */
1928                if (!skb->rxhash)
1929                        skb->rxhash = sk->sk_hash;
1930                skb_orphan(skb);
1931        }
1932}
1933
1934int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1935                        struct netdev_queue *txq)
1936{
1937        const struct net_device_ops *ops = dev->netdev_ops;
1938        int rc = NETDEV_TX_OK;
1939
1940        if (likely(!skb->next)) {
1941                if (!list_empty(&ptype_all))
1942                        dev_queue_xmit_nit(skb, dev);
1943
1944                /*
1945                 * If device doesnt need skb->dst, release it right now while
1946                 * its hot in this cpu cache
1947                 */
1948                if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
1949                        skb_dst_drop(skb);
1950
1951                skb_orphan_try(skb);
1952
1953                if (netif_needs_gso(dev, skb)) {
1954                        if (unlikely(dev_gso_segment(skb)))
1955                                goto out_kfree_skb;
1956                        if (skb->next)
1957                                goto gso;
1958                }
1959
1960                rc = ops->ndo_start_xmit(skb, dev);
1961                if (rc == NETDEV_TX_OK)
1962                        txq_trans_update(txq);
1963                return rc;
1964        }
1965
1966gso:
1967        do {
1968                struct sk_buff *nskb = skb->next;
1969
1970                skb->next = nskb->next;
1971                nskb->next = NULL;
1972
1973                /*
1974                 * If device doesnt need nskb->dst, release it right now while
1975                 * its hot in this cpu cache
1976                 */
1977                if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
1978                        skb_dst_drop(nskb);
1979
1980                rc = ops->ndo_start_xmit(nskb, dev);
1981                if (unlikely(rc != NETDEV_TX_OK)) {
1982                        if (rc & ~NETDEV_TX_MASK)
1983                                goto out_kfree_gso_skb;
1984                        nskb->next = skb->next;
1985                        skb->next = nskb;
1986                        return rc;
1987                }
1988                txq_trans_update(txq);
1989                if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
1990                        return NETDEV_TX_BUSY;
1991        } while (skb->next);
1992
1993out_kfree_gso_skb:
1994        if (likely(skb->next == NULL))
1995                skb->destructor = DEV_GSO_CB(skb)->destructor;
1996out_kfree_skb:
1997        kfree_skb(skb);
1998        return rc;
1999}
2000
2001static u32 hashrnd __read_mostly;
2002
2003u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
2004{
2005        u32 hash;
2006
2007        if (skb_rx_queue_recorded(skb)) {
2008                hash = skb_get_rx_queue(skb);
2009                while (unlikely(hash >= dev->real_num_tx_queues))
2010                        hash -= dev->real_num_tx_queues;
2011                return hash;
2012        }
2013
2014        if (skb->sk && skb->sk->sk_hash)
2015                hash = skb->sk->sk_hash;
2016        else
2017                hash = (__force u16) skb->protocol ^ skb->rxhash;
2018        hash = jhash_1word(hash, hashrnd);
2019
2020        return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
2021}
2022EXPORT_SYMBOL(skb_tx_hash);
2023
2024static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2025{
2026        if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2027                if (net_ratelimit()) {
2028                        pr_warning("%s selects TX queue %d, but "
2029                                "real number of TX queues is %d\n",
2030                                dev->name, queue_index, dev->real_num_tx_queues);
2031                }
2032                return 0;
2033        }
2034        return queue_index;
2035}
2036
2037static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2038                                        struct sk_buff *skb)
2039{
2040        int queue_index;
2041        struct sock *sk = skb->sk;
2042
2043        queue_index = sk_tx_queue_get(sk);
2044        if (queue_index < 0) {
2045                const struct net_device_ops *ops = dev->netdev_ops;
2046
2047                if (ops->ndo_select_queue) {
2048                        queue_index = ops->ndo_select_queue(dev, skb);
2049                        queue_index = dev_cap_txqueue(dev, queue_index);
2050                } else {
2051                        queue_index = 0;
2052                        if (dev->real_num_tx_queues > 1)
2053                                queue_index = skb_tx_hash(dev, skb);
2054
2055                        if (sk) {
2056                                struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1);
2057
2058                                if (dst && skb_dst(skb) == dst)
2059                                        sk_tx_queue_set(sk, queue_index);
2060                        }
2061                }
2062        }
2063
2064        skb_set_queue_mapping(skb, queue_index);
2065        return netdev_get_tx_queue(dev, queue_index);
2066}
2067
2068static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2069                                 struct net_device *dev,
2070                                 struct netdev_queue *txq)
2071{
2072        spinlock_t *root_lock = qdisc_lock(q);
2073        int rc;
2074
2075        spin_lock(root_lock);
2076        if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2077                kfree_skb(skb);
2078                rc = NET_XMIT_DROP;
2079        } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2080                   !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) {
2081                /*
2082                 * This is a work-conserving queue; there are no old skbs
2083                 * waiting to be sent out; and the qdisc is not running -
2084                 * xmit the skb directly.
2085                 */
2086                if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2087                        skb_dst_force(skb);
2088                __qdisc_update_bstats(q, skb->len);
2089                if (sch_direct_xmit(skb, q, dev, txq, root_lock))
2090                        __qdisc_run(q);
2091                else
2092                        clear_bit(__QDISC_STATE_RUNNING, &q->state);
2093
2094                rc = NET_XMIT_SUCCESS;
2095        } else {
2096                skb_dst_force(skb);
2097                rc = qdisc_enqueue_root(skb, q);
2098                qdisc_run(q);
2099        }
2100        spin_unlock(root_lock);
2101
2102        return rc;
2103}
2104
2105/*
2106 * Returns true if either:
2107 *      1. skb has frag_list and the device doesn't support FRAGLIST, or
2108 *      2. skb is fragmented and the device does not support SG, or if
2109 *         at least one of fragments is in highmem and device does not
2110 *         support DMA from it.
2111 */
2112static inline int skb_needs_linearize(struct sk_buff *skb,
2113                                      struct net_device *dev)
2114{
2115        return (skb_has_frags(skb) && !(dev->features & NETIF_F_FRAGLIST)) ||
2116               (skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) ||
2117                                              illegal_highdma(dev, skb)));
2118}
2119
2120static DEFINE_PER_CPU(int, xmit_recursion);
2121#define RECURSION_LIMIT 10
2122
2123/**
2124 *      dev_queue_xmit - transmit a buffer
2125 *      @skb: buffer to transmit
2126 *
2127 *      Queue a buffer for transmission to a network device. The caller must
2128 *      have set the device and priority and built the buffer before calling
2129 *      this function. The function can be called from an interrupt.
2130 *
2131 *      A negative errno code is returned on a failure. A success does not
2132 *      guarantee the frame will be transmitted as it may be dropped due
2133 *      to congestion or traffic shaping.
2134 *
2135 * -----------------------------------------------------------------------------------
2136 *      I notice this method can also return errors from the queue disciplines,
2137 *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
2138 *      be positive.
2139 *
2140 *      Regardless of the return value, the skb is consumed, so it is currently
2141 *      difficult to retry a send to this method.  (You can bump the ref count
2142 *      before sending to hold a reference for retry if you are careful.)
2143 *
2144 *      When calling this method, interrupts MUST be enabled.  This is because
2145 *      the BH enable code must have IRQs enabled so that it will not deadlock.
2146 *          --BLG
2147 */
2148int dev_queue_xmit(struct sk_buff *skb)
2149{
2150        struct net_device *dev = skb->dev;
2151        struct netdev_queue *txq;
2152        struct Qdisc *q;
2153        int rc = -ENOMEM;
2154
2155        /* GSO will handle the following emulations directly. */
2156        if (netif_needs_gso(dev, skb))
2157                goto gso;
2158
2159        /* Convert a paged skb to linear, if required */
2160        if (skb_needs_linearize(skb, dev) && __skb_linearize(skb))
2161                goto out_kfree_skb;
2162
2163        /* If packet is not checksummed and device does not support
2164         * checksumming for this protocol, complete checksumming here.
2165         */
2166        if (skb->ip_summed == CHECKSUM_PARTIAL) {
2167                skb_set_transport_header(skb, skb->csum_start -
2168                                              skb_headroom(skb));
2169                if (!dev_can_checksum(dev, skb) && skb_checksum_help(skb))
2170                        goto out_kfree_skb;
2171        }
2172
2173gso:
2174        /* Disable soft irqs for various locks below. Also
2175         * stops preemption for RCU.
2176         */
2177        rcu_read_lock_bh();
2178
2179        txq = dev_pick_tx(dev, skb);
2180        q = rcu_dereference_bh(txq->qdisc);
2181
2182#ifdef CONFIG_NET_CLS_ACT
2183        skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2184#endif
2185        if (q->enqueue) {
2186                rc = __dev_xmit_skb(skb, q, dev, txq);
2187                goto out;
2188        }
2189
2190        /* The device has no queue. Common case for software devices:
2191           loopback, all the sorts of tunnels...
2192
2193           Really, it is unlikely that netif_tx_lock protection is necessary
2194           here.  (f.e. loopback and IP tunnels are clean ignoring statistics
2195           counters.)
2196           However, it is possible, that they rely on protection
2197           made by us here.
2198
2199           Check this and shot the lock. It is not prone from deadlocks.
2200           Either shot noqueue qdisc, it is even simpler 8)
2201         */
2202        if (dev->flags & IFF_UP) {
2203                int cpu = smp_processor_id(); /* ok because BHs are off */
2204
2205                if (txq->xmit_lock_owner != cpu) {
2206
2207                        if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2208                                goto recursion_alert;
2209
2210                        HARD_TX_LOCK(dev, txq, cpu);
2211
2212                        if (!netif_tx_queue_stopped(txq)) {
2213                                __this_cpu_inc(xmit_recursion);
2214                                rc = dev_hard_start_xmit(skb, dev, txq);
2215                                __this_cpu_dec(xmit_recursion);
2216                                if (dev_xmit_complete(rc)) {
2217                                        HARD_TX_UNLOCK(dev, txq);
2218                                        goto out;
2219                                }
2220                        }
2221                        HARD_TX_UNLOCK(dev, txq);
2222                        if (net_ratelimit())
2223                                printk(KERN_CRIT "Virtual device %s asks to "
2224                                       "queue packet!\n", dev->name);
2225                } else {
2226                        /* Recursion is detected! It is possible,
2227                         * unfortunately
2228                         */
2229recursion_alert:
2230                        if (net_ratelimit())
2231                                printk(KERN_CRIT "Dead loop on virtual device "
2232                                       "%s, fix it urgently!\n", dev->name);
2233                }
2234        }
2235
2236        rc = -ENETDOWN;
2237        rcu_read_unlock_bh();
2238
2239out_kfree_skb:
2240        kfree_skb(skb);
2241        return rc;
2242out:
2243        rcu_read_unlock_bh();
2244        return rc;
2245}
2246EXPORT_SYMBOL(dev_queue_xmit);
2247
2248
2249/*=======================================================================
2250                        Receiver routines
2251  =======================================================================*/
2252
2253int netdev_max_backlog __read_mostly = 1000;
2254int netdev_tstamp_prequeue __read_mostly = 1;
2255int netdev_budget __read_mostly = 300;
2256int weight_p __read_mostly = 64;            /* old backlog weight */
2257
2258/* Called with irq disabled */
2259static inline void ____napi_schedule(struct softnet_data *sd,
2260                                     struct napi_struct *napi)
2261{
2262        list_add_tail(&napi->poll_list, &sd->poll_list);
2263        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2264}
2265
2266#ifdef CONFIG_RPS
2267
2268/* One global table that all flow-based protocols share. */
2269struct rps_sock_flow_table *rps_sock_flow_table __read_mostly;
2270EXPORT_SYMBOL(rps_sock_flow_table);
2271
2272/*
2273 * get_rps_cpu is called from netif_receive_skb and returns the target
2274 * CPU from the RPS map of the receiving queue for a given skb.
2275 * rcu_read_lock must be held on entry.
2276 */
2277static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2278                       struct rps_dev_flow **rflowp)
2279{
2280        struct ipv6hdr *ip6;
2281        struct iphdr *ip;
2282        struct netdev_rx_queue *rxqueue;
2283        struct rps_map *map;
2284        struct rps_dev_flow_table *flow_table;
2285        struct rps_sock_flow_table *sock_flow_table;
2286        int cpu = -1;
2287        u8 ip_proto;
2288        u16 tcpu;
2289        u32 addr1, addr2, ihl;
2290        union {
2291                u32 v32;
2292                u16 v16[2];
2293        } ports;
2294
2295        if (skb_rx_queue_recorded(skb)) {
2296                u16 index = skb_get_rx_queue(skb);
2297                if (unlikely(index >= dev->num_rx_queues)) {
2298                        WARN_ONCE(dev->num_rx_queues > 1, "%s received packet "
2299                                "on queue %u, but number of RX queues is %u\n",
2300                                dev->name, index, dev->num_rx_queues);
2301                        goto done;
2302                }
2303                rxqueue = dev->_rx + index;
2304        } else
2305                rxqueue = dev->_rx;
2306
2307        if (!rxqueue->rps_map && !rxqueue->rps_flow_table)
2308                goto done;
2309
2310        if (skb->rxhash)
2311                goto got_hash; /* Skip hash computation on packet header */
2312
2313        switch (skb->protocol) {
2314        case __constant_htons(ETH_P_IP):
2315                if (!pskb_may_pull(skb, sizeof(*ip)))
2316                        goto done;
2317
2318                ip = (struct iphdr *) skb->data;
2319                ip_proto = ip->protocol;
2320                addr1 = (__force u32) ip->saddr;
2321                addr2 = (__force u32) ip->daddr;
2322                ihl = ip->ihl;
2323                break;
2324        case __constant_htons(ETH_P_IPV6):
2325                if (!pskb_may_pull(skb, sizeof(*ip6)))
2326                        goto done;
2327
2328                ip6 = (struct ipv6hdr *) skb->data;
2329                ip_proto = ip6->nexthdr;
2330                addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2331                addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2332                ihl = (40 >> 2);
2333                break;
2334        default:
2335                goto done;
2336        }
2337        switch (ip_proto) {
2338        case IPPROTO_TCP:
2339        case IPPROTO_UDP:
2340        case IPPROTO_DCCP:
2341        case IPPROTO_ESP:
2342        case IPPROTO_AH:
2343        case IPPROTO_SCTP:
2344        case IPPROTO_UDPLITE:
2345                if (pskb_may_pull(skb, (ihl * 4) + 4)) {
2346                        ports.v32 = * (__force u32 *) (skb->data + (ihl * 4));
2347                        if (ports.v16[1] < ports.v16[0])
2348                                swap(ports.v16[0], ports.v16[1]);
2349                        break;
2350                }
2351        default:
2352                ports.v32 = 0;
2353                break;
2354        }
2355
2356        /* get a consistent hash (same value on both flow directions) */
2357        if (addr2 < addr1)
2358                swap(addr1, addr2);
2359        skb->rxhash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2360        if (!skb->rxhash)
2361                skb->rxhash = 1;
2362
2363got_hash:
2364        flow_table = rcu_dereference(rxqueue->rps_flow_table);
2365        sock_flow_table = rcu_dereference(rps_sock_flow_table);
2366        if (flow_table && sock_flow_table) {
2367                u16 next_cpu;
2368                struct rps_dev_flow *rflow;
2369
2370                rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2371                tcpu = rflow->cpu;
2372
2373                next_cpu = sock_flow_table->ents[skb->rxhash &
2374                    sock_flow_table->mask];
2375
2376                /*
2377                 * If the desired CPU (where last recvmsg was done) is
2378                 * different from current CPU (one in the rx-queue flow
2379                 * table entry), switch if one of the following holds:
2380                 *   - Current CPU is unset (equal to RPS_NO_CPU).
2381                 *   - Current CPU is offline.
2382                 *   - The current CPU's queue tail has advanced beyond the
2383                 *     last packet that was enqueued using this table entry.
2384                 *     This guarantees that all previous packets for the flow
2385                 *     have been dequeued, thus preserving in order delivery.
2386                 */
2387                if (unlikely(tcpu != next_cpu) &&
2388                    (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2389                     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2390                      rflow->last_qtail)) >= 0)) {
2391                        tcpu = rflow->cpu = next_cpu;
2392                        if (tcpu != RPS_NO_CPU)
2393                                rflow->last_qtail = per_cpu(softnet_data,
2394                                    tcpu).input_queue_head;
2395                }
2396                if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2397                        *rflowp = rflow;
2398                        cpu = tcpu;
2399                        goto done;
2400                }
2401        }
2402
2403        map = rcu_dereference(rxqueue->rps_map);
2404        if (map) {
2405                tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
2406
2407                if (cpu_online(tcpu)) {
2408                        cpu = tcpu;
2409                        goto done;
2410                }
2411        }
2412
2413done:
2414        return cpu;
2415}
2416
2417/* Called from hardirq (IPI) context */
2418static void rps_trigger_softirq(void *data)
2419{
2420        struct softnet_data *sd = data;
2421
2422        ____napi_schedule(sd, &sd->backlog);
2423        sd->received_rps++;
2424}
2425
2426#endif /* CONFIG_RPS */
2427
2428/*
2429 * Check if this softnet_data structure is another cpu one
2430 * If yes, queue it to our IPI list and return 1
2431 * If no, return 0
2432 */
2433static int rps_ipi_queued(struct softnet_data *sd)
2434{
2435#ifdef CONFIG_RPS
2436        struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2437
2438        if (sd != mysd) {
2439                sd->rps_ipi_next = mysd->rps_ipi_list;
2440                mysd->rps_ipi_list = sd;
2441
2442                __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2443                return 1;
2444        }
2445#endif /* CONFIG_RPS */
2446        return 0;
2447}
2448
2449/*
2450 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2451 * queue (may be a remote CPU queue).
2452 */
2453static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2454                              unsigned int *qtail)
2455{
2456        struct softnet_data *sd;
2457        unsigned long flags;
2458
2459        sd = &per_cpu(softnet_data, cpu);
2460
2461        local_irq_save(flags);
2462
2463        rps_lock(sd);
2464        if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2465                if (skb_queue_len(&sd->input_pkt_queue)) {
2466enqueue:
2467                        __skb_queue_tail(&sd->input_pkt_queue, skb);
2468                        input_queue_tail_incr_save(sd, qtail);
2469                        rps_unlock(sd);
2470                        local_irq_restore(flags);
2471                        return NET_RX_SUCCESS;
2472                }
2473
2474                /* Schedule NAPI for backlog device
2475                 * We can use non atomic operation since we own the queue lock
2476                 */
2477                if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
2478                        if (!rps_ipi_queued(sd))
2479                                ____napi_schedule(sd, &sd->backlog);
2480                }
2481                goto enqueue;
2482        }
2483
2484        sd->dropped++;
2485        rps_unlock(sd);
2486
2487        local_irq_restore(flags);
2488
2489        kfree_skb(skb);
2490        return NET_RX_DROP;
2491}
2492
2493/**
2494 *      netif_rx        -       post buffer to the network code
2495 *      @skb: buffer to post
2496 *
2497 *      This function receives a packet from a device driver and queues it for
2498 *      the upper (protocol) levels to process.  It always succeeds. The buffer
2499 *      may be dropped during processing for congestion control or by the
2500 *      protocol layers.
2501 *
2502 *      return values:
2503 *      NET_RX_SUCCESS  (no congestion)
2504 *      NET_RX_DROP     (packet was dropped)
2505 *
2506 */
2507
2508int netif_rx(struct sk_buff *skb)
2509{
2510        int ret;
2511
2512        /* if netpoll wants it, pretend we never saw it */
2513        if (netpoll_rx(skb))
2514                return NET_RX_DROP;
2515
2516        if (netdev_tstamp_prequeue)
2517                net_timestamp_check(skb);
2518
2519#ifdef CONFIG_RPS
2520        {
2521                struct rps_dev_flow voidflow, *rflow = &voidflow;
2522                int cpu;
2523
2524                preempt_disable();
2525                rcu_read_lock();
2526
2527                cpu = get_rps_cpu(skb->dev, skb, &rflow);
2528                if (cpu < 0)
2529                        cpu = smp_processor_id();
2530
2531                ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2532
2533                rcu_read_unlock();
2534                preempt_enable();
2535        }
2536#else
2537        {
2538                unsigned int qtail;
2539                ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2540                put_cpu();
2541        }
2542#endif
2543        return ret;
2544}
2545EXPORT_SYMBOL(netif_rx);
2546
2547int netif_rx_ni(struct sk_buff *skb)
2548{
2549        int err;
2550
2551        preempt_disable();
2552        err = netif_rx(skb);
2553        if (local_softirq_pending())
2554                do_softirq();
2555        preempt_enable();
2556
2557        return err;
2558}
2559EXPORT_SYMBOL(netif_rx_ni);
2560
2561static void net_tx_action(struct softirq_action *h)
2562{
2563        struct softnet_data *sd = &__get_cpu_var(softnet_data);
2564
2565        if (sd->completion_queue) {
2566                struct sk_buff *clist;
2567
2568                local_irq_disable();
2569                clist = sd->completion_queue;
2570                sd->completion_queue = NULL;
2571                local_irq_enable();
2572
2573                while (clist) {
2574                        struct sk_buff *skb = clist;
2575                        clist = clist->next;
2576
2577                        WARN_ON(atomic_read(&skb->users));
2578                        __kfree_skb(skb);
2579                }
2580        }
2581
2582        if (sd->output_queue) {
2583                struct Qdisc *head;
2584
2585                local_irq_disable();
2586                head = sd->output_queue;
2587                sd->output_queue = NULL;
2588                sd->output_queue_tailp = &sd->output_queue;
2589                local_irq_enable();
2590
2591                while (head) {
2592                        struct Qdisc *q = head;
2593                        spinlock_t *root_lock;
2594
2595                        head = head->next_sched;
2596
2597                        root_lock = qdisc_lock(q);
2598                        if (spin_trylock(root_lock)) {
2599                                smp_mb__before_clear_bit();
2600                                clear_bit(__QDISC_STATE_SCHED,
2601                                          &q->state);
2602                                qdisc_run(q);
2603                                spin_unlock(root_lock);
2604                        } else {
2605                                if (!test_bit(__QDISC_STATE_DEACTIVATED,
2606                                              &q->state)) {
2607                                        __netif_reschedule(q);
2608                                } else {
2609                                        smp_mb__before_clear_bit();
2610                                        clear_bit(__QDISC_STATE_SCHED,
2611                                                  &q->state);
2612                                }
2613                        }
2614                }
2615        }
2616}
2617
2618static inline int deliver_skb(struct sk_buff *skb,
2619                              struct packet_type *pt_prev,
2620                              struct net_device *orig_dev)
2621{
2622        atomic_inc(&skb->users);
2623        return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2624}
2625
2626#if defined(CONFIG_BRIDGE) || defined (CONFIG_BRIDGE_MODULE)
2627
2628#if defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)
2629/* This hook is defined here for ATM LANE */
2630int (*br_fdb_test_addr_hook)(struct net_device *dev,
2631                             unsigned char *addr) __read_mostly;
2632EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
2633#endif
2634
2635/*
2636 * If bridge module is loaded call bridging hook.
2637 *  returns NULL if packet was consumed.
2638 */
2639struct sk_buff *(*br_handle_frame_hook)(struct net_bridge_port *p,
2640                                        struct sk_buff *skb) __read_mostly;
2641EXPORT_SYMBOL_GPL(br_handle_frame_hook);
2642
2643static inline struct sk_buff *handle_bridge(struct sk_buff *skb,
2644                                            struct packet_type **pt_prev, int *ret,
2645                                            struct net_device *orig_dev)
2646{
2647        struct net_bridge_port *port;
2648
2649        if (skb->pkt_type == PACKET_LOOPBACK ||
2650            (port = rcu_dereference(skb->dev->br_port)) == NULL)
2651                return skb;
2652
2653        if (*pt_prev) {
2654                *ret = deliver_skb(skb, *pt_prev, orig_dev);
2655                *pt_prev = NULL;
2656        }
2657
2658        return br_handle_frame_hook(port, skb);
2659}
2660#else
2661#define handle_bridge(skb, pt_prev, ret, orig_dev)      (skb)
2662#endif
2663
2664#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE)
2665struct sk_buff *(*macvlan_handle_frame_hook)(struct macvlan_port *p,
2666                                             struct sk_buff *skb) __read_mostly;
2667EXPORT_SYMBOL_GPL(macvlan_handle_frame_hook);
2668
2669static inline struct sk_buff *handle_macvlan(struct sk_buff *skb,
2670                                             struct packet_type **pt_prev,
2671                                             int *ret,
2672                                             struct net_device *orig_dev)
2673{
2674        struct macvlan_port *port;
2675
2676        port = rcu_dereference(skb->dev->macvlan_port);
2677        if (!port)
2678                return skb;
2679
2680        if (*pt_prev) {
2681                *ret = deliver_skb(skb, *pt_prev, orig_dev);
2682                *pt_prev = NULL;
2683        }
2684        return macvlan_handle_frame_hook(port, skb);
2685}
2686#else
2687#define handle_macvlan(skb, pt_prev, ret, orig_dev)     (skb)
2688#endif
2689
2690#ifdef CONFIG_NET_CLS_ACT
2691/* TODO: Maybe we should just force sch_ingress to be compiled in
2692 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2693 * a compare and 2 stores extra right now if we dont have it on
2694 * but have CONFIG_NET_CLS_ACT
2695 * NOTE: This doesnt stop any functionality; if you dont have
2696 * the ingress scheduler, you just cant add policies on ingress.
2697 *
2698 */
2699static int ing_filter(struct sk_buff *skb)
2700{
2701        struct net_device *dev = skb->dev;
2702        u32 ttl = G_TC_RTTL(skb->tc_verd);
2703        struct netdev_queue *rxq;
2704        int result = TC_ACT_OK;
2705        struct Qdisc *q;
2706
2707        if (MAX_RED_LOOP < ttl++) {
2708                printk(KERN_WARNING
2709                       "Redir loop detected Dropping packet (%d->%d)\n",
2710                       skb->skb_iif, dev->ifindex);
2711                return TC_ACT_SHOT;
2712        }
2713
2714        skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2715        skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2716
2717        rxq = &dev->rx_queue;
2718
2719        q = rxq->qdisc;
2720        if (q != &noop_qdisc) {
2721                spin_lock(qdisc_lock(q));
2722                if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2723                        result = qdisc_enqueue_root(skb, q);
2724                spin_unlock(qdisc_lock(q));
2725        }
2726
2727        return result;
2728}
2729
2730static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2731                                         struct packet_type **pt_prev,
2732                                         int *ret, struct net_device *orig_dev)
2733{
2734        if (skb->dev->rx_queue.qdisc == &noop_qdisc)
2735                goto out;
2736
2737        if (*pt_prev) {
2738                *ret = deliver_skb(skb, *pt_prev, orig_dev);
2739                *pt_prev = NULL;
2740        } else {
2741                /* Huh? Why does turning on AF_PACKET affect this? */
2742                skb->tc_verd = SET_TC_OK2MUNGE(skb->tc_verd);
2743        }
2744
2745        switch (ing_filter(skb)) {
2746        case TC_ACT_SHOT:
2747        case TC_ACT_STOLEN:
2748                kfree_skb(skb);
2749                return NULL;
2750        }
2751
2752out:
2753        skb->tc_verd = 0;
2754        return skb;
2755}
2756#endif
2757
2758/*
2759 *      netif_nit_deliver - deliver received packets to network taps
2760 *      @skb: buffer
2761 *
2762 *      This function is used to deliver incoming packets to network
2763 *      taps. It should be used when the normal netif_receive_skb path
2764 *      is bypassed, for example because of VLAN acceleration.
2765 */
2766void netif_nit_deliver(struct sk_buff *skb)
2767{
2768        struct packet_type *ptype;
2769
2770        if (list_empty(&ptype_all))
2771                return;
2772
2773        skb_reset_network_header(skb);
2774        skb_reset_transport_header(skb);
2775        skb->mac_len = skb->network_header - skb->mac_header;
2776
2777        rcu_read_lock();
2778        list_for_each_entry_rcu(ptype, &ptype_all, list) {
2779                if (!ptype->dev || ptype->dev == skb->dev)
2780                        deliver_skb(skb, ptype, skb->dev);
2781        }
2782        rcu_read_unlock();
2783}
2784
2785static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,
2786                                              struct net_device *master)
2787{
2788        if (skb->pkt_type == PACKET_HOST) {
2789                u16 *dest = (u16 *) eth_hdr(skb)->h_dest;
2790
2791                memcpy(dest, master->dev_addr, ETH_ALEN);
2792        }
2793}
2794
2795/* On bonding slaves other than the currently active slave, suppress
2796 * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
2797 * ARP on active-backup slaves with arp_validate enabled.
2798 */
2799int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)
2800{
2801        struct net_device *dev = skb->dev;
2802
2803        if (master->priv_flags & IFF_MASTER_ARPMON)
2804                dev->last_rx = jiffies;
2805
2806        if ((master->priv_flags & IFF_MASTER_ALB) && master->br_port) {
2807                /* Do address unmangle. The local destination address
2808                 * will be always the one master has. Provides the right
2809                 * functionality in a bridge.
2810                 */
2811                skb_bond_set_mac_by_master(skb, master);
2812        }
2813
2814        if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
2815                if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&
2816                    skb->protocol == __cpu_to_be16(ETH_P_ARP))
2817                        return 0;
2818
2819                if (master->priv_flags & IFF_MASTER_ALB) {
2820                        if (skb->pkt_type != PACKET_BROADCAST &&
2821                            skb->pkt_type != PACKET_MULTICAST)
2822                                return 0;
2823                }
2824                if (master->priv_flags & IFF_MASTER_8023AD &&
2825                    skb->protocol == __cpu_to_be16(ETH_P_SLOW))
2826                        return 0;
2827
2828                return 1;
2829        }
2830        return 0;
2831}
2832EXPORT_SYMBOL(__skb_bond_should_drop);
2833
2834static int __netif_receive_skb(struct sk_buff *skb)
2835{
2836        struct packet_type *ptype, *pt_prev;
2837        struct net_device *orig_dev;
2838        struct net_device *master;
2839        struct net_device *null_or_orig;
2840        struct net_device *orig_or_bond;
2841        int ret = NET_RX_DROP;
2842        __be16 type;
2843
2844        if (!netdev_tstamp_prequeue)
2845                net_timestamp_check(skb);
2846
2847        if (vlan_tx_tag_present(skb) && vlan_hwaccel_do_receive(skb))
2848                return NET_RX_SUCCESS;
2849
2850        /* if we've gotten here through NAPI, check netpoll */
2851        if (netpoll_receive_skb(skb))
2852                return NET_RX_DROP;
2853
2854        if (!skb->skb_iif)
2855                skb->skb_iif = skb->dev->ifindex;
2856
2857        /*
2858         * bonding note: skbs received on inactive slaves should only
2859         * be delivered to pkt handlers that are exact matches.  Also
2860         * the deliver_no_wcard flag will be set.  If packet handlers
2861         * are sensitive to duplicate packets these skbs will need to
2862         * be dropped at the handler.  The vlan accel path may have
2863         * already set the deliver_no_wcard flag.
2864         */
2865        null_or_orig = NULL;
2866        orig_dev = skb->dev;
2867        master = ACCESS_ONCE(orig_dev->master);
2868        if (skb->deliver_no_wcard)
2869                null_or_orig = orig_dev;
2870        else if (master) {
2871                if (skb_bond_should_drop(skb, master)) {
2872                        skb->deliver_no_wcard = 1;
2873                        null_or_orig = orig_dev; /* deliver only exact match */
2874                } else
2875                        skb->dev = master;
2876        }
2877
2878        __get_cpu_var(softnet_data).processed++;
2879
2880        skb_reset_network_header(skb);
2881        skb_reset_transport_header(skb);
2882        skb->mac_len = skb->network_header - skb->mac_header;
2883
2884        pt_prev = NULL;
2885
2886        rcu_read_lock();
2887
2888#ifdef CONFIG_NET_CLS_ACT
2889        if (skb->tc_verd & TC_NCLS) {
2890                skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
2891                goto ncls;
2892        }
2893#endif
2894
2895        list_for_each_entry_rcu(ptype, &ptype_all, list) {
2896                if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2897                    ptype->dev == orig_dev) {
2898                        if (pt_prev)
2899                                ret = deliver_skb(skb, pt_prev, orig_dev);
2900                        pt_prev = ptype;
2901                }
2902        }
2903
2904#ifdef CONFIG_NET_CLS_ACT
2905        skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
2906        if (!skb)
2907                goto out;
2908ncls:
2909#endif
2910
2911        skb = handle_bridge(skb, &pt_prev, &ret, orig_dev);
2912        if (!skb)
2913                goto out;
2914        skb = handle_macvlan(skb, &pt_prev, &ret, orig_dev);
2915        if (!skb)
2916                goto out;
2917
2918        /*
2919         * Make sure frames received on VLAN interfaces stacked on
2920         * bonding interfaces still make their way to any base bonding
2921         * device that may have registered for a specific ptype.  The
2922         * handler may have to adjust skb->dev and orig_dev.
2923         */
2924        orig_or_bond = orig_dev;
2925        if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
2926            (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
2927                orig_or_bond = vlan_dev_real_dev(skb->dev);
2928        }
2929
2930        type = skb->protocol;
2931        list_for_each_entry_rcu(ptype,
2932                        &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
2933                if (ptype->type == type && (ptype->dev == null_or_orig ||
2934                     ptype->dev == skb->dev || ptype->dev == orig_dev ||
2935                     ptype->dev == orig_or_bond)) {
2936                        if (pt_prev)
2937                                ret = deliver_skb(skb, pt_prev, orig_dev);
2938                        pt_prev = ptype;
2939                }
2940        }
2941
2942        if (pt_prev) {
2943                ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2944        } else {
2945                kfree_skb(skb);
2946                /* Jamal, now you will not able to escape explaining
2947                 * me how you were going to use this. :-)
2948                 */
2949                ret = NET_RX_DROP;
2950        }
2951
2952out:
2953        rcu_read_unlock();
2954        return ret;
2955}
2956
2957/**
2958 *      netif_receive_skb - process receive buffer from network
2959 *      @skb: buffer to process
2960 *
2961 *      netif_receive_skb() is the main receive data processing function.
2962 *      It always succeeds. The buffer may be dropped during processing
2963 *      for congestion control or by the protocol layers.
2964 *
2965 *      This function may only be called from softirq context and interrupts
2966 *      should be enabled.
2967 *
2968 *      Return values (usually ignored):
2969 *      NET_RX_SUCCESS: no congestion
2970 *      NET_RX_DROP: packet was dropped
2971 */
2972int netif_receive_skb(struct sk_buff *skb)
2973{
2974        if (netdev_tstamp_prequeue)
2975                net_timestamp_check(skb);
2976
2977#ifdef CONFIG_RPS
2978        {
2979                struct rps_dev_flow voidflow, *rflow = &voidflow;
2980                int cpu, ret;
2981
2982                rcu_read_lock();
2983
2984                cpu = get_rps_cpu(skb->dev, skb, &rflow);
2985
2986                if (cpu >= 0) {
2987                        ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2988                        rcu_read_unlock();
2989                } else {
2990                        rcu_read_unlock();
2991                        ret = __netif_receive_skb(skb);
2992                }
2993
2994                return ret;
2995        }
2996#else
2997        return __netif_receive_skb(skb);
2998#endif
2999}
3000EXPORT_SYMBOL(netif_receive_skb);
3001
3002/* Network device is going away, flush any packets still pending
3003 * Called with irqs disabled.
3004 */
3005static void flush_backlog(void *arg)
3006{
3007        struct net_device *dev = arg;
3008        struct softnet_data *sd = &__get_cpu_var(softnet_data);
3009        struct sk_buff *skb, *tmp;
3010
3011        rps_lock(sd);
3012        skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3013                if (skb->dev == dev) {
3014                        __skb_unlink(skb, &sd->input_pkt_queue);
3015                        kfree_skb(skb);
3016                        input_queue_head_incr(sd);
3017                }
3018        }
3019        rps_unlock(sd);
3020
3021        skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3022                if (skb->dev == dev) {
3023                        __skb_unlink(skb, &sd->process_queue);
3024                        kfree_skb(skb);
3025                        input_queue_head_incr(sd);
3026                }
3027        }
3028}
3029
3030static int napi_gro_complete(struct sk_buff *skb)
3031{
3032        struct packet_type *ptype;
3033        __be16 type = skb->protocol;
3034        struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3035        int err = -ENOENT;
3036
3037        if (NAPI_GRO_CB(skb)->count == 1) {
3038                skb_shinfo(skb)->gso_size = 0;
3039                goto out;
3040        }
3041
3042        rcu_read_lock();
3043        list_for_each_entry_rcu(ptype, head, list) {
3044                if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3045                        continue;
3046
3047                err = ptype->gro_complete(skb);
3048                break;
3049        }
3050        rcu_read_unlock();
3051
3052        if (err) {
3053                WARN_ON(&ptype->list == head);
3054                kfree_skb(skb);
3055                return NET_RX_SUCCESS;
3056        }
3057
3058out:
3059        return netif_receive_skb(skb);
3060}
3061
3062static void napi_gro_flush(struct napi_struct *napi)
3063{
3064        struct sk_buff *skb, *next;
3065
3066        for (skb = napi->gro_list; skb; skb = next) {
3067                next = skb->next;
3068                skb->next = NULL;
3069                napi_gro_complete(skb);
3070        }
3071
3072        napi->gro_count = 0;
3073        napi->gro_list = NULL;
3074}
3075
3076enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3077{
3078        struct sk_buff **pp = NULL;
3079        struct packet_type *ptype;
3080        __be16 type = skb->protocol;
3081        struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3082        int same_flow;
3083        int mac_len;
3084        enum gro_result ret;
3085
3086        if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3087                goto normal;
3088
3089        if (skb_is_gso(skb) || skb_has_frags(skb))
3090                goto normal;
3091
3092        rcu_read_lock();
3093        list_for_each_entry_rcu(ptype, head, list) {
3094                if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3095                        continue;
3096
3097                skb_set_network_header(skb, skb_gro_offset(skb));
3098                mac_len = skb->network_header - skb->mac_header;
3099                skb->mac_len = mac_len;
3100                NAPI_GRO_CB(skb)->same_flow = 0;
3101                NAPI_GRO_CB(skb)->flush = 0;
3102                NAPI_GRO_CB(skb)->free = 0;
3103
3104                pp = ptype->gro_receive(&napi->gro_list, skb);
3105                break;
3106        }
3107        rcu_read_unlock();
3108
3109        if (&ptype->list == head)
3110                goto normal;
3111
3112        same_flow = NAPI_GRO_CB(skb)->same_flow;
3113        ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3114
3115        if (pp) {
3116                struct sk_buff *nskb = *pp;
3117
3118                *pp = nskb->next;
3119                nskb->next = NULL;
3120                napi_gro_complete(nskb);
3121                napi->gro_count--;
3122        }
3123
3124        if (same_flow)
3125                goto ok;
3126
3127        if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3128                goto normal;
3129
3130        napi->gro_count++;
3131        NAPI_GRO_CB(skb)->count = 1;
3132        skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3133        skb->next = napi->gro_list;
3134        napi->gro_list = skb;
3135        ret = GRO_HELD;
3136
3137pull:
3138        if (skb_headlen(skb) < skb_gro_offset(skb)) {
3139                int grow = skb_gro_offset(skb) - skb_headlen(skb);
3140
3141                BUG_ON(skb->end - skb->tail < grow);
3142
3143                memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3144
3145                skb->tail += grow;
3146                skb->data_len -= grow;
3147
3148                skb_shinfo(skb)->frags[0].page_offset += grow;
3149                skb_shinfo(skb)->frags[0].size -= grow;
3150
3151                if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3152                        put_page(skb_shinfo(skb)->frags[0].page);
3153                        memmove(skb_shinfo(skb)->frags,
3154                                skb_shinfo(skb)->frags + 1,
3155                                --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3156                }
3157        }
3158
3159ok:
3160        return ret;
3161
3162normal:
3163        ret = GRO_NORMAL;
3164        goto pull;
3165}
3166EXPORT_SYMBOL(dev_gro_receive);
3167
3168static gro_result_t
3169__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3170{
3171        struct sk_buff *p;
3172
3173        for (p = napi->gro_list; p; p = p->next) {
3174                NAPI_GRO_CB(p)->same_flow =
3175                        (p->dev == skb->dev) &&
3176                        !compare_ether_header(skb_mac_header(p),
3177                                              skb_gro_mac_header(skb));
3178                NAPI_GRO_CB(p)->flush = 0;
3179        }
3180
3181        return dev_gro_receive(napi, skb);
3182}
3183
3184gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3185{
3186        switch (ret) {
3187        case GRO_NORMAL:
3188                if (netif_receive_skb(skb))
3189                        ret = GRO_DROP;
3190                break;
3191
3192        case GRO_DROP:
3193        case GRO_MERGED_FREE:
3194                kfree_skb(skb);
3195                break;
3196
3197        case GRO_HELD:
3198        case GRO_MERGED:
3199                break;
3200        }
3201
3202        return ret;
3203}
3204EXPORT_SYMBOL(napi_skb_finish);
3205
3206void skb_gro_reset_offset(struct sk_buff *skb)
3207{
3208        NAPI_GRO_CB(skb)->data_offset = 0;
3209        NAPI_GRO_CB(skb)->frag0 = NULL;
3210        NAPI_GRO_CB(skb)->frag0_len = 0;
3211
3212        if (skb->mac_header == skb->tail &&
3213            !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
3214                NAPI_GRO_CB(skb)->frag0 =
3215                        page_address(skb_shinfo(skb)->frags[0].page) +
3216                        skb_shinfo(skb)->frags[0].page_offset;
3217                NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3218        }
3219}
3220EXPORT_SYMBOL(skb_gro_reset_offset);
3221
3222gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3223{
3224        skb_gro_reset_offset(skb);
3225
3226        return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
3227}
3228EXPORT_SYMBOL(napi_gro_receive);
3229
3230void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3231{
3232        __skb_pull(skb, skb_headlen(skb));
3233        skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
3234        skb->dev = napi->dev;
3235        skb->skb_iif = 0;
3236
3237        napi->skb = skb;
3238}
3239EXPORT_SYMBOL(napi_reuse_skb);
3240
3241struct sk_buff *napi_get_frags(struct napi_struct *napi)
3242{
3243        struct sk_buff *skb = napi->skb;
3244
3245        if (!skb) {
3246                skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3247                if (skb)
3248                        napi->skb = skb;
3249        }
3250        return skb;
3251}
3252EXPORT_SYMBOL(napi_get_frags);
3253
3254gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3255                               gro_result_t ret)
3256{
3257        switch (ret) {
3258        case GRO_NORMAL:
3259        case GRO_HELD:
3260                skb->protocol = eth_type_trans(skb, skb->dev);
3261
3262                if (ret == GRO_HELD)
3263                        skb_gro_pull(skb, -ETH_HLEN);
3264                else if (netif_receive_skb(skb))
3265                        ret = GRO_DROP;
3266                break;
3267
3268        case GRO_DROP:
3269        case GRO_MERGED_FREE:
3270                napi_reuse_skb(napi, skb);
3271                break;
3272
3273        case GRO_MERGED:
3274                break;
3275        }
3276
3277        return ret;
3278}
3279EXPORT_SYMBOL(napi_frags_finish);
3280
3281struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3282{
3283        struct sk_buff *skb = napi->skb;
3284        struct ethhdr *eth;
3285        unsigned int hlen;
3286        unsigned int off;
3287
3288        napi->skb = NULL;
3289
3290        skb_reset_mac_header(skb);
3291        skb_gro_reset_offset(skb);
3292
3293        off = skb_gro_offset(skb);
3294        hlen = off + sizeof(*eth);
3295        eth = skb_gro_header_fast(skb, off);
3296        if (skb_gro_header_hard(skb, hlen)) {
3297                eth = skb_gro_header_slow(skb, hlen, off);
3298                if (unlikely(!eth)) {
3299                        napi_reuse_skb(napi, skb);
3300                        skb = NULL;
3301                        goto out;
3302                }
3303        }
3304
3305        skb_gro_pull(skb, sizeof(*eth));
3306
3307        /*
3308         * This works because the only protocols we care about don't require
3309         * special handling.  We'll fix it up properly at the end.
3310         */
3311        skb->protocol = eth->h_proto;
3312
3313out:
3314        return skb;
3315}
3316EXPORT_SYMBOL(napi_frags_skb);
3317
3318gro_result_t napi_gro_frags(struct napi_struct *napi)
3319{
3320        struct sk_buff *skb = napi_frags_skb(napi);
3321
3322        if (!skb)
3323                return GRO_DROP;
3324
3325        return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
3326}
3327EXPORT_SYMBOL(napi_gro_frags);
3328
3329/*
3330 * net_rps_action sends any pending IPI's for rps.
3331 * Note: called with local irq disabled, but exits with local irq enabled.
3332 */
3333static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3334{
3335#ifdef CONFIG_RPS
3336        struct softnet_data *remsd = sd->rps_ipi_list;
3337
3338        if (remsd) {
3339                sd->rps_ipi_list = NULL;
3340
3341                local_irq_enable();
3342
3343                /* Send pending IPI's to kick RPS processing on remote cpus. */
3344                while (remsd) {
3345                        struct softnet_data *next = remsd->rps_ipi_next;
3346
3347                        if (cpu_online(remsd->cpu))
3348                                __smp_call_function_single(remsd->cpu,
3349                                                           &remsd->csd, 0);
3350                        remsd = next;
3351                }
3352        } else
3353#endif
3354                local_irq_enable();
3355}
3356
3357static int process_backlog(struct napi_struct *napi, int quota)
3358{
3359        int work = 0;
3360        struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
3361
3362#ifdef CONFIG_RPS
3363        /* Check if we have pending ipi, its better to send them now,
3364         * not waiting net_rx_action() end.
3365         */
3366        if (sd->rps_ipi_list) {
3367                local_irq_disable();
3368                net_rps_action_and_irq_enable(sd);
3369        }
3370#endif
3371        napi->weight = weight_p;
3372        local_irq_disable();
3373        while (work < quota) {
3374                struct sk_buff *skb;
3375                unsigned int qlen;
3376
3377                while ((skb = __skb_dequeue(&sd->process_queue))) {
3378                        local_irq_enable();
3379                        __netif_receive_skb(skb);
3380                        local_irq_disable();
3381                        input_queue_head_incr(sd);
3382                        if (++work >= quota) {
3383                                local_irq_enable();
3384                                return work;
3385                        }
3386                }
3387
3388                rps_lock(sd);
3389                qlen = skb_queue_len(&sd->input_pkt_queue);
3390                if (qlen)
3391                        skb_queue_splice_tail_init(&sd->input_pkt_queue,
3392                                                   &sd->process_queue);
3393
3394                if (qlen < quota - work) {
3395                        /*
3396                         * Inline a custom version of __napi_complete().
3397                         * only current cpu owns and manipulates this napi,
3398                         * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3399                         * we can use a plain write instead of clear_bit(),
3400                         * and we dont need an smp_mb() memory barrier.
3401                         */
3402                        list_del(&napi->poll_list);
3403                        napi->state = 0;
3404
3405                        quota = work + qlen;
3406                }
3407                rps_unlock(sd);
3408        }
3409        local_irq_enable();
3410
3411        return work;
3412}
3413
3414/**
3415 * __napi_schedule - schedule for receive
3416 * @n: entry to schedule
3417 *
3418 * The entry's receive function will be scheduled to run
3419 */
3420void __napi_schedule(struct napi_struct *n)
3421{
3422        unsigned long flags;
3423
3424        local_irq_save(flags);
3425        ____napi_schedule(&__get_cpu_var(softnet_data), n);
3426        local_irq_restore(flags);
3427}
3428EXPORT_SYMBOL(__napi_schedule);
3429
3430void __napi_complete(struct napi_struct *n)
3431{
3432        BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3433        BUG_ON(n->gro_list);
3434
3435        list_del(&n->poll_list);
3436        smp_mb__before_clear_bit();
3437        clear_bit(NAPI_STATE_SCHED, &n->state);
3438}
3439EXPORT_SYMBOL(__napi_complete);
3440
3441void napi_complete(struct napi_struct *n)
3442{
3443        unsigned long flags;
3444
3445        /*
3446         * don't let napi dequeue from the cpu poll list
3447         * just in case its running on a different cpu
3448         */
3449        if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3450                return;
3451
3452        napi_gro_flush(n);
3453        local_irq_save(flags);
3454        __napi_complete(n);
3455        local_irq_restore(flags);
3456}
3457EXPORT_SYMBOL(napi_complete);
3458
3459void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3460                    int (*poll)(struct napi_struct *, int), int weight)
3461{
3462        INIT_LIST_HEAD(&napi->poll_list);
3463        napi->gro_count = 0;
3464        napi->gro_list = NULL;
3465        napi->skb = NULL;
3466        napi->poll = poll;
3467        napi->weight = weight;
3468        list_add(&napi->dev_list, &dev->napi_list);
3469        napi->dev = dev;
3470#ifdef CONFIG_NETPOLL
3471        spin_lock_init(&napi->poll_lock);
3472        napi->poll_owner = -1;
3473#endif
3474        set_bit(NAPI_STATE_SCHED, &napi->state);
3475}
3476EXPORT_SYMBOL(netif_napi_add);
3477
3478void netif_napi_del(struct napi_struct *napi)
3479{
3480        struct sk_buff *skb, *next;
3481
3482        list_del_init(&napi->dev_list);
3483        napi_free_frags(napi);
3484
3485        for (skb = napi->gro_list; skb; skb = next) {
3486                next = skb->next;
3487                skb->next = NULL;
3488                kfree_skb(skb);
3489        }
3490
3491        napi->gro_list = NULL;
3492        napi->gro_count = 0;
3493}
3494EXPORT_SYMBOL(netif_napi_del);
3495
3496static void net_rx_action(struct softirq_action *h)
3497{
3498        struct softnet_data *sd = &__get_cpu_var(softnet_data);
3499        unsigned long time_limit = jiffies + 2;
3500        int budget = netdev_budget;
3501        void *have;
3502
3503        local_irq_disable();
3504
3505        while (!list_empty(&sd->poll_list)) {
3506                struct napi_struct *n;
3507                int work, weight;
3508
3509                /* If softirq window is exhuasted then punt.
3510                 * Allow this to run for 2 jiffies since which will allow
3511                 * an average latency of 1.5/HZ.
3512                 */
3513                if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
3514                        goto softnet_break;
3515
3516                local_irq_enable();
3517
3518                /* Even though interrupts have been re-enabled, this
3519                 * access is safe because interrupts can only add new
3520                 * entries to the tail of this list, and only ->poll()
3521                 * calls can remove this head entry from the list.
3522                 */
3523                n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
3524
3525                have = netpoll_poll_lock(n);
3526
3527                weight = n->weight;
3528
3529                /* This NAPI_STATE_SCHED test is for avoiding a race
3530                 * with netpoll's poll_napi().  Only the entity which
3531                 * obtains the lock and sees NAPI_STATE_SCHED set will
3532                 * actually make the ->poll() call.  Therefore we avoid
3533                 * accidently calling ->poll() when NAPI is not scheduled.
3534                 */
3535                work = 0;
3536                if (test_bit(NAPI_STATE_SCHED, &n->state)) {
3537                        work = n->poll(n, weight);
3538                        trace_napi_poll(n);
3539                }
3540
3541                WARN_ON_ONCE(work > weight);
3542
3543                budget -= work;
3544
3545                local_irq_disable();
3546
3547                /* Drivers must not modify the NAPI state if they
3548                 * consume the entire weight.  In such cases this code
3549                 * still "owns" the NAPI instance and therefore can
3550                 * move the instance around on the list at-will.
3551                 */
3552                if (unlikely(work == weight)) {
3553                        if (unlikely(napi_disable_pending(n))) {
3554                                local_irq_enable();
3555                                napi_complete(n);
3556                                local_irq_disable();
3557                        } else
3558                                list_move_tail(&n->poll_list, &sd->poll_list);
3559                }
3560
3561                netpoll_poll_unlock(have);
3562        }
3563out:
3564        net_rps_action_and_irq_enable(sd);
3565
3566#ifdef CONFIG_NET_DMA
3567        /*
3568         * There may not be any more sk_buffs coming right now, so push
3569         * any pending DMA copies to hardware
3570         */
3571        dma_issue_pending_all();
3572#endif
3573
3574        return;
3575
3576softnet_break:
3577        sd->time_squeeze++;
3578        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3579        goto out;
3580}
3581
3582static gifconf_func_t *gifconf_list[NPROTO];
3583
3584/**
3585 *      register_gifconf        -       register a SIOCGIF handler
3586 *      @family: Address family
3587 *      @gifconf: Function handler
3588 *
3589 *      Register protocol dependent address dumping routines. The handler
3590 *      that is passed must not be freed or reused until it has been replaced
3591 *      by another handler.
3592 */
3593int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
3594{
3595        if (family >= NPROTO)
3596                return -EINVAL;
3597        gifconf_list[family] = gifconf;
3598        return 0;
3599}
3600EXPORT_SYMBOL(register_gifconf);
3601
3602
3603/*
3604 *      Map an interface index to its name (SIOCGIFNAME)
3605 */
3606
3607/*
3608 *      We need this ioctl for efficient implementation of the
3609 *      if_indextoname() function required by the IPv6 API.  Without
3610 *      it, we would have to search all the interfaces to find a
3611 *      match.  --pb
3612 */
3613
3614static int dev_ifname(struct net *net, struct ifreq __user *arg)
3615{
3616        struct net_device *dev;
3617        struct ifreq ifr;
3618
3619        /*
3620         *      Fetch the caller's info block.
3621         */
3622
3623        if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3624                return -EFAULT;
3625
3626        rcu_read_lock();
3627        dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
3628        if (!dev) {
3629                rcu_read_unlock();
3630                return -ENODEV;
3631        }
3632
3633        strcpy(ifr.ifr_name, dev->name);
3634        rcu_read_unlock();
3635
3636        if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3637                return -EFAULT;
3638        return 0;
3639}
3640
3641/*
3642 *      Perform a SIOCGIFCONF call. This structure will change
3643 *      size eventually, and there is nothing I can do about it.
3644 *      Thus we will need a 'compatibility mode'.
3645 */
3646
3647static int dev_ifconf(struct net *net, char __user *arg)
3648{
3649        struct ifconf ifc;
3650        struct net_device *dev;
3651        char __user *pos;
3652        int len;
3653        int total;
3654        int i;
3655
3656        /*
3657         *      Fetch the caller's info block.
3658         */
3659
3660        if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3661                return -EFAULT;
3662
3663        pos = ifc.ifc_buf;
3664        len = ifc.ifc_len;
3665
3666        /*
3667         *      Loop over the interfaces, and write an info block for each.
3668         */
3669
3670        total = 0;
3671        for_each_netdev(net, dev) {
3672                for (i = 0; i < NPROTO; i++) {
3673                        if (gifconf_list[i]) {
3674                                int done;
3675                                if (!pos)
3676                                        done = gifconf_list[i](dev, NULL, 0);
3677                                else
3678                                        done = gifconf_list[i](dev, pos + total,
3679                                                               len - total);
3680                                if (done < 0)
3681                                        return -EFAULT;
3682                                total += done;
3683                        }
3684                }
3685        }
3686
3687        /*
3688         *      All done.  Write the updated control block back to the caller.
3689         */
3690        ifc.ifc_len = total;
3691
3692        /*
3693         *      Both BSD and Solaris return 0 here, so we do too.
3694         */
3695        return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3696}
3697
3698#ifdef CONFIG_PROC_FS
3699/*
3700 *      This is invoked by the /proc filesystem handler to display a device
3701 *      in detail.
3702 */
3703void *dev_seq_start(struct seq_file *seq, loff_t *pos)
3704        __acquires(RCU)
3705{
3706        struct net *net = seq_file_net(seq);
3707        loff_t off;
3708        struct net_device *dev;
3709
3710        rcu_read_lock();
3711        if (!*pos)
3712                return SEQ_START_TOKEN;
3713
3714        off = 1;
3715        for_each_netdev_rcu(net, dev)
3716                if (off++ == *pos)
3717                        return dev;
3718
3719        return NULL;
3720}
3721
3722void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3723{
3724        struct net_device *dev = (v == SEQ_START_TOKEN) ?
3725                                  first_net_device(seq_file_net(seq)) :
3726                                  next_net_device((struct net_device *)v);
3727
3728        ++*pos;
3729        return rcu_dereference(dev);
3730}
3731
3732void dev_seq_stop(struct seq_file *seq, void *v)
3733        __releases(RCU)
3734{
3735        rcu_read_unlock();
3736}
3737
3738static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
3739{
3740        const struct net_device_stats *stats = dev_get_stats(dev);
3741
3742        seq_printf(seq, "%6s: %7lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu "
3743                   "%8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n",
3744                   dev->name, stats->rx_bytes, stats->rx_packets,
3745                   stats->rx_errors,
3746                   stats->rx_dropped + stats->rx_missed_errors,
3747                   stats->rx_fifo_errors,
3748                   stats->rx_length_errors + stats->rx_over_errors +
3749                    stats->rx_crc_errors + stats->rx_frame_errors,
3750                   stats->rx_compressed, stats->multicast,
3751                   stats->tx_bytes, stats->tx_packets,
3752                   stats->tx_errors, stats->tx_dropped,
3753                   stats->tx_fifo_errors, stats->collisions,
3754                   stats->tx_carrier_errors +
3755                    stats->tx_aborted_errors +
3756                    stats->tx_window_errors +
3757                    stats->tx_heartbeat_errors,
3758                   stats->tx_compressed);
3759}
3760
3761/*
3762 *      Called from the PROCfs module. This now uses the new arbitrary sized
3763 *      /proc/net interface to create /proc/net/dev
3764 */
3765static int dev_seq_show(struct seq_file *seq, void *v)
3766{
3767        if (v == SEQ_START_TOKEN)
3768                seq_puts(seq, "Inter-|   Receive                            "
3769                              "                    |  Transmit\n"
3770                              " face |bytes    packets errs drop fifo frame "
3771                              "compressed multicast|bytes    packets errs "
3772                              "drop fifo colls carrier compressed\n");
3773        else
3774                dev_seq_printf_stats(seq, v);
3775        return 0;
3776}
3777
3778static struct softnet_data *softnet_get_online(loff_t *pos)
3779{
3780        struct softnet_data *sd = NULL;
3781
3782        while (*pos < nr_cpu_ids)
3783                if (cpu_online(*pos)) {
3784                        sd = &per_cpu(softnet_data, *pos);
3785                        break;
3786                } else
3787                        ++*pos;
3788        return sd;
3789}
3790
3791static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
3792{
3793        return softnet_get_online(pos);
3794}
3795
3796static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3797{
3798        ++*pos;
3799        return softnet_get_online(pos);
3800}
3801
3802static void softnet_seq_stop(struct seq_file *seq, void *v)
3803{
3804}
3805
3806static int softnet_seq_show(struct seq_file *seq, void *v)
3807{
3808        struct softnet_data *sd = v;
3809
3810        seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
3811                   sd->processed, sd->dropped, sd->time_squeeze, 0,
3812                   0, 0, 0, 0, /* was fastroute */
3813                   sd->cpu_collision, sd->received_rps);
3814        return 0;
3815}
3816
3817static const struct seq_operations dev_seq_ops = {
3818        .start = dev_seq_start,
3819        .next  = dev_seq_next,
3820        .stop  = dev_seq_stop,
3821        .show  = dev_seq_show,
3822};
3823
3824static int dev_seq_open(struct inode *inode, struct file *file)
3825{
3826        return seq_open_net(inode, file, &dev_seq_ops,
3827                            sizeof(struct seq_net_private));
3828}
3829
3830static const struct file_operations dev_seq_fops = {
3831        .owner   = THIS_MODULE,
3832        .open    = dev_seq_open,
3833        .read    = seq_read,
3834        .llseek  = seq_lseek,
3835        .release = seq_release_net,
3836};
3837
3838static const struct seq_operations softnet_seq_ops = {
3839        .start = softnet_seq_start,
3840        .next  = softnet_seq_next,
3841        .stop  = softnet_seq_stop,
3842        .show  = softnet_seq_show,
3843};
3844
3845static int softnet_seq_open(struct inode *inode, struct file *file)
3846{
3847        return seq_open(file, &softnet_seq_ops);
3848}
3849
3850static const struct file_operations softnet_seq_fops = {
3851        .owner   = THIS_MODULE,
3852        .open    = softnet_seq_open,
3853        .read    = seq_read,
3854        .llseek  = seq_lseek,
3855        .release = seq_release,
3856};
3857
3858static void *ptype_get_idx(loff_t pos)
3859{
3860        struct packet_type *pt = NULL;
3861        loff_t i = 0;
3862        int t;
3863
3864        list_for_each_entry_rcu(pt, &ptype_all, list) {
3865                if (i == pos)
3866                        return pt;
3867                ++i;
3868        }
3869
3870        for (t = 0; t < PTYPE_HASH_SIZE; t++) {
3871                list_for_each_entry_rcu(pt, &ptype_base[t], list) {
3872                        if (i == pos)
3873                                return pt;
3874                        ++i;
3875                }
3876        }
3877        return NULL;
3878}
3879
3880static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
3881        __acquires(RCU)
3882{
3883        rcu_read_lock();
3884        return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
3885}
3886
3887static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3888{
3889        struct packet_type *pt;
3890        struct list_head *nxt;
3891        int hash;
3892
3893        ++*pos;
3894        if (v == SEQ_START_TOKEN)
3895                return ptype_get_idx(0);
3896
3897        pt = v;
3898        nxt = pt->list.next;
3899        if (pt->type == htons(ETH_P_ALL)) {
3900                if (nxt != &ptype_all)
3901                        goto found;
3902                hash = 0;
3903                nxt = ptype_base[0].next;
3904        } else
3905                hash = ntohs(pt->type) & PTYPE_HASH_MASK;
3906
3907        while (nxt == &ptype_base[hash]) {
3908                if (++hash >= PTYPE_HASH_SIZE)
3909                        return NULL;
3910                nxt = ptype_base[hash].next;
3911        }
3912found:
3913        return list_entry(nxt, struct packet_type, list);
3914}
3915
3916static void ptype_seq_stop(struct seq_file *seq, void *v)
3917        __releases(RCU)
3918{
3919        rcu_read_unlock();
3920}
3921
3922static int ptype_seq_show(struct seq_file *seq, void *v)
3923{
3924        struct packet_type *pt = v;
3925
3926        if (v == SEQ_START_TOKEN)
3927                seq_puts(seq, "Type Device      Function\n");
3928        else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
3929                if (pt->type == htons(ETH_P_ALL))
3930                        seq_puts(seq, "ALL ");
3931                else
3932                        seq_printf(seq, "%04x", ntohs(pt->type));
3933
3934                seq_printf(seq, " %-8s %pF\n",
3935                           pt->dev ? pt->dev->name : "", pt->func);
3936        }
3937
3938        return 0;
3939}
3940
3941static const struct seq_operations ptype_seq_ops = {
3942        .start = ptype_seq_start,
3943        .next  = ptype_seq_next,
3944        .stop  = ptype_seq_stop,
3945        .show  = ptype_seq_show,
3946};
3947
3948static int ptype_seq_open(struct inode *inode, struct file *file)
3949{
3950        return seq_open_net(inode, file, &ptype_seq_ops,
3951                        sizeof(struct seq_net_private));
3952}
3953
3954static const struct file_operations ptype_seq_fops = {
3955        .owner   = THIS_MODULE,
3956        .open    = ptype_seq_open,
3957        .read    = seq_read,
3958        .llseek  = seq_lseek,
3959        .release = seq_release_net,
3960};
3961
3962
3963static int __net_init dev_proc_net_init(struct net *net)
3964{
3965        int rc = -ENOMEM;
3966
3967        if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
3968                goto out;
3969        if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
3970                goto out_dev;
3971        if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
3972                goto out_softnet;
3973
3974        if (wext_proc_init(net))
3975                goto out_ptype;
3976        rc = 0;
3977out:
3978        return rc;
3979out_ptype:
3980        proc_net_remove(net, "ptype");
3981out_softnet:
3982        proc_net_remove(net, "softnet_stat");
3983out_dev:
3984        proc_net_remove(net, "dev");
3985        goto out;
3986}
3987
3988static void __net_exit dev_proc_net_exit(struct net *net)
3989{
3990        wext_proc_exit(net);
3991
3992        proc_net_remove(net, "ptype");
3993        proc_net_remove(net, "softnet_stat");
3994        proc_net_remove(net, "dev");
3995}
3996
3997static struct pernet_operations __net_initdata dev_proc_ops = {
3998        .init = dev_proc_net_init,
3999        .exit = dev_proc_net_exit,
4000};
4001
4002static int __init dev_proc_init(void)
4003{
4004        return register_pernet_subsys(&dev_proc_ops);
4005}
4006#else
4007#define dev_proc_init() 0
4008#endif  /* CONFIG_PROC_FS */
4009
4010
4011/**
4012 *      netdev_set_master       -       set up master/slave pair
4013 *      @slave: slave device
4014 *      @master: new master device
4015 *
4016 *      Changes the master device of the slave. Pass %NULL to break the
4017 *      bonding. The caller must hold the RTNL semaphore. On a failure
4018 *      a negative errno code is returned. On success the reference counts
4019 *      are adjusted, %RTM_NEWLINK is sent to the routing socket and the
4020 *      function returns zero.
4021 */
4022int netdev_set_master(struct net_device *slave, struct net_device *master)
4023{
4024        struct net_device *old = slave->master;
4025
4026        ASSERT_RTNL();
4027
4028        if (master) {
4029                if (old)
4030                        return -EBUSY;
4031                dev_hold(master);
4032        }
4033
4034        slave->master = master;
4035
4036        if (old) {
4037                synchronize_net();
4038                dev_put(old);
4039        }
4040        if (master)
4041                slave->flags |= IFF_SLAVE;
4042        else
4043                slave->flags &= ~IFF_SLAVE;
4044
4045        rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4046        return 0;
4047}
4048EXPORT_SYMBOL(netdev_set_master);
4049
4050static void dev_change_rx_flags(struct net_device *dev, int flags)
4051{
4052        const struct net_device_ops *ops = dev->netdev_ops;
4053
4054        if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4055                ops->ndo_change_rx_flags(dev, flags);
4056}
4057
4058static int __dev_set_promiscuity(struct net_device *dev, int inc)
4059{
4060        unsigned short old_flags = dev->flags;
4061        uid_t uid;
4062        gid_t gid;
4063
4064        ASSERT_RTNL();
4065
4066        dev->flags |= IFF_PROMISC;
4067        dev->promiscuity += inc;
4068        if (dev->promiscuity == 0) {
4069                /*
4070                 * Avoid overflow.
4071                 * If inc causes overflow, untouch promisc and return error.
4072                 */
4073                if (inc < 0)
4074                        dev->flags &= ~IFF_PROMISC;
4075                else {
4076                        dev->promiscuity -= inc;
4077                        printk(KERN_WARNING "%s: promiscuity touches roof, "
4078                                "set promiscuity failed, promiscuity feature "
4079                                "of device might be broken.\n", dev->name);
4080                        return -EOVERFLOW;
4081                }
4082        }
4083        if (dev->flags != old_flags) {
4084                printk(KERN_INFO "device %s %s promiscuous mode\n",
4085                       dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4086                                                               "left");
4087                if (audit_enabled) {
4088                        current_uid_gid(&uid, &gid);
4089                        audit_log(current->audit_context, GFP_ATOMIC,
4090                                AUDIT_ANOM_PROMISCUOUS,
4091                                "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4092                                dev->name, (dev->flags & IFF_PROMISC),
4093                                (old_flags & IFF_PROMISC),
4094                                audit_get_loginuid(current),
4095                                uid, gid,
4096                                audit_get_sessionid(current));
4097                }
4098
4099                dev_change_rx_flags(dev, IFF_PROMISC);
4100        }
4101        return 0;
4102}
4103
4104/**
4105 *      dev_set_promiscuity     - update promiscuity count on a device
4106 *      @dev: device
4107 *      @inc: modifier
4108 *
4109 *      Add or remove promiscuity from a device. While the count in the device
4110 *      remains above zero the interface remains promiscuous. Once it hits zero
4111 *      the device reverts back to normal filtering operation. A negative inc
4112 *      value is used to drop promiscuity on the device.
4113 *      Return 0 if successful or a negative errno code on error.
4114 */
4115int dev_set_promiscuity(struct net_device *dev, int inc)
4116{
4117        unsigned short old_flags = dev->flags;
4118        int err;
4119
4120        err = __dev_set_promiscuity(dev, inc);
4121        if (err < 0)
4122                return err;
4123        if (dev->flags != old_flags)
4124                dev_set_rx_mode(dev);
4125        return err;
4126}
4127EXPORT_SYMBOL(dev_set_promiscuity);
4128
4129/**
4130 *      dev_set_allmulti        - update allmulti count on a device
4131 *      @dev: device
4132 *      @inc: modifier
4133 *
4134 *      Add or remove reception of all multicast frames to a device. While the
4135 *      count in the device remains above zero the interface remains listening
4136 *      to all interfaces. Once it hits zero the device reverts back to normal
4137 *      filtering operation. A negative @inc value is used to drop the counter
4138 *      when releasing a resource needing all multicasts.
4139 *      Return 0 if successful or a negative errno code on error.
4140 */
4141
4142int dev_set_allmulti(struct net_device *dev, int inc)
4143{
4144        unsigned short old_flags = dev->flags;
4145
4146        ASSERT_RTNL();
4147
4148        dev->flags |= IFF_ALLMULTI;
4149        dev->allmulti += inc;
4150        if (dev->allmulti == 0) {
4151                /*
4152                 * Avoid overflow.
4153                 * If inc causes overflow, untouch allmulti and return error.
4154                 */
4155                if (inc < 0)
4156                        dev->flags &= ~IFF_ALLMULTI;
4157                else {
4158                        dev->allmulti -= inc;
4159                        printk(KERN_WARNING "%s: allmulti touches roof, "
4160                                "set allmulti failed, allmulti feature of "
4161                                "device might be broken.\n", dev->name);
4162                        return -EOVERFLOW;
4163                }
4164        }
4165        if (dev->flags ^ old_flags) {
4166                dev_change_rx_flags(dev, IFF_ALLMULTI);
4167                dev_set_rx_mode(dev);
4168        }
4169        return 0;
4170}
4171EXPORT_SYMBOL(dev_set_allmulti);
4172
4173/*
4174 *      Upload unicast and multicast address lists to device and
4175 *      configure RX filtering. When the device doesn't support unicast
4176 *      filtering it is put in promiscuous mode while unicast addresses
4177 *      are present.
4178 */
4179void __dev_set_rx_mode(struct net_device *dev)
4180{
4181        const struct net_device_ops *ops = dev->netdev_ops;
4182
4183        /* dev_open will call this function so the list will stay sane. */
4184        if (!(dev->flags&IFF_UP))
4185                return;
4186
4187        if (!netif_device_present(dev))
4188                return;
4189
4190        if (ops->ndo_set_rx_mode)
4191                ops->ndo_set_rx_mode(dev);
4192        else {
4193                /* Unicast addresses changes may only happen under the rtnl,
4194                 * therefore calling __dev_set_promiscuity here is safe.
4195                 */
4196                if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4197                        __dev_set_promiscuity(dev, 1);
4198                        dev->uc_promisc = 1;
4199                } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4200                        __dev_set_promiscuity(dev, -1);
4201                        dev->uc_promisc = 0;
4202                }
4203
4204                if (ops->ndo_set_multicast_list)
4205                        ops->ndo_set_multicast_list(dev);
4206        }
4207}
4208
4209void dev_set_rx_mode(struct net_device *dev)
4210{
4211        netif_addr_lock_bh(dev);
4212        __dev_set_rx_mode(dev);
4213        netif_addr_unlock_bh(dev);
4214}
4215
4216/**
4217 *      dev_get_flags - get flags reported to userspace
4218 *      @dev: device
4219 *
4220 *      Get the combination of flag bits exported through APIs to userspace.
4221 */
4222unsigned dev_get_flags(const struct net_device *dev)
4223{
4224        unsigned flags;
4225
4226        flags = (dev->flags & ~(IFF_PROMISC |
4227                                IFF_ALLMULTI |
4228                                IFF_RUNNING |
4229                                IFF_LOWER_UP |
4230                                IFF_DORMANT)) |
4231                (dev->gflags & (IFF_PROMISC |
4232                                IFF_ALLMULTI));
4233
4234        if (netif_running(dev)) {
4235                if (netif_oper_up(dev))
4236                        flags |= IFF_RUNNING;
4237                if (netif_carrier_ok(dev))
4238                        flags |= IFF_LOWER_UP;
4239                if (netif_dormant(dev))
4240                        flags |= IFF_DORMANT;
4241        }
4242
4243        return flags;
4244}
4245EXPORT_SYMBOL(dev_get_flags);
4246
4247int __dev_change_flags(struct net_device *dev, unsigned int flags)
4248{
4249        int old_flags = dev->flags;
4250        int ret;
4251
4252        ASSERT_RTNL();
4253
4254        /*
4255         *      Set the flags on our device.
4256         */
4257
4258        dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4259                               IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4260                               IFF_AUTOMEDIA)) |
4261                     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4262                                    IFF_ALLMULTI));
4263
4264        /*
4265         *      Load in the correct multicast list now the flags have changed.
4266         */
4267
4268        if ((old_flags ^ flags) & IFF_MULTICAST)
4269                dev_change_rx_flags(dev, IFF_MULTICAST);
4270
4271        dev_set_rx_mode(dev);
4272
4273        /*
4274         *      Have we downed the interface. We handle IFF_UP ourselves
4275         *      according to user attempts to set it, rather than blindly
4276         *      setting it.
4277         */
4278
4279        ret = 0;
4280        if ((old_flags ^ flags) & IFF_UP) {     /* Bit is different  ? */
4281                ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4282
4283                if (!ret)
4284                        dev_set_rx_mode(dev);
4285        }
4286
4287        if ((flags ^ dev->gflags) & IFF_PROMISC) {
4288                int inc = (flags & IFF_PROMISC) ? 1 : -1;
4289
4290                dev->gflags ^= IFF_PROMISC;
4291                dev_set_promiscuity(dev, inc);
4292        }
4293
4294        /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4295           is important. Some (broken) drivers set IFF_PROMISC, when
4296           IFF_ALLMULTI is requested not asking us and not reporting.
4297         */
4298        if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4299                int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4300
4301                dev->gflags ^= IFF_ALLMULTI;
4302                dev_set_allmulti(dev, inc);
4303        }
4304
4305        return ret;
4306}
4307
4308void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4309{
4310        unsigned int changes = dev->flags ^ old_flags;
4311
4312        if (changes & IFF_UP) {
4313                if (dev->flags & IFF_UP)
4314                        call_netdevice_notifiers(NETDEV_UP, dev);
4315                else
4316                        call_netdevice_notifiers(NETDEV_DOWN, dev);
4317        }
4318
4319        if (dev->flags & IFF_UP &&
4320            (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4321                call_netdevice_notifiers(NETDEV_CHANGE, dev);
4322}
4323
4324/**
4325 *      dev_change_flags - change device settings
4326 *      @dev: device
4327 *      @flags: device state flags
4328 *
4329 *      Change settings on device based state flags. The flags are
4330 *      in the userspace exported format.
4331 */
4332int dev_change_flags(struct net_device *dev, unsigned flags)
4333{
4334        int ret, changes;
4335        int old_flags = dev->flags;
4336
4337        ret = __dev_change_flags(dev, flags);
4338        if (ret < 0)
4339                return ret;
4340
4341        changes = old_flags ^ dev->flags;
4342        if (changes)
4343                rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4344
4345        __dev_notify_flags(dev, old_flags);
4346        return ret;
4347}
4348EXPORT_SYMBOL(dev_change_flags);
4349
4350/**
4351 *      dev_set_mtu - Change maximum transfer unit
4352 *      @dev: device
4353 *      @new_mtu: new transfer unit
4354 *
4355 *      Change the maximum transfer size of the network device.
4356 */
4357int dev_set_mtu(struct net_device *dev, int new_mtu)
4358{
4359        const struct net_device_ops *ops = dev->netdev_ops;
4360        int err;
4361
4362        if (new_mtu == dev->mtu)
4363                return 0;
4364
4365        /*      MTU must be positive.    */
4366        if (new_mtu < 0)
4367                return -EINVAL;
4368
4369        if (!netif_device_present(dev))
4370                return -ENODEV;
4371
4372        err = 0;
4373        if (ops->ndo_change_mtu)
4374                err = ops->ndo_change_mtu(dev, new_mtu);
4375        else
4376                dev->mtu = new_mtu;
4377
4378        if (!err && dev->flags & IFF_UP)
4379                call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4380        return err;
4381}
4382EXPORT_SYMBOL(dev_set_mtu);
4383
4384/**
4385 *      dev_set_mac_address - Change Media Access Control Address
4386 *      @dev: device
4387 *      @sa: new address
4388 *
4389 *      Change the hardware (MAC) address of the device
4390 */
4391int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4392{
4393        const struct net_device_ops *ops = dev->netdev_ops;
4394        int err;
4395
4396        if (!ops->ndo_set_mac_address)
4397                return -EOPNOTSUPP;
4398        if (sa->sa_family != dev->type)
4399                return -EINVAL;
4400        if (!netif_device_present(dev))
4401                return -ENODEV;
4402        err = ops->ndo_set_mac_address(dev, sa);
4403        if (!err)
4404                call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4405        return err;
4406}
4407EXPORT_SYMBOL(dev_set_mac_address);
4408
4409/*
4410 *      Perform the SIOCxIFxxx calls, inside rcu_read_lock()
4411 */
4412static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
4413{
4414        int err;
4415        struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
4416
4417        if (!dev)
4418                return -ENODEV;
4419
4420        switch (cmd) {
4421        case SIOCGIFFLAGS:      /* Get interface flags */
4422                ifr->ifr_flags = (short) dev_get_flags(dev);
4423                return 0;
4424
4425        case SIOCGIFMETRIC:     /* Get the metric on the interface
4426                                   (currently unused) */
4427                ifr->ifr_metric = 0;
4428                return 0;
4429
4430        case SIOCGIFMTU:        /* Get the MTU of a device */
4431                ifr->ifr_mtu = dev->mtu;
4432                return 0;
4433
4434        case SIOCGIFHWADDR:
4435                if (!dev->addr_len)
4436                        memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4437                else
4438                        memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4439                               min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4440                ifr->ifr_hwaddr.sa_family = dev->type;
4441                return 0;
4442
4443        case SIOCGIFSLAVE:
4444                err = -EINVAL;
4445                break;
4446
4447        case SIOCGIFMAP:
4448                ifr->ifr_map.mem_start = dev->mem_start;
4449                ifr->ifr_map.mem_end   = dev->mem_end;
4450                ifr->ifr_map.base_addr = dev->base_addr;
4451                ifr->ifr_map.irq       = dev->irq;
4452                ifr->ifr_map.dma       = dev->dma;
4453                ifr->ifr_map.port      = dev->if_port;
4454                return 0;
4455
4456        case SIOCGIFINDEX:
4457                ifr->ifr_ifindex = dev->ifindex;
4458                return 0;
4459
4460        case SIOCGIFTXQLEN:
4461                ifr->ifr_qlen = dev->tx_queue_len;
4462                return 0;
4463
4464        default:
4465                /* dev_ioctl() should ensure this case
4466                 * is never reached
4467                 */
4468                WARN_ON(1);
4469                err = -EINVAL;
4470                break;
4471
4472        }
4473        return err;
4474}
4475
4476/*
4477 *      Perform the SIOCxIFxxx calls, inside rtnl_lock()
4478 */
4479static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4480{
4481        int err;
4482        struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4483        const struct net_device_ops *ops;
4484
4485        if (!dev)
4486                return -ENODEV;
4487
4488        ops = dev->netdev_ops;
4489
4490        switch (cmd) {
4491        case SIOCSIFFLAGS:      /* Set interface flags */
4492                return dev_change_flags(dev, ifr->ifr_flags);
4493
4494        case SIOCSIFMETRIC:     /* Set the metric on the interface
4495                                   (currently unused) */
4496                return -EOPNOTSUPP;
4497
4498        case SIOCSIFMTU:        /* Set the MTU of a device */
4499                return dev_set_mtu(dev, ifr->ifr_mtu);
4500
4501        case SIOCSIFHWADDR:
4502                return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
4503
4504        case SIOCSIFHWBROADCAST:
4505                if (ifr->ifr_hwaddr.sa_family != dev->type)
4506                        return -EINVAL;
4507                memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4508                       min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4509                call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4510                return 0;
4511
4512        case SIOCSIFMAP:
4513                if (ops->ndo_set_config) {
4514                        if (!netif_device_present(dev))
4515                                return -ENODEV;
4516                        return ops->ndo_set_config(dev, &ifr->ifr_map);
4517                }
4518                return -EOPNOTSUPP;
4519
4520        case SIOCADDMULTI:
4521                if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4522                    ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4523                        return -EINVAL;
4524                if (!netif_device_present(dev))
4525                        return -ENODEV;
4526                return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
4527
4528        case SIOCDELMULTI:
4529                if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4530                    ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4531                        return -EINVAL;
4532                if (!netif_device_present(dev))
4533                        return -ENODEV;
4534                return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
4535
4536        case SIOCSIFTXQLEN:
4537                if (ifr->ifr_qlen < 0)
4538                        return -EINVAL;
4539                dev->tx_queue_len = ifr->ifr_qlen;
4540                return 0;
4541
4542        case SIOCSIFNAME:
4543                ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4544                return dev_change_name(dev, ifr->ifr_newname);
4545
4546        /*
4547         *      Unknown or private ioctl
4548         */
4549        default:
4550                if ((cmd >= SIOCDEVPRIVATE &&
4551                    cmd <= SIOCDEVPRIVATE + 15) ||
4552                    cmd == SIOCBONDENSLAVE ||
4553                    cmd == SIOCBONDRELEASE ||
4554                    cmd == SIOCBONDSETHWADDR ||
4555                    cmd == SIOCBONDSLAVEINFOQUERY ||
4556                    cmd == SIOCBONDINFOQUERY ||
4557                    cmd == SIOCBONDCHANGEACTIVE ||
4558                    cmd == SIOCGMIIPHY ||
4559                    cmd == SIOCGMIIREG ||
4560                    cmd == SIOCSMIIREG ||
4561                    cmd == SIOCBRADDIF ||
4562                    cmd == SIOCBRDELIF ||
4563                    cmd == SIOCSHWTSTAMP ||
4564                    cmd == SIOCWANDEV) {
4565                        err = -EOPNOTSUPP;
4566                        if (ops->ndo_do_ioctl) {
4567                                if (netif_device_present(dev))
4568                                        err = ops->ndo_do_ioctl(dev, ifr, cmd);
4569                                else
4570                                        err = -ENODEV;
4571                        }
4572                } else
4573                        err = -EINVAL;
4574
4575        }
4576        return err;
4577}
4578
4579/*
4580 *      This function handles all "interface"-type I/O control requests. The actual
4581 *      'doing' part of this is dev_ifsioc above.
4582 */
4583
4584/**
4585 *      dev_ioctl       -       network device ioctl
4586 *      @net: the applicable net namespace
4587 *      @cmd: command to issue
4588 *      @arg: pointer to a struct ifreq in user space
4589 *
4590 *      Issue ioctl functions to devices. This is normally called by the
4591 *      user space syscall interfaces but can sometimes be useful for
4592 *      other purposes. The return value is the return from the syscall if
4593 *      positive or a negative errno code on error.
4594 */
4595
4596int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4597{
4598        struct ifreq ifr;
4599        int ret;
4600        char *colon;
4601
4602        /* One special case: SIOCGIFCONF takes ifconf argument
4603           and requires shared lock, because it sleeps writing
4604           to user space.
4605         */
4606
4607        if (cmd == SIOCGIFCONF) {
4608                rtnl_lock();
4609                ret = dev_ifconf(net, (char __user *) arg);
4610                rtnl_unlock();
4611                return ret;
4612        }
4613        if (cmd == SIOCGIFNAME)
4614                return dev_ifname(net, (struct ifreq __user *)arg);
4615
4616        if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4617                return -EFAULT;
4618
4619        ifr.ifr_name[IFNAMSIZ-1] = 0;
4620
4621        colon = strchr(ifr.ifr_name, ':');
4622        if (colon)
4623                *colon = 0;
4624
4625        /*
4626         *      See which interface the caller is talking about.
4627         */
4628
4629        switch (cmd) {
4630        /*
4631         *      These ioctl calls:
4632         *      - can be done by all.
4633         *      - atomic and do not require locking.
4634         *      - return a value
4635         */
4636        case SIOCGIFFLAGS:
4637        case SIOCGIFMETRIC:
4638        case SIOCGIFMTU:
4639        case SIOCGIFHWADDR:
4640        case SIOCGIFSLAVE:
4641        case SIOCGIFMAP:
4642        case SIOCGIFINDEX:
4643        case SIOCGIFTXQLEN:
4644                dev_load(net, ifr.ifr_name);
4645                rcu_read_lock();
4646                ret = dev_ifsioc_locked(net, &ifr, cmd);
4647                rcu_read_unlock();
4648                if (!ret) {
4649                        if (colon)
4650                                *colon = ':';
4651                        if (copy_to_user(arg, &ifr,
4652                                         sizeof(struct ifreq)))
4653                                ret = -EFAULT;
4654                }
4655                return ret;
4656
4657        case SIOCETHTOOL:
4658                dev_load(net, ifr.ifr_name);
4659                rtnl_lock();
4660                ret = dev_ethtool(net, &ifr);
4661                rtnl_unlock();
4662                if (!ret) {
4663                        if (colon)
4664                                *colon = ':';
4665                        if (copy_to_user(arg, &ifr,
4666                                         sizeof(struct ifreq)))
4667                                ret = -EFAULT;
4668                }
4669                return ret;
4670
4671        /*
4672         *      These ioctl calls:
4673         *      - require superuser power.
4674         *      - require strict serialization.
4675         *      - return a value
4676         */
4677        case SIOCGMIIPHY:
4678        case SIOCGMIIREG:
4679        case SIOCSIFNAME:
4680                if (!capable(CAP_NET_ADMIN))
4681                        return -EPERM;
4682                dev_load(net, ifr.ifr_name);
4683                rtnl_lock();
4684                ret = dev_ifsioc(net, &ifr, cmd);
4685                rtnl_unlock();
4686                if (!ret) {
4687                        if (colon)
4688                                *colon = ':';
4689                        if (copy_to_user(arg, &ifr,
4690                                         sizeof(struct ifreq)))
4691                                ret = -EFAULT;
4692                }
4693                return ret;
4694
4695        /*
4696         *      These ioctl calls:
4697         *      - require superuser power.
4698         *      - require strict serialization.
4699         *      - do not return a value
4700         */
4701        case SIOCSIFFLAGS:
4702        case SIOCSIFMETRIC:
4703        case SIOCSIFMTU:
4704        case SIOCSIFMAP:
4705        case SIOCSIFHWADDR:
4706        case SIOCSIFSLAVE:
4707        case SIOCADDMULTI:
4708        case SIOCDELMULTI:
4709        case SIOCSIFHWBROADCAST:
4710        case SIOCSIFTXQLEN:
4711        case SIOCSMIIREG:
4712        case SIOCBONDENSLAVE:
4713        case SIOCBONDRELEASE:
4714        case SIOCBONDSETHWADDR:
4715        case SIOCBONDCHANGEACTIVE:
4716        case SIOCBRADDIF:
4717        case SIOCBRDELIF:
4718        case SIOCSHWTSTAMP:
4719                if (!capable(CAP_NET_ADMIN))
4720                        return -EPERM;
4721                /* fall through */
4722        case SIOCBONDSLAVEINFOQUERY:
4723        case SIOCBONDINFOQUERY:
4724                dev_load(net, ifr.ifr_name);
4725                rtnl_lock();
4726                ret = dev_ifsioc(net, &ifr, cmd);
4727                rtnl_unlock();
4728                return ret;
4729
4730        case SIOCGIFMEM:
4731                /* Get the per device memory space. We can add this but
4732                 * currently do not support it */
4733        case SIOCSIFMEM:
4734                /* Set the per device memory buffer space.
4735                 * Not applicable in our case */
4736        case SIOCSIFLINK:
4737                return -EINVAL;
4738
4739        /*
4740         *      Unknown or private ioctl.
4741         */
4742        default:
4743                if (cmd == SIOCWANDEV ||
4744                    (cmd >= SIOCDEVPRIVATE &&
4745                     cmd <= SIOCDEVPRIVATE + 15)) {
4746                        dev_load(net, ifr.ifr_name);
4747                        rtnl_lock();
4748                        ret = dev_ifsioc(net, &ifr, cmd);
4749                        rtnl_unlock();
4750                        if (!ret && copy_to_user(arg, &ifr,
4751                                                 sizeof(struct ifreq)))
4752                                ret = -EFAULT;
4753                        return ret;
4754                }
4755                /* Take care of Wireless Extensions */
4756                if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
4757                        return wext_handle_ioctl(net, &ifr, cmd, arg);
4758                return -EINVAL;
4759        }
4760}
4761
4762
4763/**
4764 *      dev_new_index   -       allocate an ifindex
4765 *      @net: the applicable net namespace
4766 *
4767 *      Returns a suitable unique value for a new device interface
4768 *      number.  The caller must hold the rtnl semaphore or the
4769 *      dev_base_lock to be sure it remains unique.
4770 */
4771static int dev_new_index(struct net *net)
4772{
4773        static int ifindex;
4774        for (;;) {
4775                if (++ifindex <= 0)
4776                        ifindex = 1;
4777                if (!__dev_get_by_index(net, ifindex))
4778                        return ifindex;
4779        }
4780}
4781
4782/* Delayed registration/unregisteration */
4783static LIST_HEAD(net_todo_list);
4784
4785static void net_set_todo(struct net_device *dev)
4786{
4787        list_add_tail(&dev->todo_list, &net_todo_list);
4788}
4789
4790static void rollback_registered_many(struct list_head *head)
4791{
4792        struct net_device *dev, *tmp;
4793
4794        BUG_ON(dev_boot_phase);
4795        ASSERT_RTNL();
4796
4797        list_for_each_entry_safe(dev, tmp, head, unreg_list) {
4798                /* Some devices call without registering
4799                 * for initialization unwind. Remove those
4800                 * devices and proceed with the remaining.
4801                 */
4802                if (dev->reg_state == NETREG_UNINITIALIZED) {
4803                        pr_debug("unregister_netdevice: device %s/%p never "
4804                                 "was registered\n", dev->name, dev);
4805
4806                        WARN_ON(1);
4807                        list_del(&dev->unreg_list);
4808                        continue;
4809                }
4810
4811                BUG_ON(dev->reg_state != NETREG_REGISTERED);
4812
4813                /* If device is running, close it first. */
4814                dev_close(dev);
4815
4816                /* And unlink it from device chain. */
4817                unlist_netdevice(dev);
4818
4819                dev->reg_state = NETREG_UNREGISTERING;
4820        }
4821
4822        synchronize_net();
4823
4824        list_for_each_entry(dev, head, unreg_list) {
4825                /* Shutdown queueing discipline. */
4826                dev_shutdown(dev);
4827
4828
4829                /* Notify protocols, that we are about to destroy
4830                   this device. They should clean all the things.
4831                */
4832                call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4833
4834                if (!dev->rtnl_link_ops ||
4835                    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4836                        rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
4837
4838                /*
4839                 *      Flush the unicast and multicast chains
4840                 */
4841                dev_uc_flush(dev);
4842                dev_mc_flush(dev);
4843
4844                if (dev->netdev_ops->ndo_uninit)
4845                        dev->netdev_ops->ndo_uninit(dev);
4846
4847                /* Notifier chain MUST detach us from master device. */
4848                WARN_ON(dev->master);
4849
4850                /* Remove entries from kobject tree */
4851                netdev_unregister_kobject(dev);
4852        }
4853
4854        /* Process any work delayed until the end of the batch */
4855        dev = list_first_entry(head, struct net_device, unreg_list);
4856        call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
4857
4858        synchronize_net();
4859
4860        list_for_each_entry(dev, head, unreg_list)
4861                dev_put(dev);
4862}
4863
4864static void rollback_registered(struct net_device *dev)
4865{
4866        LIST_HEAD(single);
4867
4868        list_add(&dev->unreg_list, &single);
4869        rollback_registered_many(&single);
4870}
4871
4872static void __netdev_init_queue_locks_one(struct net_device *dev,
4873                                          struct netdev_queue *dev_queue,
4874                                          void *_unused)
4875{
4876        spin_lock_init(&dev_queue->_xmit_lock);
4877        netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);
4878        dev_queue->xmit_lock_owner = -1;
4879}
4880
4881static void netdev_init_queue_locks(struct net_device *dev)
4882{
4883        netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);
4884        __netdev_init_queue_locks_one(dev, &dev->rx_queue, NULL);
4885}
4886
4887unsigned long netdev_fix_features(unsigned long features, const char *name)
4888{
4889        /* Fix illegal SG+CSUM combinations. */
4890        if ((features & NETIF_F_SG) &&
4891            !(features & NETIF_F_ALL_CSUM)) {
4892                if (name)
4893                        printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
4894                               "checksum feature.\n", name);
4895                features &= ~NETIF_F_SG;
4896        }
4897
4898        /* TSO requires that SG is present as well. */
4899        if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
4900                if (name)
4901                        printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
4902                               "SG feature.\n", name);
4903                features &= ~NETIF_F_TSO;
4904        }
4905
4906        if (features & NETIF_F_UFO) {
4907                if (!(features & NETIF_F_GEN_CSUM)) {
4908                        if (name)
4909                                printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4910                                       "since no NETIF_F_HW_CSUM feature.\n",
4911                                       name);
4912                        features &= ~NETIF_F_UFO;
4913                }
4914
4915                if (!(features & NETIF_F_SG)) {
4916                        if (name)
4917                                printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4918                                       "since no NETIF_F_SG feature.\n", name);
4919                        features &= ~NETIF_F_UFO;
4920                }
4921        }
4922
4923        return features;
4924}
4925EXPORT_SYMBOL(netdev_fix_features);
4926
4927/**
4928 *      netif_stacked_transfer_operstate -      transfer operstate
4929 *      @rootdev: the root or lower level device to transfer state from
4930 *      @dev: the device to transfer operstate to
4931 *
4932 *      Transfer operational state from root to device. This is normally
4933 *      called when a stacking relationship exists between the root
4934 *      device and the device(a leaf device).
4935 */
4936void netif_stacked_transfer_operstate(const struct net_device *rootdev,
4937                                        struct net_device *dev)
4938{
4939        if (rootdev->operstate == IF_OPER_DORMANT)
4940                netif_dormant_on(dev);
4941        else
4942                netif_dormant_off(dev);
4943
4944        if (netif_carrier_ok(rootdev)) {
4945                if (!netif_carrier_ok(dev))
4946                        netif_carrier_on(dev);
4947        } else {
4948                if (netif_carrier_ok(dev))
4949                        netif_carrier_off(dev);
4950        }
4951}
4952EXPORT_SYMBOL(netif_stacked_transfer_operstate);
4953
4954/**
4955 *      register_netdevice      - register a network device
4956 *      @dev: device to register
4957 *
4958 *      Take a completed network device structure and add it to the kernel
4959 *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
4960 *      chain. 0 is returned on success. A negative errno code is returned
4961 *      on a failure to set up the device, or if the name is a duplicate.
4962 *
4963 *      Callers must hold the rtnl semaphore. You may want
4964 *      register_netdev() instead of this.
4965 *
4966 *      BUGS:
4967 *      The locking appears insufficient to guarantee two parallel registers
4968 *      will not get the same name.
4969 */
4970
4971int register_netdevice(struct net_device *dev)
4972{
4973        int ret;
4974        struct net *net = dev_net(dev);
4975
4976        BUG_ON(dev_boot_phase);
4977        ASSERT_RTNL();
4978
4979        might_sleep();
4980
4981        /* When net_device's are persistent, this will be fatal. */
4982        BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
4983        BUG_ON(!net);
4984
4985        spin_lock_init(&dev->addr_list_lock);
4986        netdev_set_addr_lockdep_class(dev);
4987        netdev_init_queue_locks(dev);
4988
4989        dev->iflink = -1;
4990
4991#ifdef CONFIG_RPS
4992        if (!dev->num_rx_queues) {
4993                /*
4994                 * Allocate a single RX queue if driver never called
4995                 * alloc_netdev_mq
4996                 */
4997
4998                dev->_rx = kzalloc(sizeof(struct netdev_rx_queue), GFP_KERNEL);
4999                if (!dev->_rx) {
5000                        ret = -ENOMEM;
5001                        goto out;
5002                }
5003
5004                dev->_rx->first = dev->_rx;
5005                atomic_set(&dev->_rx->count, 1);
5006                dev->num_rx_queues = 1;
5007        }
5008#endif
5009        /* Init, if this function is available */
5010        if (dev->netdev_ops->ndo_init) {
5011                ret = dev->netdev_ops->ndo_init(dev);
5012                if (ret) {
5013                        if (ret > 0)
5014                                ret = -EIO;
5015                        goto out;
5016                }
5017        }
5018
5019        ret = dev_get_valid_name(dev, dev->name, 0);
5020        if (ret)
5021                goto err_uninit;
5022
5023        dev->ifindex = dev_new_index(net);
5024        if (dev->iflink == -1)
5025                dev->iflink = dev->ifindex;
5026
5027        /* Fix illegal checksum combinations */
5028        if ((dev->features & NETIF_F_HW_CSUM) &&
5029            (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5030                printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
5031                       dev->name);
5032                dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5033        }
5034
5035        if ((dev->features & NETIF_F_NO_CSUM) &&
5036            (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5037                printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
5038                       dev->name);
5039                dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5040        }
5041
5042        dev->features = netdev_fix_features(dev->features, dev->name);
5043
5044        /* Enable software GSO if SG is supported. */
5045        if (dev->features & NETIF_F_SG)
5046                dev->features |= NETIF_F_GSO;
5047
5048        ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5049        ret = notifier_to_errno(ret);
5050        if (ret)
5051                goto err_uninit;
5052
5053        ret = netdev_register_kobject(dev);
5054        if (ret)
5055                goto err_uninit;
5056        dev->reg_state = NETREG_REGISTERED;
5057
5058        /*
5059         *      Default initial state at registry is that the
5060         *      device is present.
5061         */
5062
5063        set_bit(__LINK_STATE_PRESENT, &dev->state);
5064
5065        dev_init_scheduler(dev);
5066        dev_hold(dev);
5067        list_netdevice(dev);
5068
5069        /* Notify protocols, that a new device appeared. */
5070        ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5071        ret = notifier_to_errno(ret);
5072        if (ret) {
5073                rollback_registered(dev);
5074                dev->reg_state = NETREG_UNREGISTERED;
5075        }
5076        /*
5077         *      Prevent userspace races by waiting until the network
5078         *      device is fully setup before sending notifications.
5079         */
5080        if (!dev->rtnl_link_ops ||
5081            dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5082                rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5083
5084out:
5085        return ret;
5086
5087err_uninit:
5088        if (dev->netdev_ops->ndo_uninit)
5089                dev->netdev_ops->ndo_uninit(dev);
5090        goto out;
5091}
5092EXPORT_SYMBOL(register_netdevice);
5093
5094/**
5095 *      init_dummy_netdev       - init a dummy network device for NAPI
5096 *      @dev: device to init
5097 *
5098 *      This takes a network device structure and initialize the minimum
5099 *      amount of fields so it can be used to schedule NAPI polls without
5100 *      registering a full blown interface. This is to be used by drivers
5101 *      that need to tie several hardware interfaces to a single NAPI
5102 *      poll scheduler due to HW limitations.
5103 */
5104int init_dummy_netdev(struct net_device *dev)
5105{
5106        /* Clear everything. Note we don't initialize spinlocks
5107         * are they aren't supposed to be taken by any of the
5108         * NAPI code and this dummy netdev is supposed to be
5109         * only ever used for NAPI polls
5110         */
5111        memset(dev, 0, sizeof(struct net_device));
5112
5113        /* make sure we BUG if trying to hit standard
5114         * register/unregister code path
5115         */
5116        dev->reg_state = NETREG_DUMMY;
5117
5118        /* initialize the ref count */
5119        atomic_set(&dev->refcnt, 1);
5120
5121        /* NAPI wants this */
5122        INIT_LIST_HEAD(&dev->napi_list);
5123
5124        /* a dummy interface is started by default */
5125        set_bit(__LINK_STATE_PRESENT, &dev->state);
5126        set_bit(__LINK_STATE_START, &dev->state);
5127
5128        return 0;
5129}
5130EXPORT_SYMBOL_GPL(init_dummy_netdev);
5131
5132
5133/**
5134 *      register_netdev - register a network device
5135 *      @dev: device to register
5136 *
5137 *      Take a completed network device structure and add it to the kernel
5138 *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5139 *      chain. 0 is returned on success. A negative errno code is returned
5140 *      on a failure to set up the device, or if the name is a duplicate.
5141 *
5142 *      This is a wrapper around register_netdevice that takes the rtnl semaphore
5143 *      and expands the device name if you passed a format string to
5144 *      alloc_netdev.
5145 */
5146int register_netdev(struct net_device *dev)
5147{
5148        int err;
5149
5150        rtnl_lock();
5151
5152        /*
5153         * If the name is a format string the caller wants us to do a
5154         * name allocation.
5155         */
5156        if (strchr(dev->name, '%')) {
5157                err = dev_alloc_name(dev, dev->name);
5158                if (err < 0)
5159                        goto out;
5160        }
5161
5162        err = register_netdevice(dev);
5163out:
5164        rtnl_unlock();
5165        return err;
5166}
5167EXPORT_SYMBOL(register_netdev);
5168
5169/*
5170 * netdev_wait_allrefs - wait until all references are gone.
5171 *
5172 * This is called when unregistering network devices.
5173 *
5174 * Any protocol or device that holds a reference should register
5175 * for netdevice notification, and cleanup and put back the
5176 * reference if they receive an UNREGISTER event.
5177 * We can get stuck here if buggy protocols don't correctly
5178 * call dev_put.
5179 */
5180static void netdev_wait_allrefs(struct net_device *dev)
5181{
5182        unsigned long rebroadcast_time, warning_time;
5183
5184        linkwatch_forget_dev(dev);
5185
5186        rebroadcast_time = warning_time = jiffies;
5187        while (atomic_read(&dev->refcnt) != 0) {
5188                if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5189                        rtnl_lock();
5190
5191                        /* Rebroadcast unregister notification */
5192                        call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5193                        /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
5194                         * should have already handle it the first time */
5195
5196                        if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5197                                     &dev->state)) {
5198                                /* We must not have linkwatch events
5199                                 * pending on unregister. If this
5200                                 * happens, we simply run the queue
5201                                 * unscheduled, resulting in a noop
5202                                 * for this device.
5203                                 */
5204                                linkwatch_run_queue();
5205                        }
5206
5207                        __rtnl_unlock();
5208
5209                        rebroadcast_time = jiffies;
5210                }
5211
5212                msleep(250);
5213
5214                if (time_after(jiffies, warning_time + 10 * HZ)) {
5215                        printk(KERN_EMERG "unregister_netdevice: "
5216                               "waiting for %s to become free. Usage "
5217                               "count = %d\n",
5218                               dev->name, atomic_read(&dev->refcnt));
5219                        warning_time = jiffies;
5220                }
5221        }
5222}
5223
5224/* The sequence is:
5225 *
5226 *      rtnl_lock();
5227 *      ...
5228 *      register_netdevice(x1);
5229 *      register_netdevice(x2);
5230 *      ...
5231 *      unregister_netdevice(y1);
5232 *      unregister_netdevice(y2);
5233 *      ...
5234 *      rtnl_unlock();
5235 *      free_netdev(y1);
5236 *      free_netdev(y2);
5237 *
5238 * We are invoked by rtnl_unlock().
5239 * This allows us to deal with problems:
5240 * 1) We can delete sysfs objects which invoke hotplug
5241 *    without deadlocking with linkwatch via keventd.
5242 * 2) Since we run with the RTNL semaphore not held, we can sleep
5243 *    safely in order to wait for the netdev refcnt to drop to zero.
5244 *
5245 * We must not return until all unregister events added during
5246 * the interval the lock was held have been completed.
5247 */
5248void netdev_run_todo(void)
5249{
5250        struct list_head list;
5251
5252        /* Snapshot list, allow later requests */
5253        list_replace_init(&net_todo_list, &list);
5254
5255        __rtnl_unlock();
5256
5257        while (!list_empty(&list)) {
5258                struct net_device *dev
5259                        = list_first_entry(&list, struct net_device, todo_list);
5260                list_del(&dev->todo_list);
5261
5262                if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5263                        printk(KERN_ERR "network todo '%s' but state %d\n",
5264                               dev->name, dev->reg_state);
5265                        dump_stack();
5266                        continue;
5267                }
5268
5269                dev->reg_state = NETREG_UNREGISTERED;
5270
5271                on_each_cpu(flush_backlog, dev, 1);
5272
5273                netdev_wait_allrefs(dev);
5274
5275                /* paranoia */
5276                BUG_ON(atomic_read(&dev->refcnt));
5277                WARN_ON(dev->ip_ptr);
5278                WARN_ON(dev->ip6_ptr);
5279                WARN_ON(dev->dn_ptr);
5280
5281                if (dev->destructor)
5282                        dev->destructor(dev);
5283
5284                /* Free network device */
5285                kobject_put(&dev->dev.kobj);
5286        }
5287}
5288
5289/**
5290 *      dev_txq_stats_fold - fold tx_queues stats
5291 *      @dev: device to get statistics from
5292 *      @stats: struct net_device_stats to hold results
5293 */
5294void dev_txq_stats_fold(const struct net_device *dev,
5295                        struct net_device_stats *stats)
5296{
5297        unsigned long tx_bytes = 0, tx_packets = 0, tx_dropped = 0;
5298        unsigned int i;
5299        struct netdev_queue *txq;
5300
5301        for (i = 0; i < dev->num_tx_queues; i++) {
5302                txq = netdev_get_tx_queue(dev, i);
5303                tx_bytes   += txq->tx_bytes;
5304                tx_packets += txq->tx_packets;
5305                tx_dropped += txq->tx_dropped;
5306        }
5307        if (tx_bytes || tx_packets || tx_dropped) {
5308                stats->tx_bytes   = tx_bytes;
5309                stats->tx_packets = tx_packets;
5310                stats->tx_dropped = tx_dropped;
5311        }
5312}
5313EXPORT_SYMBOL(dev_txq_stats_fold);
5314
5315/**
5316 *      dev_get_stats   - get network device statistics
5317 *      @dev: device to get statistics from
5318 *
5319 *      Get network statistics from device. The device driver may provide
5320 *      its own method by setting dev->netdev_ops->get_stats; otherwise
5321 *      the internal statistics structure is used.
5322 */
5323const struct net_device_stats *dev_get_stats(struct net_device *dev)
5324{
5325        const struct net_device_ops *ops = dev->netdev_ops;
5326
5327        if (ops->ndo_get_stats)
5328                return ops->ndo_get_stats(dev);
5329
5330        dev_txq_stats_fold(dev, &dev->stats);
5331        return &dev->stats;
5332}
5333EXPORT_SYMBOL(dev_get_stats);
5334
5335static void netdev_init_one_queue(struct net_device *dev,
5336                                  struct netdev_queue *queue,
5337                                  void *_unused)
5338{
5339        queue->dev = dev;
5340}
5341
5342static void netdev_init_queues(struct net_device *dev)
5343{
5344        netdev_init_one_queue(dev, &dev->rx_queue, NULL);
5345        netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5346        spin_lock_init(&dev->tx_global_lock);
5347}
5348
5349/**
5350 *      alloc_netdev_mq - allocate network device
5351 *      @sizeof_priv:   size of private data to allocate space for
5352 *      @name:          device name format string
5353 *      @setup:         callback to initialize device
5354 *      @queue_count:   the number of subqueues to allocate
5355 *
5356 *      Allocates a struct net_device with private data area for driver use
5357 *      and performs basic initialization.  Also allocates subquue structs
5358 *      for each queue on the device at the end of the netdevice.
5359 */
5360struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
5361                void (*setup)(struct net_device *), unsigned int queue_count)
5362{
5363        struct netdev_queue *tx;
5364        struct net_device *dev;
5365        size_t alloc_size;
5366        struct net_device *p;
5367#ifdef CONFIG_RPS
5368        struct netdev_rx_queue *rx;
5369        int i;
5370#endif
5371
5372        BUG_ON(strlen(name) >= sizeof(dev->name));
5373
5374        alloc_size = sizeof(struct net_device);
5375        if (sizeof_priv) {
5376                /* ensure 32-byte alignment of private area */
5377                alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5378                alloc_size += sizeof_priv;
5379        }
5380        /* ensure 32-byte alignment of whole construct */
5381        alloc_size += NETDEV_ALIGN - 1;
5382
5383        p = kzalloc(alloc_size, GFP_KERNEL);
5384        if (!p) {
5385                printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
5386                return NULL;
5387        }
5388
5389        tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);
5390        if (!tx) {
5391                printk(KERN_ERR "alloc_netdev: Unable to allocate "
5392                       "tx qdiscs.\n");
5393                goto free_p;
5394        }
5395
5396#ifdef CONFIG_RPS
5397        rx = kcalloc(queue_count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5398        if (!rx) {
5399                printk(KERN_ERR "alloc_netdev: Unable to allocate "
5400                       "rx queues.\n");
5401                goto free_tx;
5402        }
5403
5404        atomic_set(&rx->count, queue_count);
5405
5406        /*
5407         * Set a pointer to first element in the array which holds the
5408         * reference count.
5409         */
5410        for (i = 0; i < queue_count; i++)
5411                rx[i].first = rx;
5412#endif
5413
5414        dev = PTR_ALIGN(p, NETDEV_ALIGN);
5415        dev->padded = (char *)dev - (char *)p;
5416
5417        if (dev_addr_init(dev))
5418                goto free_rx;
5419
5420        dev_mc_init(dev);
5421        dev_uc_init(dev);
5422
5423        dev_net_set(dev, &init_net);
5424
5425        dev->_tx = tx;
5426        dev->num_tx_queues = queue_count;
5427        dev->real_num_tx_queues = queue_count;
5428
5429#ifdef CONFIG_RPS
5430        dev->_rx = rx;
5431        dev->num_rx_queues = queue_count;
5432#endif
5433
5434        dev->gso_max_size = GSO_MAX_SIZE;
5435
5436        netdev_init_queues(dev);
5437
5438        INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5439        dev->ethtool_ntuple_list.count = 0;
5440        INIT_LIST_HEAD(&dev->napi_list);
5441        INIT_LIST_HEAD(&dev->unreg_list);
5442        INIT_LIST_HEAD(&dev->link_watch_list);
5443        dev->priv_flags = IFF_XMIT_DST_RELEASE;
5444        setup(dev);
5445        strcpy(dev->name, name);
5446        return dev;
5447
5448free_rx:
5449#ifdef CONFIG_RPS
5450        kfree(rx);
5451free_tx:
5452#endif
5453        kfree(tx);
5454free_p:
5455        kfree(p);
5456        return NULL;
5457}
5458EXPORT_SYMBOL(alloc_netdev_mq);
5459
5460/**
5461 *      free_netdev - free network device
5462 *      @dev: device
5463 *
5464 *      This function does the last stage of destroying an allocated device
5465 *      interface. The reference to the device object is released.
5466 *      If this is the last reference then it will be freed.
5467 */
5468void free_netdev(struct net_device *dev)
5469{
5470        struct napi_struct *p, *n;
5471
5472        release_net(dev_net(dev));
5473
5474        kfree(dev->_tx);
5475
5476        /* Flush device addresses */
5477        dev_addr_flush(dev);
5478
5479        /* Clear ethtool n-tuple list */
5480        ethtool_ntuple_flush(dev);
5481
5482        list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5483                netif_napi_del(p);
5484
5485        /*  Compatibility with error handling in drivers */
5486        if (dev->reg_state == NETREG_UNINITIALIZED) {
5487                kfree((char *)dev - dev->padded);
5488                return;
5489        }
5490
5491        BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5492        dev->reg_state = NETREG_RELEASED;
5493
5494        /* will free via device release */
5495        put_device(&dev->dev);
5496}
5497EXPORT_SYMBOL(free_netdev);
5498
5499/**
5500 *      synchronize_net -  Synchronize with packet receive processing
5501 *
5502 *      Wait for packets currently being received to be done.
5503 *      Does not block later packets from starting.
5504 */
5505void synchronize_net(void)
5506{
5507        might_sleep();
5508        synchronize_rcu();
5509}
5510EXPORT_SYMBOL(synchronize_net);
5511
5512/**
5513 *      unregister_netdevice_queue - remove device from the kernel
5514 *      @dev: device
5515 *      @head: list
5516 *
5517 *      This function shuts down a device interface and removes it
5518 *      from the kernel tables.
5519 *      If head not NULL, device is queued to be unregistered later.
5520 *
5521 *      Callers must hold the rtnl semaphore.  You may want
5522 *      unregister_netdev() instead of this.
5523 */
5524
5525void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5526{
5527        ASSERT_RTNL();
5528
5529        if (head) {
5530                list_move_tail(&dev->unreg_list, head);
5531        } else {
5532                rollback_registered(dev);
5533                /* Finish processing unregister after unlock */
5534                net_set_todo(dev);
5535        }
5536}
5537EXPORT_SYMBOL(unregister_netdevice_queue);
5538
5539/**
5540 *      unregister_netdevice_many - unregister many devices
5541 *      @head: list of devices
5542 */
5543void unregister_netdevice_many(struct list_head *head)
5544{
5545        struct net_device *dev;
5546
5547        if (!list_empty(head)) {
5548                rollback_registered_many(head);
5549                list_for_each_entry(dev, head, unreg_list)
5550                        net_set_todo(dev);
5551        }
5552}
5553EXPORT_SYMBOL(unregister_netdevice_many);
5554
5555/**
5556 *      unregister_netdev - remove device from the kernel
5557 *      @dev: device
5558 *
5559 *      This function shuts down a device interface and removes it
5560 *      from the kernel tables.
5561 *
5562 *      This is just a wrapper for unregister_netdevice that takes
5563 *      the rtnl semaphore.  In general you want to use this and not
5564 *      unregister_netdevice.
5565 */
5566void unregister_netdev(struct net_device *dev)
5567{
5568        rtnl_lock();
5569        unregister_netdevice(dev);
5570        rtnl_unlock();
5571}
5572EXPORT_SYMBOL(unregister_netdev);
5573
5574/**
5575 *      dev_change_net_namespace - move device to different nethost namespace
5576 *      @dev: device
5577 *      @net: network namespace
5578 *      @pat: If not NULL name pattern to try if the current device name
5579 *            is already taken in the destination network namespace.
5580 *
5581 *      This function shuts down a device interface and moves it
5582 *      to a new network namespace. On success 0 is returned, on
5583 *      a failure a netagive errno code is returned.
5584 *
5585 *      Callers must hold the rtnl semaphore.
5586 */
5587
5588int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5589{
5590        int err;
5591
5592        ASSERT_RTNL();
5593
5594        /* Don't allow namespace local devices to be moved. */
5595        err = -EINVAL;
5596        if (dev->features & NETIF_F_NETNS_LOCAL)
5597                goto out;
5598
5599        /* Ensure the device has been registrered */
5600        err = -EINVAL;
5601        if (dev->reg_state != NETREG_REGISTERED)
5602                goto out;
5603
5604        /* Get out if there is nothing todo */
5605        err = 0;
5606        if (net_eq(dev_net(dev), net))
5607                goto out;
5608
5609        /* Pick the destination device name, and ensure
5610         * we can use it in the destination network namespace.
5611         */
5612        err = -EEXIST;
5613        if (__dev_get_by_name(net, dev->name)) {
5614                /* We get here if we can't use the current device name */
5615                if (!pat)
5616                        goto out;
5617                if (dev_get_valid_name(dev, pat, 1))
5618                        goto out;
5619        }
5620
5621        /*
5622         * And now a mini version of register_netdevice unregister_netdevice.
5623         */
5624
5625        /* If device is running close it first. */
5626        dev_close(dev);
5627
5628        /* And unlink it from device chain */
5629        err = -ENODEV;
5630        unlist_netdevice(dev);
5631
5632        synchronize_net();
5633
5634        /* Shutdown queueing discipline. */
5635        dev_shutdown(dev);
5636
5637        /* Notify protocols, that we are about to destroy
5638           this device. They should clean all the things.
5639        */
5640        call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5641        call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5642
5643        /*
5644         *      Flush the unicast and multicast chains
5645         */
5646        dev_uc_flush(dev);
5647        dev_mc_flush(dev);
5648
5649        /* Actually switch the network namespace */
5650        dev_net_set(dev, net);
5651
5652        /* If there is an ifindex conflict assign a new one */
5653        if (__dev_get_by_index(net, dev->ifindex)) {
5654                int iflink = (dev->iflink == dev->ifindex);
5655                dev->ifindex = dev_new_index(net);
5656                if (iflink)
5657                        dev->iflink = dev->ifindex;
5658        }
5659
5660        /* Fixup kobjects */
5661        err = device_rename(&dev->dev, dev->name);
5662        WARN_ON(err);
5663
5664        /* Add the device back in the hashes */
5665        list_netdevice(dev);
5666
5667        /* Notify protocols, that a new device appeared. */
5668        call_netdevice_notifiers(NETDEV_REGISTER, dev);
5669
5670        /*
5671         *      Prevent userspace races by waiting until the network
5672         *      device is fully setup before sending notifications.
5673         */
5674        rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5675
5676        synchronize_net();
5677        err = 0;
5678out:
5679        return err;
5680}
5681EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5682
5683static int dev_cpu_callback(struct notifier_block *nfb,
5684                            unsigned long action,
5685                            void *ocpu)
5686{
5687        struct sk_buff **list_skb;
5688        struct sk_buff *skb;
5689        unsigned int cpu, oldcpu = (unsigned long)ocpu;
5690        struct softnet_data *sd, *oldsd;
5691
5692        if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
5693                return NOTIFY_OK;
5694
5695        local_irq_disable();
5696        cpu = smp_processor_id();
5697        sd = &per_cpu(softnet_data, cpu);
5698        oldsd = &per_cpu(softnet_data, oldcpu);
5699
5700        /* Find end of our completion_queue. */
5701        list_skb = &sd->completion_queue;
5702        while (*list_skb)
5703                list_skb = &(*list_skb)->next;
5704        /* Append completion queue from offline CPU. */
5705        *list_skb = oldsd->completion_queue;
5706        oldsd->completion_queue = NULL;
5707
5708        /* Append output queue from offline CPU. */
5709        if (oldsd->output_queue