linux/net/core/dev.c
<<
>>
Prefs
   1/*
   2 *      NET3    Protocol independent device support routines.
   3 *
   4 *              This program is free software; you can redistribute it and/or
   5 *              modify it under the terms of the GNU General Public License
   6 *              as published by the Free Software Foundation; either version
   7 *              2 of the License, or (at your option) any later version.
   8 *
   9 *      Derived from the non IP parts of dev.c 1.0.19
  10 *              Authors:        Ross Biro
  11 *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12 *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
  13 *
  14 *      Additional Authors:
  15 *              Florian la Roche <rzsfl@rz.uni-sb.de>
  16 *              Alan Cox <gw4pts@gw4pts.ampr.org>
  17 *              David Hinds <dahinds@users.sourceforge.net>
  18 *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  19 *              Adam Sulmicki <adam@cfar.umd.edu>
  20 *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  21 *
  22 *      Changes:
  23 *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  24 *                                      to 2 if register_netdev gets called
  25 *                                      before net_dev_init & also removed a
  26 *                                      few lines of code in the process.
  27 *              Alan Cox        :       device private ioctl copies fields back.
  28 *              Alan Cox        :       Transmit queue code does relevant
  29 *                                      stunts to keep the queue safe.
  30 *              Alan Cox        :       Fixed double lock.
  31 *              Alan Cox        :       Fixed promisc NULL pointer trap
  32 *              ????????        :       Support the full private ioctl range
  33 *              Alan Cox        :       Moved ioctl permission check into
  34 *                                      drivers
  35 *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
  36 *              Alan Cox        :       100 backlog just doesn't cut it when
  37 *                                      you start doing multicast video 8)
  38 *              Alan Cox        :       Rewrote net_bh and list manager.
  39 *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
  40 *              Alan Cox        :       Took out transmit every packet pass
  41 *                                      Saved a few bytes in the ioctl handler
  42 *              Alan Cox        :       Network driver sets packet type before
  43 *                                      calling netif_rx. Saves a function
  44 *                                      call a packet.
  45 *              Alan Cox        :       Hashed net_bh()
  46 *              Richard Kooijman:       Timestamp fixes.
  47 *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
  48 *              Alan Cox        :       Device lock protection.
  49 *              Alan Cox        :       Fixed nasty side effect of device close
  50 *                                      changes.
  51 *              Rudi Cilibrasi  :       Pass the right thing to
  52 *                                      set_mac_address()
  53 *              Dave Miller     :       32bit quantity for the device lock to
  54 *                                      make it work out on a Sparc.
  55 *              Bjorn Ekwall    :       Added KERNELD hack.
  56 *              Alan Cox        :       Cleaned up the backlog initialise.
  57 *              Craig Metz      :       SIOCGIFCONF fix if space for under
  58 *                                      1 device.
  59 *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
  60 *                                      is no device open function.
  61 *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
  62 *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
  63 *              Cyrus Durgin    :       Cleaned for KMOD
  64 *              Adam Sulmicki   :       Bug Fix : Network Device Unload
  65 *                                      A network device unload needs to purge
  66 *                                      the backlog queue.
  67 *      Paul Rusty Russell      :       SIOCSIFNAME
  68 *              Pekka Riikonen  :       Netdev boot-time settings code
  69 *              Andrew Morton   :       Make unregister_netdevice wait
  70 *                                      indefinitely on dev->refcnt
  71 *              J Hadi Salim    :       - Backlog queue sampling
  72 *                                      - netif_rx() feedback
  73 */
  74
  75#include <asm/uaccess.h>
  76#include <linux/bitops.h>
  77#include <linux/capability.h>
  78#include <linux/cpu.h>
  79#include <linux/types.h>
  80#include <linux/kernel.h>
  81#include <linux/hash.h>
  82#include <linux/slab.h>
  83#include <linux/sched.h>
  84#include <linux/mutex.h>
  85#include <linux/string.h>
  86#include <linux/mm.h>
  87#include <linux/socket.h>
  88#include <linux/sockios.h>
  89#include <linux/errno.h>
  90#include <linux/interrupt.h>
  91#include <linux/if_ether.h>
  92#include <linux/netdevice.h>
  93#include <linux/etherdevice.h>
  94#include <linux/ethtool.h>
  95#include <linux/notifier.h>
  96#include <linux/skbuff.h>
  97#include <linux/bpf.h>
  98#include <net/net_namespace.h>
  99#include <net/sock.h>
 100#include <net/busy_poll.h>
 101#include <linux/rtnetlink.h>
 102#include <linux/stat.h>
 103#include <net/dst.h>
 104#include <net/dst_metadata.h>
 105#include <net/pkt_sched.h>
 106#include <net/checksum.h>
 107#include <net/xfrm.h>
 108#include <linux/highmem.h>
 109#include <linux/init.h>
 110#include <linux/module.h>
 111#include <linux/netpoll.h>
 112#include <linux/rcupdate.h>
 113#include <linux/delay.h>
 114#include <net/iw_handler.h>
 115#include <asm/current.h>
 116#include <linux/audit.h>
 117#include <linux/dmaengine.h>
 118#include <linux/err.h>
 119#include <linux/ctype.h>
 120#include <linux/if_arp.h>
 121#include <linux/if_vlan.h>
 122#include <linux/ip.h>
 123#include <net/ip.h>
 124#include <net/mpls.h>
 125#include <linux/ipv6.h>
 126#include <linux/in.h>
 127#include <linux/jhash.h>
 128#include <linux/random.h>
 129#include <trace/events/napi.h>
 130#include <trace/events/net.h>
 131#include <trace/events/skb.h>
 132#include <linux/pci.h>
 133#include <linux/inetdevice.h>
 134#include <linux/cpu_rmap.h>
 135#include <linux/static_key.h>
 136#include <linux/hashtable.h>
 137#include <linux/vmalloc.h>
 138#include <linux/if_macvlan.h>
 139#include <linux/errqueue.h>
 140#include <linux/hrtimer.h>
 141#include <linux/netfilter_ingress.h>
 142#include <linux/sctp.h>
 143#include <linux/crash_dump.h>
 144
 145#include "net-sysfs.h"
 146
 147/* Instead of increasing this, you should create a hash table. */
 148#define MAX_GRO_SKBS 8
 149
 150/* This should be increased if a protocol with a bigger head is added. */
 151#define GRO_MAX_HEAD (MAX_HEADER + 128)
 152
 153static DEFINE_SPINLOCK(ptype_lock);
 154static DEFINE_SPINLOCK(offload_lock);
 155struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 156struct list_head ptype_all __read_mostly;       /* Taps */
 157static struct list_head offload_base __read_mostly;
 158
 159static int netif_rx_internal(struct sk_buff *skb);
 160static int call_netdevice_notifiers_info(unsigned long val,
 161                                         struct net_device *dev,
 162                                         struct netdev_notifier_info *info);
 163
 164/*
 165 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
 166 * semaphore.
 167 *
 168 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
 169 *
 170 * Writers must hold the rtnl semaphore while they loop through the
 171 * dev_base_head list, and hold dev_base_lock for writing when they do the
 172 * actual updates.  This allows pure readers to access the list even
 173 * while a writer is preparing to update it.
 174 *
 175 * To put it another way, dev_base_lock is held for writing only to
 176 * protect against pure readers; the rtnl semaphore provides the
 177 * protection against other writers.
 178 *
 179 * See, for example usages, register_netdevice() and
 180 * unregister_netdevice(), which must be called with the rtnl
 181 * semaphore held.
 182 */
 183DEFINE_RWLOCK(dev_base_lock);
 184EXPORT_SYMBOL(dev_base_lock);
 185
 186/* protects napi_hash addition/deletion and napi_gen_id */
 187static DEFINE_SPINLOCK(napi_hash_lock);
 188
 189static unsigned int napi_gen_id = NR_CPUS;
 190static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
 191
 192static seqcount_t devnet_rename_seq;
 193
 194static inline void dev_base_seq_inc(struct net *net)
 195{
 196        while (++net->dev_base_seq == 0);
 197}
 198
 199static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 200{
 201        unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
 202
 203        return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 204}
 205
 206static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 207{
 208        return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 209}
 210
 211static inline void rps_lock(struct softnet_data *sd)
 212{
 213#ifdef CONFIG_RPS
 214        spin_lock(&sd->input_pkt_queue.lock);
 215#endif
 216}
 217
 218static inline void rps_unlock(struct softnet_data *sd)
 219{
 220#ifdef CONFIG_RPS
 221        spin_unlock(&sd->input_pkt_queue.lock);
 222#endif
 223}
 224
 225/* Device list insertion */
 226static void list_netdevice(struct net_device *dev)
 227{
 228        struct net *net = dev_net(dev);
 229
 230        ASSERT_RTNL();
 231
 232        write_lock_bh(&dev_base_lock);
 233        list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 234        hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
 235        hlist_add_head_rcu(&dev->index_hlist,
 236                           dev_index_hash(net, dev->ifindex));
 237        write_unlock_bh(&dev_base_lock);
 238
 239        dev_base_seq_inc(net);
 240}
 241
 242/* Device list removal
 243 * caller must respect a RCU grace period before freeing/reusing dev
 244 */
 245static void unlist_netdevice(struct net_device *dev)
 246{
 247        ASSERT_RTNL();
 248
 249        /* Unlink dev from the device chain */
 250        write_lock_bh(&dev_base_lock);
 251        list_del_rcu(&dev->dev_list);
 252        hlist_del_rcu(&dev->name_hlist);
 253        hlist_del_rcu(&dev->index_hlist);
 254        write_unlock_bh(&dev_base_lock);
 255
 256        dev_base_seq_inc(dev_net(dev));
 257}
 258
 259/*
 260 *      Our notifier list
 261 */
 262
 263static RAW_NOTIFIER_HEAD(netdev_chain);
 264
 265/*
 266 *      Device drivers call our routines to queue packets here. We empty the
 267 *      queue in the local softnet handler.
 268 */
 269
 270DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 271EXPORT_PER_CPU_SYMBOL(softnet_data);
 272
 273#ifdef CONFIG_LOCKDEP
 274/*
 275 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
 276 * according to dev->type
 277 */
 278static const unsigned short netdev_lock_type[] =
 279        {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 280         ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 281         ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 282         ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 283         ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 284         ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 285         ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 286         ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 287         ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 288         ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 289         ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 290         ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 291         ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
 292         ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
 293         ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
 294
 295static const char *const netdev_lock_name[] =
 296        {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 297         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 298         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 299         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 300         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 301         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 302         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 303         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 304         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 305         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 306         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 307         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 308         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
 309         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
 310         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
 311
 312static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 313static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 314
 315static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 316{
 317        int i;
 318
 319        for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 320                if (netdev_lock_type[i] == dev_type)
 321                        return i;
 322        /* the last key is used by default */
 323        return ARRAY_SIZE(netdev_lock_type) - 1;
 324}
 325
 326static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 327                                                 unsigned short dev_type)
 328{
 329        int i;
 330
 331        i = netdev_lock_pos(dev_type);
 332        lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 333                                   netdev_lock_name[i]);
 334}
 335
 336static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 337{
 338        int i;
 339
 340        i = netdev_lock_pos(dev->type);
 341        lockdep_set_class_and_name(&dev->addr_list_lock,
 342                                   &netdev_addr_lock_key[i],
 343                                   netdev_lock_name[i]);
 344}
 345#else
 346static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 347                                                 unsigned short dev_type)
 348{
 349}
 350static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 351{
 352}
 353#endif
 354
 355/*******************************************************************************
 356
 357                Protocol management and registration routines
 358
 359*******************************************************************************/
 360
 361/*
 362 *      Add a protocol ID to the list. Now that the input handler is
 363 *      smarter we can dispense with all the messy stuff that used to be
 364 *      here.
 365 *
 366 *      BEWARE!!! Protocol handlers, mangling input packets,
 367 *      MUST BE last in hash buckets and checking protocol handlers
 368 *      MUST start from promiscuous ptype_all chain in net_bh.
 369 *      It is true now, do not change it.
 370 *      Explanation follows: if protocol handler, mangling packet, will
 371 *      be the first on list, it is not able to sense, that packet
 372 *      is cloned and should be copied-on-write, so that it will
 373 *      change it and subsequent readers will get broken packet.
 374 *                                                      --ANK (980803)
 375 */
 376
 377static inline struct list_head *ptype_head(const struct packet_type *pt)
 378{
 379        if (pt->type == htons(ETH_P_ALL))
 380                return pt->dev ? &pt->dev->ptype_all : &ptype_all;
 381        else
 382                return pt->dev ? &pt->dev->ptype_specific :
 383                                 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 384}
 385
 386/**
 387 *      dev_add_pack - add packet handler
 388 *      @pt: packet type declaration
 389 *
 390 *      Add a protocol handler to the networking stack. The passed &packet_type
 391 *      is linked into kernel lists and may not be freed until it has been
 392 *      removed from the kernel lists.
 393 *
 394 *      This call does not sleep therefore it can not
 395 *      guarantee all CPU's that are in middle of receiving packets
 396 *      will see the new packet type (until the next received packet).
 397 */
 398
 399void dev_add_pack(struct packet_type *pt)
 400{
 401        struct list_head *head = ptype_head(pt);
 402
 403        spin_lock(&ptype_lock);
 404        list_add_rcu(&pt->list, head);
 405        spin_unlock(&ptype_lock);
 406}
 407EXPORT_SYMBOL(dev_add_pack);
 408
 409/**
 410 *      __dev_remove_pack        - remove packet handler
 411 *      @pt: packet type declaration
 412 *
 413 *      Remove a protocol handler that was previously added to the kernel
 414 *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 415 *      from the kernel lists and can be freed or reused once this function
 416 *      returns.
 417 *
 418 *      The packet type might still be in use by receivers
 419 *      and must not be freed until after all the CPU's have gone
 420 *      through a quiescent state.
 421 */
 422void __dev_remove_pack(struct packet_type *pt)
 423{
 424        struct list_head *head = ptype_head(pt);
 425        struct packet_type *pt1;
 426
 427        spin_lock(&ptype_lock);
 428
 429        list_for_each_entry(pt1, head, list) {
 430                if (pt == pt1) {
 431                        list_del_rcu(&pt->list);
 432                        goto out;
 433                }
 434        }
 435
 436        pr_warn("dev_remove_pack: %p not found\n", pt);
 437out:
 438        spin_unlock(&ptype_lock);
 439}
 440EXPORT_SYMBOL(__dev_remove_pack);
 441
 442/**
 443 *      dev_remove_pack  - remove packet handler
 444 *      @pt: packet type declaration
 445 *
 446 *      Remove a protocol handler that was previously added to the kernel
 447 *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 448 *      from the kernel lists and can be freed or reused once this function
 449 *      returns.
 450 *
 451 *      This call sleeps to guarantee that no CPU is looking at the packet
 452 *      type after return.
 453 */
 454void dev_remove_pack(struct packet_type *pt)
 455{
 456        __dev_remove_pack(pt);
 457
 458        synchronize_net();
 459}
 460EXPORT_SYMBOL(dev_remove_pack);
 461
 462
 463/**
 464 *      dev_add_offload - register offload handlers
 465 *      @po: protocol offload declaration
 466 *
 467 *      Add protocol offload handlers to the networking stack. The passed
 468 *      &proto_offload is linked into kernel lists and may not be freed until
 469 *      it has been removed from the kernel lists.
 470 *
 471 *      This call does not sleep therefore it can not
 472 *      guarantee all CPU's that are in middle of receiving packets
 473 *      will see the new offload handlers (until the next received packet).
 474 */
 475void dev_add_offload(struct packet_offload *po)
 476{
 477        struct packet_offload *elem;
 478
 479        spin_lock(&offload_lock);
 480        list_for_each_entry(elem, &offload_base, list) {
 481                if (po->priority < elem->priority)
 482                        break;
 483        }
 484        list_add_rcu(&po->list, elem->list.prev);
 485        spin_unlock(&offload_lock);
 486}
 487EXPORT_SYMBOL(dev_add_offload);
 488
 489/**
 490 *      __dev_remove_offload     - remove offload handler
 491 *      @po: packet offload declaration
 492 *
 493 *      Remove a protocol offload handler that was previously added to the
 494 *      kernel offload handlers by dev_add_offload(). The passed &offload_type
 495 *      is removed from the kernel lists and can be freed or reused once this
 496 *      function returns.
 497 *
 498 *      The packet type might still be in use by receivers
 499 *      and must not be freed until after all the CPU's have gone
 500 *      through a quiescent state.
 501 */
 502static void __dev_remove_offload(struct packet_offload *po)
 503{
 504        struct list_head *head = &offload_base;
 505        struct packet_offload *po1;
 506
 507        spin_lock(&offload_lock);
 508
 509        list_for_each_entry(po1, head, list) {
 510                if (po == po1) {
 511                        list_del_rcu(&po->list);
 512                        goto out;
 513                }
 514        }
 515
 516        pr_warn("dev_remove_offload: %p not found\n", po);
 517out:
 518        spin_unlock(&offload_lock);
 519}
 520
 521/**
 522 *      dev_remove_offload       - remove packet offload handler
 523 *      @po: packet offload declaration
 524 *
 525 *      Remove a packet offload handler that was previously added to the kernel
 526 *      offload handlers by dev_add_offload(). The passed &offload_type is
 527 *      removed from the kernel lists and can be freed or reused once this
 528 *      function returns.
 529 *
 530 *      This call sleeps to guarantee that no CPU is looking at the packet
 531 *      type after return.
 532 */
 533void dev_remove_offload(struct packet_offload *po)
 534{
 535        __dev_remove_offload(po);
 536
 537        synchronize_net();
 538}
 539EXPORT_SYMBOL(dev_remove_offload);
 540
 541/******************************************************************************
 542
 543                      Device Boot-time Settings Routines
 544
 545*******************************************************************************/
 546
 547/* Boot time configuration table */
 548static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
 549
 550/**
 551 *      netdev_boot_setup_add   - add new setup entry
 552 *      @name: name of the device
 553 *      @map: configured settings for the device
 554 *
 555 *      Adds new setup entry to the dev_boot_setup list.  The function
 556 *      returns 0 on error and 1 on success.  This is a generic routine to
 557 *      all netdevices.
 558 */
 559static int netdev_boot_setup_add(char *name, struct ifmap *map)
 560{
 561        struct netdev_boot_setup *s;
 562        int i;
 563
 564        s = dev_boot_setup;
 565        for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 566                if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
 567                        memset(s[i].name, 0, sizeof(s[i].name));
 568                        strlcpy(s[i].name, name, IFNAMSIZ);
 569                        memcpy(&s[i].map, map, sizeof(s[i].map));
 570                        break;
 571                }
 572        }
 573
 574        return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
 575}
 576
 577/**
 578 *      netdev_boot_setup_check - check boot time settings
 579 *      @dev: the netdevice
 580 *
 581 *      Check boot time settings for the device.
 582 *      The found settings are set for the device to be used
 583 *      later in the device probing.
 584 *      Returns 0 if no settings found, 1 if they are.
 585 */
 586int netdev_boot_setup_check(struct net_device *dev)
 587{
 588        struct netdev_boot_setup *s = dev_boot_setup;
 589        int i;
 590
 591        for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 592                if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
 593                    !strcmp(dev->name, s[i].name)) {
 594                        dev->irq        = s[i].map.irq;
 595                        dev->base_addr  = s[i].map.base_addr;
 596                        dev->mem_start  = s[i].map.mem_start;
 597                        dev->mem_end    = s[i].map.mem_end;
 598                        return 1;
 599                }
 600        }
 601        return 0;
 602}
 603EXPORT_SYMBOL(netdev_boot_setup_check);
 604
 605
 606/**
 607 *      netdev_boot_base        - get address from boot time settings
 608 *      @prefix: prefix for network device
 609 *      @unit: id for network device
 610 *
 611 *      Check boot time settings for the base address of device.
 612 *      The found settings are set for the device to be used
 613 *      later in the device probing.
 614 *      Returns 0 if no settings found.
 615 */
 616unsigned long netdev_boot_base(const char *prefix, int unit)
 617{
 618        const struct netdev_boot_setup *s = dev_boot_setup;
 619        char name[IFNAMSIZ];
 620        int i;
 621
 622        sprintf(name, "%s%d", prefix, unit);
 623
 624        /*
 625         * If device already registered then return base of 1
 626         * to indicate not to probe for this interface
 627         */
 628        if (__dev_get_by_name(&init_net, name))
 629                return 1;
 630
 631        for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
 632                if (!strcmp(name, s[i].name))
 633                        return s[i].map.base_addr;
 634        return 0;
 635}
 636
 637/*
 638 * Saves at boot time configured settings for any netdevice.
 639 */
 640int __init netdev_boot_setup(char *str)
 641{
 642        int ints[5];
 643        struct ifmap map;
 644
 645        str = get_options(str, ARRAY_SIZE(ints), ints);
 646        if (!str || !*str)
 647                return 0;
 648
 649        /* Save settings */
 650        memset(&map, 0, sizeof(map));
 651        if (ints[0] > 0)
 652                map.irq = ints[1];
 653        if (ints[0] > 1)
 654                map.base_addr = ints[2];
 655        if (ints[0] > 2)
 656                map.mem_start = ints[3];
 657        if (ints[0] > 3)
 658                map.mem_end = ints[4];
 659
 660        /* Add new entry to the list */
 661        return netdev_boot_setup_add(str, &map);
 662}
 663
 664__setup("netdev=", netdev_boot_setup);
 665
 666/*******************************************************************************
 667
 668                            Device Interface Subroutines
 669
 670*******************************************************************************/
 671
 672/**
 673 *      dev_get_iflink  - get 'iflink' value of a interface
 674 *      @dev: targeted interface
 675 *
 676 *      Indicates the ifindex the interface is linked to.
 677 *      Physical interfaces have the same 'ifindex' and 'iflink' values.
 678 */
 679
 680int dev_get_iflink(const struct net_device *dev)
 681{
 682        if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
 683                return dev->netdev_ops->ndo_get_iflink(dev);
 684
 685        return dev->ifindex;
 686}
 687EXPORT_SYMBOL(dev_get_iflink);
 688
 689/**
 690 *      dev_fill_metadata_dst - Retrieve tunnel egress information.
 691 *      @dev: targeted interface
 692 *      @skb: The packet.
 693 *
 694 *      For better visibility of tunnel traffic OVS needs to retrieve
 695 *      egress tunnel information for a packet. Following API allows
 696 *      user to get this info.
 697 */
 698int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 699{
 700        struct ip_tunnel_info *info;
 701
 702        if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
 703                return -EINVAL;
 704
 705        info = skb_tunnel_info_unclone(skb);
 706        if (!info)
 707                return -ENOMEM;
 708        if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
 709                return -EINVAL;
 710
 711        return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
 712}
 713EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
 714
 715/**
 716 *      __dev_get_by_name       - find a device by its name
 717 *      @net: the applicable net namespace
 718 *      @name: name to find
 719 *
 720 *      Find an interface by name. Must be called under RTNL semaphore
 721 *      or @dev_base_lock. If the name is found a pointer to the device
 722 *      is returned. If the name is not found then %NULL is returned. The
 723 *      reference counters are not incremented so the caller must be
 724 *      careful with locks.
 725 */
 726
 727struct net_device *__dev_get_by_name(struct net *net, const char *name)
 728{
 729        struct net_device *dev;
 730        struct hlist_head *head = dev_name_hash(net, name);
 731
 732        hlist_for_each_entry(dev, head, name_hlist)
 733                if (!strncmp(dev->name, name, IFNAMSIZ))
 734                        return dev;
 735
 736        return NULL;
 737}
 738EXPORT_SYMBOL(__dev_get_by_name);
 739
 740/**
 741 *      dev_get_by_name_rcu     - find a device by its name
 742 *      @net: the applicable net namespace
 743 *      @name: name to find
 744 *
 745 *      Find an interface by name.
 746 *      If the name is found a pointer to the device is returned.
 747 *      If the name is not found then %NULL is returned.
 748 *      The reference counters are not incremented so the caller must be
 749 *      careful with locks. The caller must hold RCU lock.
 750 */
 751
 752struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 753{
 754        struct net_device *dev;
 755        struct hlist_head *head = dev_name_hash(net, name);
 756
 757        hlist_for_each_entry_rcu(dev, head, name_hlist)
 758                if (!strncmp(dev->name, name, IFNAMSIZ))
 759                        return dev;
 760
 761        return NULL;
 762}
 763EXPORT_SYMBOL(dev_get_by_name_rcu);
 764
 765/**
 766 *      dev_get_by_name         - find a device by its name
 767 *      @net: the applicable net namespace
 768 *      @name: name to find
 769 *
 770 *      Find an interface by name. This can be called from any
 771 *      context and does its own locking. The returned handle has
 772 *      the usage count incremented and the caller must use dev_put() to
 773 *      release it when it is no longer needed. %NULL is returned if no
 774 *      matching device is found.
 775 */
 776
 777struct net_device *dev_get_by_name(struct net *net, const char *name)
 778{
 779        struct net_device *dev;
 780
 781        rcu_read_lock();
 782        dev = dev_get_by_name_rcu(net, name);
 783        if (dev)
 784                dev_hold(dev);
 785        rcu_read_unlock();
 786        return dev;
 787}
 788EXPORT_SYMBOL(dev_get_by_name);
 789
 790/**
 791 *      __dev_get_by_index - find a device by its ifindex
 792 *      @net: the applicable net namespace
 793 *      @ifindex: index of device
 794 *
 795 *      Search for an interface by index. Returns %NULL if the device
 796 *      is not found or a pointer to the device. The device has not
 797 *      had its reference counter increased so the caller must be careful
 798 *      about locking. The caller must hold either the RTNL semaphore
 799 *      or @dev_base_lock.
 800 */
 801
 802struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 803{
 804        struct net_device *dev;
 805        struct hlist_head *head = dev_index_hash(net, ifindex);
 806
 807        hlist_for_each_entry(dev, head, index_hlist)
 808                if (dev->ifindex == ifindex)
 809                        return dev;
 810
 811        return NULL;
 812}
 813EXPORT_SYMBOL(__dev_get_by_index);
 814
 815/**
 816 *      dev_get_by_index_rcu - find a device by its ifindex
 817 *      @net: the applicable net namespace
 818 *      @ifindex: index of device
 819 *
 820 *      Search for an interface by index. Returns %NULL if the device
 821 *      is not found or a pointer to the device. The device has not
 822 *      had its reference counter increased so the caller must be careful
 823 *      about locking. The caller must hold RCU lock.
 824 */
 825
 826struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 827{
 828        struct net_device *dev;
 829        struct hlist_head *head = dev_index_hash(net, ifindex);
 830
 831        hlist_for_each_entry_rcu(dev, head, index_hlist)
 832                if (dev->ifindex == ifindex)
 833                        return dev;
 834
 835        return NULL;
 836}
 837EXPORT_SYMBOL(dev_get_by_index_rcu);
 838
 839
 840/**
 841 *      dev_get_by_index - find a device by its ifindex
 842 *      @net: the applicable net namespace
 843 *      @ifindex: index of device
 844 *
 845 *      Search for an interface by index. Returns NULL if the device
 846 *      is not found or a pointer to the device. The device returned has
 847 *      had a reference added and the pointer is safe until the user calls
 848 *      dev_put to indicate they have finished with it.
 849 */
 850
 851struct net_device *dev_get_by_index(struct net *net, int ifindex)
 852{
 853        struct net_device *dev;
 854
 855        rcu_read_lock();
 856        dev = dev_get_by_index_rcu(net, ifindex);
 857        if (dev)
 858                dev_hold(dev);
 859        rcu_read_unlock();
 860        return dev;
 861}
 862EXPORT_SYMBOL(dev_get_by_index);
 863
 864/**
 865 *      netdev_get_name - get a netdevice name, knowing its ifindex.
 866 *      @net: network namespace
 867 *      @name: a pointer to the buffer where the name will be stored.
 868 *      @ifindex: the ifindex of the interface to get the name from.
 869 *
 870 *      The use of raw_seqcount_begin() and cond_resched() before
 871 *      retrying is required as we want to give the writers a chance
 872 *      to complete when CONFIG_PREEMPT is not set.
 873 */
 874int netdev_get_name(struct net *net, char *name, int ifindex)
 875{
 876        struct net_device *dev;
 877        unsigned int seq;
 878
 879retry:
 880        seq = raw_seqcount_begin(&devnet_rename_seq);
 881        rcu_read_lock();
 882        dev = dev_get_by_index_rcu(net, ifindex);
 883        if (!dev) {
 884                rcu_read_unlock();
 885                return -ENODEV;
 886        }
 887
 888        strcpy(name, dev->name);
 889        rcu_read_unlock();
 890        if (read_seqcount_retry(&devnet_rename_seq, seq)) {
 891                cond_resched();
 892                goto retry;
 893        }
 894
 895        return 0;
 896}
 897
 898/**
 899 *      dev_getbyhwaddr_rcu - find a device by its hardware address
 900 *      @net: the applicable net namespace
 901 *      @type: media type of device
 902 *      @ha: hardware address
 903 *
 904 *      Search for an interface by MAC address. Returns NULL if the device
 905 *      is not found or a pointer to the device.
 906 *      The caller must hold RCU or RTNL.
 907 *      The returned device has not had its ref count increased
 908 *      and the caller must therefore be careful about locking
 909 *
 910 */
 911
 912struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
 913                                       const char *ha)
 914{
 915        struct net_device *dev;
 916
 917        for_each_netdev_rcu(net, dev)
 918                if (dev->type == type &&
 919                    !memcmp(dev->dev_addr, ha, dev->addr_len))
 920                        return dev;
 921
 922        return NULL;
 923}
 924EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
 925
 926struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
 927{
 928        struct net_device *dev;
 929
 930        ASSERT_RTNL();
 931        for_each_netdev(net, dev)
 932                if (dev->type == type)
 933                        return dev;
 934
 935        return NULL;
 936}
 937EXPORT_SYMBOL(__dev_getfirstbyhwtype);
 938
 939struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 940{
 941        struct net_device *dev, *ret = NULL;
 942
 943        rcu_read_lock();
 944        for_each_netdev_rcu(net, dev)
 945                if (dev->type == type) {
 946                        dev_hold(dev);
 947                        ret = dev;
 948                        break;
 949                }
 950        rcu_read_unlock();
 951        return ret;
 952}
 953EXPORT_SYMBOL(dev_getfirstbyhwtype);
 954
 955/**
 956 *      __dev_get_by_flags - find any device with given flags
 957 *      @net: the applicable net namespace
 958 *      @if_flags: IFF_* values
 959 *      @mask: bitmask of bits in if_flags to check
 960 *
 961 *      Search for any interface with the given flags. Returns NULL if a device
 962 *      is not found or a pointer to the device. Must be called inside
 963 *      rtnl_lock(), and result refcount is unchanged.
 964 */
 965
 966struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
 967                                      unsigned short mask)
 968{
 969        struct net_device *dev, *ret;
 970
 971        ASSERT_RTNL();
 972
 973        ret = NULL;
 974        for_each_netdev(net, dev) {
 975                if (((dev->flags ^ if_flags) & mask) == 0) {
 976                        ret = dev;
 977                        break;
 978                }
 979        }
 980        return ret;
 981}
 982EXPORT_SYMBOL(__dev_get_by_flags);
 983
 984/**
 985 *      dev_valid_name - check if name is okay for network device
 986 *      @name: name string
 987 *
 988 *      Network device names need to be valid file names to
 989 *      to allow sysfs to work.  We also disallow any kind of
 990 *      whitespace.
 991 */
 992bool dev_valid_name(const char *name)
 993{
 994        if (*name == '\0')
 995                return false;
 996        if (strlen(name) >= IFNAMSIZ)
 997                return false;
 998        if (!strcmp(name, ".") || !strcmp(name, ".."))
 999                return false;
1000
1001        while (*name) {
1002                if (*name == '/' || *name == ':' || isspace(*name))
1003                        return false;
1004                name++;
1005        }
1006        return true;
1007}
1008EXPORT_SYMBOL(dev_valid_name);
1009
1010/**
1011 *      __dev_alloc_name - allocate a name for a device
1012 *      @net: network namespace to allocate the device name in
1013 *      @name: name format string
1014 *      @buf:  scratch buffer and result name string
1015 *
1016 *      Passed a format string - eg "lt%d" it will try and find a suitable
1017 *      id. It scans list of devices to build up a free map, then chooses
1018 *      the first empty slot. The caller must hold the dev_base or rtnl lock
1019 *      while allocating the name and adding the device in order to avoid
1020 *      duplicates.
1021 *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1022 *      Returns the number of the unit assigned or a negative errno code.
1023 */
1024
1025static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1026{
1027        int i = 0;
1028        const char *p;
1029        const int max_netdevices = 8*PAGE_SIZE;
1030        unsigned long *inuse;
1031        struct net_device *d;
1032
1033        p = strnchr(name, IFNAMSIZ-1, '%');
1034        if (p) {
1035                /*
1036                 * Verify the string as this thing may have come from
1037                 * the user.  There must be either one "%d" and no other "%"
1038                 * characters.
1039                 */
1040                if (p[1] != 'd' || strchr(p + 2, '%'))
1041                        return -EINVAL;
1042
1043                /* Use one page as a bit array of possible slots */
1044                inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1045                if (!inuse)
1046                        return -ENOMEM;
1047
1048                for_each_netdev(net, d) {
1049                        if (!sscanf(d->name, name, &i))
1050                                continue;
1051                        if (i < 0 || i >= max_netdevices)
1052                                continue;
1053
1054                        /*  avoid cases where sscanf is not exact inverse of printf */
1055                        snprintf(buf, IFNAMSIZ, name, i);
1056                        if (!strncmp(buf, d->name, IFNAMSIZ))
1057                                set_bit(i, inuse);
1058                }
1059
1060                i = find_first_zero_bit(inuse, max_netdevices);
1061                free_page((unsigned long) inuse);
1062        }
1063
1064        if (buf != name)
1065                snprintf(buf, IFNAMSIZ, name, i);
1066        if (!__dev_get_by_name(net, buf))
1067                return i;
1068
1069        /* It is possible to run out of possible slots
1070         * when the name is long and there isn't enough space left
1071         * for the digits, or if all bits are used.
1072         */
1073        return -ENFILE;
1074}
1075
1076/**
1077 *      dev_alloc_name - allocate a name for a device
1078 *      @dev: device
1079 *      @name: name format string
1080 *
1081 *      Passed a format string - eg "lt%d" it will try and find a suitable
1082 *      id. It scans list of devices to build up a free map, then chooses
1083 *      the first empty slot. The caller must hold the dev_base or rtnl lock
1084 *      while allocating the name and adding the device in order to avoid
1085 *      duplicates.
1086 *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1087 *      Returns the number of the unit assigned or a negative errno code.
1088 */
1089
1090int dev_alloc_name(struct net_device *dev, const char *name)
1091{
1092        char buf[IFNAMSIZ];
1093        struct net *net;
1094        int ret;
1095
1096        BUG_ON(!dev_net(dev));
1097        net = dev_net(dev);
1098        ret = __dev_alloc_name(net, name, buf);
1099        if (ret >= 0)
1100                strlcpy(dev->name, buf, IFNAMSIZ);
1101        return ret;
1102}
1103EXPORT_SYMBOL(dev_alloc_name);
1104
1105static int dev_alloc_name_ns(struct net *net,
1106                             struct net_device *dev,
1107                             const char *name)
1108{
1109        char buf[IFNAMSIZ];
1110        int ret;
1111
1112        ret = __dev_alloc_name(net, name, buf);
1113        if (ret >= 0)
1114                strlcpy(dev->name, buf, IFNAMSIZ);
1115        return ret;
1116}
1117
1118static int dev_get_valid_name(struct net *net,
1119                              struct net_device *dev,
1120                              const char *name)
1121{
1122        BUG_ON(!net);
1123
1124        if (!dev_valid_name(name))
1125                return -EINVAL;
1126
1127        if (strchr(name, '%'))
1128                return dev_alloc_name_ns(net, dev, name);
1129        else if (__dev_get_by_name(net, name))
1130                return -EEXIST;
1131        else if (dev->name != name)
1132                strlcpy(dev->name, name, IFNAMSIZ);
1133
1134        return 0;
1135}
1136
1137/**
1138 *      dev_change_name - change name of a device
1139 *      @dev: device
1140 *      @newname: name (or format string) must be at least IFNAMSIZ
1141 *
1142 *      Change name of a device, can pass format strings "eth%d".
1143 *      for wildcarding.
1144 */
1145int dev_change_name(struct net_device *dev, const char *newname)
1146{
1147        unsigned char old_assign_type;
1148        char oldname[IFNAMSIZ];
1149        int err = 0;
1150        int ret;
1151        struct net *net;
1152
1153        ASSERT_RTNL();
1154        BUG_ON(!dev_net(dev));
1155
1156        net = dev_net(dev);
1157        if (dev->flags & IFF_UP)
1158                return -EBUSY;
1159
1160        write_seqcount_begin(&devnet_rename_seq);
1161
1162        if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1163                write_seqcount_end(&devnet_rename_seq);
1164                return 0;
1165        }
1166
1167        memcpy(oldname, dev->name, IFNAMSIZ);
1168
1169        err = dev_get_valid_name(net, dev, newname);
1170        if (err < 0) {
1171                write_seqcount_end(&devnet_rename_seq);
1172                return err;
1173        }
1174
1175        if (oldname[0] && !strchr(oldname, '%'))
1176                netdev_info(dev, "renamed from %s\n", oldname);
1177
1178        old_assign_type = dev->name_assign_type;
1179        dev->name_assign_type = NET_NAME_RENAMED;
1180
1181rollback:
1182        ret = device_rename(&dev->dev, dev->name);
1183        if (ret) {
1184                memcpy(dev->name, oldname, IFNAMSIZ);
1185                dev->name_assign_type = old_assign_type;
1186                write_seqcount_end(&devnet_rename_seq);
1187                return ret;
1188        }
1189
1190        write_seqcount_end(&devnet_rename_seq);
1191
1192        netdev_adjacent_rename_links(dev, oldname);
1193
1194        write_lock_bh(&dev_base_lock);
1195        hlist_del_rcu(&dev->name_hlist);
1196        write_unlock_bh(&dev_base_lock);
1197
1198        synchronize_rcu();
1199
1200        write_lock_bh(&dev_base_lock);
1201        hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1202        write_unlock_bh(&dev_base_lock);
1203
1204        ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1205        ret = notifier_to_errno(ret);
1206
1207        if (ret) {
1208                /* err >= 0 after dev_alloc_name() or stores the first errno */
1209                if (err >= 0) {
1210                        err = ret;
1211                        write_seqcount_begin(&devnet_rename_seq);
1212                        memcpy(dev->name, oldname, IFNAMSIZ);
1213                        memcpy(oldname, newname, IFNAMSIZ);
1214                        dev->name_assign_type = old_assign_type;
1215                        old_assign_type = NET_NAME_RENAMED;
1216                        goto rollback;
1217                } else {
1218                        pr_err("%s: name change rollback failed: %d\n",
1219                               dev->name, ret);
1220                }
1221        }
1222
1223        return err;
1224}
1225
1226/**
1227 *      dev_set_alias - change ifalias of a device
1228 *      @dev: device
1229 *      @alias: name up to IFALIASZ
1230 *      @len: limit of bytes to copy from info
1231 *
1232 *      Set ifalias for a device,
1233 */
1234int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1235{
1236        char *new_ifalias;
1237
1238        ASSERT_RTNL();
1239
1240        if (len >= IFALIASZ)
1241                return -EINVAL;
1242
1243        if (!len) {
1244                kfree(dev->ifalias);
1245                dev->ifalias = NULL;
1246                return 0;
1247        }
1248
1249        new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1250        if (!new_ifalias)
1251                return -ENOMEM;
1252        dev->ifalias = new_ifalias;
1253
1254        strlcpy(dev->ifalias, alias, len+1);
1255        return len;
1256}
1257
1258
1259/**
1260 *      netdev_features_change - device changes features
1261 *      @dev: device to cause notification
1262 *
1263 *      Called to indicate a device has changed features.
1264 */
1265void netdev_features_change(struct net_device *dev)
1266{
1267        call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1268}
1269EXPORT_SYMBOL(netdev_features_change);
1270
1271/**
1272 *      netdev_state_change - device changes state
1273 *      @dev: device to cause notification
1274 *
1275 *      Called to indicate a device has changed state. This function calls
1276 *      the notifier chains for netdev_chain and sends a NEWLINK message
1277 *      to the routing socket.
1278 */
1279void netdev_state_change(struct net_device *dev)
1280{
1281        if (dev->flags & IFF_UP) {
1282                struct netdev_notifier_change_info change_info;
1283
1284                change_info.flags_changed = 0;
1285                call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1286                                              &change_info.info);
1287                rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1288        }
1289}
1290EXPORT_SYMBOL(netdev_state_change);
1291
1292/**
1293 *      netdev_notify_peers - notify network peers about existence of @dev
1294 *      @dev: network device
1295 *
1296 * Generate traffic such that interested network peers are aware of
1297 * @dev, such as by generating a gratuitous ARP. This may be used when
1298 * a device wants to inform the rest of the network about some sort of
1299 * reconfiguration such as a failover event or virtual machine
1300 * migration.
1301 */
1302void netdev_notify_peers(struct net_device *dev)
1303{
1304        rtnl_lock();
1305        call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1306        rtnl_unlock();
1307}
1308EXPORT_SYMBOL(netdev_notify_peers);
1309
1310static int __dev_open(struct net_device *dev)
1311{
1312        const struct net_device_ops *ops = dev->netdev_ops;
1313        int ret;
1314
1315        ASSERT_RTNL();
1316
1317        if (!netif_device_present(dev))
1318                return -ENODEV;
1319
1320        /* Block netpoll from trying to do any rx path servicing.
1321         * If we don't do this there is a chance ndo_poll_controller
1322         * or ndo_poll may be running while we open the device
1323         */
1324        netpoll_poll_disable(dev);
1325
1326        ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1327        ret = notifier_to_errno(ret);
1328        if (ret)
1329                return ret;
1330
1331        set_bit(__LINK_STATE_START, &dev->state);
1332
1333        if (ops->ndo_validate_addr)
1334                ret = ops->ndo_validate_addr(dev);
1335
1336        if (!ret && ops->ndo_open)
1337                ret = ops->ndo_open(dev);
1338
1339        netpoll_poll_enable(dev);
1340
1341        if (ret)
1342                clear_bit(__LINK_STATE_START, &dev->state);
1343        else {
1344                dev->flags |= IFF_UP;
1345                dev_set_rx_mode(dev);
1346                dev_activate(dev);
1347                add_device_randomness(dev->dev_addr, dev->addr_len);
1348        }
1349
1350        return ret;
1351}
1352
1353/**
1354 *      dev_open        - prepare an interface for use.
1355 *      @dev:   device to open
1356 *
1357 *      Takes a device from down to up state. The device's private open
1358 *      function is invoked and then the multicast lists are loaded. Finally
1359 *      the device is moved into the up state and a %NETDEV_UP message is
1360 *      sent to the netdev notifier chain.
1361 *
1362 *      Calling this function on an active interface is a nop. On a failure
1363 *      a negative errno code is returned.
1364 */
1365int dev_open(struct net_device *dev)
1366{
1367        int ret;
1368
1369        if (dev->flags & IFF_UP)
1370                return 0;
1371
1372        ret = __dev_open(dev);
1373        if (ret < 0)
1374                return ret;
1375
1376        rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1377        call_netdevice_notifiers(NETDEV_UP, dev);
1378
1379        return ret;
1380}
1381EXPORT_SYMBOL(dev_open);
1382
1383static int __dev_close_many(struct list_head *head)
1384{
1385        struct net_device *dev;
1386
1387        ASSERT_RTNL();
1388        might_sleep();
1389
1390        list_for_each_entry(dev, head, close_list) {
1391                /* Temporarily disable netpoll until the interface is down */
1392                netpoll_poll_disable(dev);
1393
1394                call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1395
1396                clear_bit(__LINK_STATE_START, &dev->state);
1397
1398                /* Synchronize to scheduled poll. We cannot touch poll list, it
1399                 * can be even on different cpu. So just clear netif_running().
1400                 *
1401                 * dev->stop() will invoke napi_disable() on all of it's
1402                 * napi_struct instances on this device.
1403                 */
1404                smp_mb__after_atomic(); /* Commit netif_running(). */
1405        }
1406
1407        dev_deactivate_many(head);
1408
1409        list_for_each_entry(dev, head, close_list) {
1410                const struct net_device_ops *ops = dev->netdev_ops;
1411
1412                /*
1413                 *      Call the device specific close. This cannot fail.
1414                 *      Only if device is UP
1415                 *
1416                 *      We allow it to be called even after a DETACH hot-plug
1417                 *      event.
1418                 */
1419                if (ops->ndo_stop)
1420                        ops->ndo_stop(dev);
1421
1422                dev->flags &= ~IFF_UP;
1423                netpoll_poll_enable(dev);
1424        }
1425
1426        return 0;
1427}
1428
1429static int __dev_close(struct net_device *dev)
1430{
1431        int retval;
1432        LIST_HEAD(single);
1433
1434        list_add(&dev->close_list, &single);
1435        retval = __dev_close_many(&single);
1436        list_del(&single);
1437
1438        return retval;
1439}
1440
1441int dev_close_many(struct list_head *head, bool unlink)
1442{
1443        struct net_device *dev, *tmp;
1444
1445        /* Remove the devices that don't need to be closed */
1446        list_for_each_entry_safe(dev, tmp, head, close_list)
1447                if (!(dev->flags & IFF_UP))
1448                        list_del_init(&dev->close_list);
1449
1450        __dev_close_many(head);
1451
1452        list_for_each_entry_safe(dev, tmp, head, close_list) {
1453                rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1454                call_netdevice_notifiers(NETDEV_DOWN, dev);
1455                if (unlink)
1456                        list_del_init(&dev->close_list);
1457        }
1458
1459        return 0;
1460}
1461EXPORT_SYMBOL(dev_close_many);
1462
1463/**
1464 *      dev_close - shutdown an interface.
1465 *      @dev: device to shutdown
1466 *
1467 *      This function moves an active device into down state. A
1468 *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1469 *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1470 *      chain.
1471 */
1472int dev_close(struct net_device *dev)
1473{
1474        if (dev->flags & IFF_UP) {
1475                LIST_HEAD(single);
1476
1477                list_add(&dev->close_list, &single);
1478                dev_close_many(&single, true);
1479                list_del(&single);
1480        }
1481        return 0;
1482}
1483EXPORT_SYMBOL(dev_close);
1484
1485
1486/**
1487 *      dev_disable_lro - disable Large Receive Offload on a device
1488 *      @dev: device
1489 *
1490 *      Disable Large Receive Offload (LRO) on a net device.  Must be
1491 *      called under RTNL.  This is needed if received packets may be
1492 *      forwarded to another interface.
1493 */
1494void dev_disable_lro(struct net_device *dev)
1495{
1496        struct net_device *lower_dev;
1497        struct list_head *iter;
1498
1499        dev->wanted_features &= ~NETIF_F_LRO;
1500        netdev_update_features(dev);
1501
1502        if (unlikely(dev->features & NETIF_F_LRO))
1503                netdev_WARN(dev, "failed to disable LRO!\n");
1504
1505        netdev_for_each_lower_dev(dev, lower_dev, iter)
1506                dev_disable_lro(lower_dev);
1507}
1508EXPORT_SYMBOL(dev_disable_lro);
1509
1510static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1511                                   struct net_device *dev)
1512{
1513        struct netdev_notifier_info info;
1514
1515        netdev_notifier_info_init(&info, dev);
1516        return nb->notifier_call(nb, val, &info);
1517}
1518
1519static int dev_boot_phase = 1;
1520
1521/**
1522 *      register_netdevice_notifier - register a network notifier block
1523 *      @nb: notifier
1524 *
1525 *      Register a notifier to be called when network device events occur.
1526 *      The notifier passed is linked into the kernel structures and must
1527 *      not be reused until it has been unregistered. A negative errno code
1528 *      is returned on a failure.
1529 *
1530 *      When registered all registration and up events are replayed
1531 *      to the new notifier to allow device to have a race free
1532 *      view of the network device list.
1533 */
1534
1535int register_netdevice_notifier(struct notifier_block *nb)
1536{
1537        struct net_device *dev;
1538        struct net_device *last;
1539        struct net *net;
1540        int err;
1541
1542        rtnl_lock();
1543        err = raw_notifier_chain_register(&netdev_chain, nb);
1544        if (err)
1545                goto unlock;
1546        if (dev_boot_phase)
1547                goto unlock;
1548        for_each_net(net) {
1549                for_each_netdev(net, dev) {
1550                        err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1551                        err = notifier_to_errno(err);
1552                        if (err)
1553                                goto rollback;
1554
1555                        if (!(dev->flags & IFF_UP))
1556                                continue;
1557
1558                        call_netdevice_notifier(nb, NETDEV_UP, dev);
1559                }
1560        }
1561
1562unlock:
1563        rtnl_unlock();
1564        return err;
1565
1566rollback:
1567        last = dev;
1568        for_each_net(net) {
1569                for_each_netdev(net, dev) {
1570                        if (dev == last)
1571                                goto outroll;
1572
1573                        if (dev->flags & IFF_UP) {
1574                                call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1575                                                        dev);
1576                                call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1577                        }
1578                        call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1579                }
1580        }
1581
1582outroll:
1583        raw_notifier_chain_unregister(&netdev_chain, nb);
1584        goto unlock;
1585}
1586EXPORT_SYMBOL(register_netdevice_notifier);
1587
1588/**
1589 *      unregister_netdevice_notifier - unregister a network notifier block
1590 *      @nb: notifier
1591 *
1592 *      Unregister a notifier previously registered by
1593 *      register_netdevice_notifier(). The notifier is unlinked into the
1594 *      kernel structures and may then be reused. A negative errno code
1595 *      is returned on a failure.
1596 *
1597 *      After unregistering unregister and down device events are synthesized
1598 *      for all devices on the device list to the removed notifier to remove
1599 *      the need for special case cleanup code.
1600 */
1601
1602int unregister_netdevice_notifier(struct notifier_block *nb)
1603{
1604        struct net_device *dev;
1605        struct net *net;
1606        int err;
1607
1608        rtnl_lock();
1609        err = raw_notifier_chain_unregister(&netdev_chain, nb);
1610        if (err)
1611                goto unlock;
1612
1613        for_each_net(net) {
1614                for_each_netdev(net, dev) {
1615                        if (dev->flags & IFF_UP) {
1616                                call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1617                                                        dev);
1618                                call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1619                        }
1620                        call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1621                }
1622        }
1623unlock:
1624        rtnl_unlock();
1625        return err;
1626}
1627EXPORT_SYMBOL(unregister_netdevice_notifier);
1628
1629/**
1630 *      call_netdevice_notifiers_info - call all network notifier blocks
1631 *      @val: value passed unmodified to notifier function
1632 *      @dev: net_device pointer passed unmodified to notifier function
1633 *      @info: notifier information data
1634 *
1635 *      Call all network notifier blocks.  Parameters and return value
1636 *      are as for raw_notifier_call_chain().
1637 */
1638
1639static int call_netdevice_notifiers_info(unsigned long val,
1640                                         struct net_device *dev,
1641                                         struct netdev_notifier_info *info)
1642{
1643        ASSERT_RTNL();
1644        netdev_notifier_info_init(info, dev);
1645        return raw_notifier_call_chain(&netdev_chain, val, info);
1646}
1647
1648/**
1649 *      call_netdevice_notifiers - call all network notifier blocks
1650 *      @val: value passed unmodified to notifier function
1651 *      @dev: net_device pointer passed unmodified to notifier function
1652 *
1653 *      Call all network notifier blocks.  Parameters and return value
1654 *      are as for raw_notifier_call_chain().
1655 */
1656
1657int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1658{
1659        struct netdev_notifier_info info;
1660
1661        return call_netdevice_notifiers_info(val, dev, &info);
1662}
1663EXPORT_SYMBOL(call_netdevice_notifiers);
1664
1665#ifdef CONFIG_NET_INGRESS
1666static struct static_key ingress_needed __read_mostly;
1667
1668void net_inc_ingress_queue(void)
1669{
1670        static_key_slow_inc(&ingress_needed);
1671}
1672EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1673
1674void net_dec_ingress_queue(void)
1675{
1676        static_key_slow_dec(&ingress_needed);
1677}
1678EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1679#endif
1680
1681#ifdef CONFIG_NET_EGRESS
1682static struct static_key egress_needed __read_mostly;
1683
1684void net_inc_egress_queue(void)
1685{
1686        static_key_slow_inc(&egress_needed);
1687}
1688EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1689
1690void net_dec_egress_queue(void)
1691{
1692        static_key_slow_dec(&egress_needed);
1693}
1694EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1695#endif
1696
1697static struct static_key netstamp_needed __read_mostly;
1698#ifdef HAVE_JUMP_LABEL
1699/* We are not allowed to call static_key_slow_dec() from irq context
1700 * If net_disable_timestamp() is called from irq context, defer the
1701 * static_key_slow_dec() calls.
1702 */
1703static atomic_t netstamp_needed_deferred;
1704#endif
1705
1706void net_enable_timestamp(void)
1707{
1708#ifdef HAVE_JUMP_LABEL
1709        int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1710
1711        if (deferred) {
1712                while (--deferred)
1713                        static_key_slow_dec(&netstamp_needed);
1714                return;
1715        }
1716#endif
1717        static_key_slow_inc(&netstamp_needed);
1718}
1719EXPORT_SYMBOL(net_enable_timestamp);
1720
1721void net_disable_timestamp(void)
1722{
1723#ifdef HAVE_JUMP_LABEL
1724        if (in_interrupt()) {
1725                atomic_inc(&netstamp_needed_deferred);
1726                return;
1727        }
1728#endif
1729        static_key_slow_dec(&netstamp_needed);
1730}
1731EXPORT_SYMBOL(net_disable_timestamp);
1732
1733static inline void net_timestamp_set(struct sk_buff *skb)
1734{
1735        skb->tstamp.tv64 = 0;
1736        if (static_key_false(&netstamp_needed))
1737                __net_timestamp(skb);
1738}
1739
1740#define net_timestamp_check(COND, SKB)                  \
1741        if (static_key_false(&netstamp_needed)) {               \
1742                if ((COND) && !(SKB)->tstamp.tv64)      \
1743                        __net_timestamp(SKB);           \
1744        }                                               \
1745
1746bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1747{
1748        unsigned int len;
1749
1750        if (!(dev->flags & IFF_UP))
1751                return false;
1752
1753        len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1754        if (skb->len <= len)
1755                return true;
1756
1757        /* if TSO is enabled, we don't care about the length as the packet
1758         * could be forwarded without being segmented before
1759         */
1760        if (skb_is_gso(skb))
1761                return true;
1762
1763        return false;
1764}
1765EXPORT_SYMBOL_GPL(is_skb_forwardable);
1766
1767int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1768{
1769        int ret = ____dev_forward_skb(dev, skb);
1770
1771        if (likely(!ret)) {
1772                skb->protocol = eth_type_trans(skb, dev);
1773                skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1774        }
1775
1776        return ret;
1777}
1778EXPORT_SYMBOL_GPL(__dev_forward_skb);
1779
1780/**
1781 * dev_forward_skb - loopback an skb to another netif
1782 *
1783 * @dev: destination network device
1784 * @skb: buffer to forward
1785 *
1786 * return values:
1787 *      NET_RX_SUCCESS  (no congestion)
1788 *      NET_RX_DROP     (packet was dropped, but freed)
1789 *
1790 * dev_forward_skb can be used for injecting an skb from the
1791 * start_xmit function of one device into the receive queue
1792 * of another device.
1793 *
1794 * The receiving device may be in another namespace, so
1795 * we have to clear all information in the skb that could
1796 * impact namespace isolation.
1797 */
1798int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1799{
1800        return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1801}
1802EXPORT_SYMBOL_GPL(dev_forward_skb);
1803
1804static inline int deliver_skb(struct sk_buff *skb,
1805                              struct packet_type *pt_prev,
1806                              struct net_device *orig_dev)
1807{
1808        if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1809                return -ENOMEM;
1810        atomic_inc(&skb->users);
1811        return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1812}
1813
1814static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1815                                          struct packet_type **pt,
1816                                          struct net_device *orig_dev,
1817                                          __be16 type,
1818                                          struct list_head *ptype_list)
1819{
1820        struct packet_type *ptype, *pt_prev = *pt;
1821
1822        list_for_each_entry_rcu(ptype, ptype_list, list) {
1823                if (ptype->type != type)
1824                        continue;
1825                if (pt_prev)
1826                        deliver_skb(skb, pt_prev, orig_dev);
1827                pt_prev = ptype;
1828        }
1829        *pt = pt_prev;
1830}
1831
1832static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1833{
1834        if (!ptype->af_packet_priv || !skb->sk)
1835                return false;
1836
1837        if (ptype->id_match)
1838                return ptype->id_match(ptype, skb->sk);
1839        else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1840                return true;
1841
1842        return false;
1843}
1844
1845/*
1846 *      Support routine. Sends outgoing frames to any network
1847 *      taps currently in use.
1848 */
1849
1850void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1851{
1852        struct packet_type *ptype;
1853        struct sk_buff *skb2 = NULL;
1854        struct packet_type *pt_prev = NULL;
1855        struct list_head *ptype_list = &ptype_all;
1856
1857        rcu_read_lock();
1858again:
1859        list_for_each_entry_rcu(ptype, ptype_list, list) {
1860                /* Never send packets back to the socket
1861                 * they originated from - MvS (miquels@drinkel.ow.org)
1862                 */
1863                if (skb_loop_sk(ptype, skb))
1864                        continue;
1865
1866                if (pt_prev) {
1867                        deliver_skb(skb2, pt_prev, skb->dev);
1868                        pt_prev = ptype;
1869                        continue;
1870                }
1871
1872                /* need to clone skb, done only once */
1873                skb2 = skb_clone(skb, GFP_ATOMIC);
1874                if (!skb2)
1875                        goto out_unlock;
1876
1877                net_timestamp_set(skb2);
1878
1879                /* skb->nh should be correctly
1880                 * set by sender, so that the second statement is
1881                 * just protection against buggy protocols.
1882                 */
1883                skb_reset_mac_header(skb2);
1884
1885                if (skb_network_header(skb2) < skb2->data ||
1886                    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1887                        net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1888                                             ntohs(skb2->protocol),
1889                                             dev->name);
1890                        skb_reset_network_header(skb2);
1891                }
1892
1893                skb2->transport_header = skb2->network_header;
1894                skb2->pkt_type = PACKET_OUTGOING;
1895                pt_prev = ptype;
1896        }
1897
1898        if (ptype_list == &ptype_all) {
1899                ptype_list = &dev->ptype_all;
1900                goto again;
1901        }
1902out_unlock:
1903        if (pt_prev)
1904                pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1905        rcu_read_unlock();
1906}
1907EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1908
1909/**
1910 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1911 * @dev: Network device
1912 * @txq: number of queues available
1913 *
1914 * If real_num_tx_queues is changed the tc mappings may no longer be
1915 * valid. To resolve this verify the tc mapping remains valid and if
1916 * not NULL the mapping. With no priorities mapping to this
1917 * offset/count pair it will no longer be used. In the worst case TC0
1918 * is invalid nothing can be done so disable priority mappings. If is
1919 * expected that drivers will fix this mapping if they can before
1920 * calling netif_set_real_num_tx_queues.
1921 */
1922static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1923{
1924        int i;
1925        struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1926
1927        /* If TC0 is invalidated disable TC mapping */
1928        if (tc->offset + tc->count > txq) {
1929                pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1930                dev->num_tc = 0;
1931                return;
1932        }
1933
1934        /* Invalidated prio to tc mappings set to TC0 */
1935        for (i = 1; i < TC_BITMASK + 1; i++) {
1936                int q = netdev_get_prio_tc_map(dev, i);
1937
1938                tc = &dev->tc_to_txq[q];
1939                if (tc->offset + tc->count > txq) {
1940                        pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1941                                i, q);
1942                        netdev_set_prio_tc_map(dev, i, 0);
1943                }
1944        }
1945}
1946
1947#ifdef CONFIG_XPS
1948static DEFINE_MUTEX(xps_map_mutex);
1949#define xmap_dereference(P)             \
1950        rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1951
1952static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1953                                        int cpu, u16 index)
1954{
1955        struct xps_map *map = NULL;
1956        int pos;
1957
1958        if (dev_maps)
1959                map = xmap_dereference(dev_maps->cpu_map[cpu]);
1960
1961        for (pos = 0; map && pos < map->len; pos++) {
1962                if (map->queues[pos] == index) {
1963                        if (map->len > 1) {
1964                                map->queues[pos] = map->queues[--map->len];
1965                        } else {
1966                                RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1967                                kfree_rcu(map, rcu);
1968                                map = NULL;
1969                        }
1970                        break;
1971                }
1972        }
1973
1974        return map;
1975}
1976
1977static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1978{
1979        struct xps_dev_maps *dev_maps;
1980        int cpu, i;
1981        bool active = false;
1982
1983        mutex_lock(&xps_map_mutex);
1984        dev_maps = xmap_dereference(dev->xps_maps);
1985
1986        if (!dev_maps)
1987                goto out_no_maps;
1988
1989        for_each_possible_cpu(cpu) {
1990                for (i = index; i < dev->num_tx_queues; i++) {
1991                        if (!remove_xps_queue(dev_maps, cpu, i))
1992                                break;
1993                }
1994                if (i == dev->num_tx_queues)
1995                        active = true;
1996        }
1997
1998        if (!active) {
1999                RCU_INIT_POINTER(dev->xps_maps, NULL);
2000                kfree_rcu(dev_maps, rcu);
2001        }
2002
2003        for (i = index; i < dev->num_tx_queues; i++)
2004                netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2005                                             NUMA_NO_NODE);
2006
2007out_no_maps:
2008        mutex_unlock(&xps_map_mutex);
2009}
2010
2011static struct xps_map *expand_xps_map(struct xps_map *map,
2012                                      int cpu, u16 index)
2013{
2014        struct xps_map *new_map;
2015        int alloc_len = XPS_MIN_MAP_ALLOC;
2016        int i, pos;
2017
2018        for (pos = 0; map && pos < map->len; pos++) {
2019                if (map->queues[pos] != index)
2020                        continue;
2021                return map;
2022        }
2023
2024        /* Need to add queue to this CPU's existing map */
2025        if (map) {
2026                if (pos < map->alloc_len)
2027                        return map;
2028
2029                alloc_len = map->alloc_len * 2;
2030        }
2031
2032        /* Need to allocate new map to store queue on this CPU's map */
2033        new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2034                               cpu_to_node(cpu));
2035        if (!new_map)
2036                return NULL;
2037
2038        for (i = 0; i < pos; i++)
2039                new_map->queues[i] = map->queues[i];
2040        new_map->alloc_len = alloc_len;
2041        new_map->len = pos;
2042
2043        return new_map;
2044}
2045
2046int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2047                        u16 index)
2048{
2049        struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2050        struct xps_map *map, *new_map;
2051        int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
2052        int cpu, numa_node_id = -2;
2053        bool active = false;
2054
2055        mutex_lock(&xps_map_mutex);
2056
2057        dev_maps = xmap_dereference(dev->xps_maps);
2058
2059        /* allocate memory for queue storage */
2060        for_each_online_cpu(cpu) {
2061                if (!cpumask_test_cpu(cpu, mask))
2062                        continue;
2063
2064                if (!new_dev_maps)
2065                        new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2066                if (!new_dev_maps) {
2067                        mutex_unlock(&xps_map_mutex);
2068                        return -ENOMEM;
2069                }
2070
2071                map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2072                                 NULL;
2073
2074                map = expand_xps_map(map, cpu, index);
2075                if (!map)
2076                        goto error;
2077
2078                RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2079        }
2080
2081        if (!new_dev_maps)
2082                goto out_no_new_maps;
2083
2084        for_each_possible_cpu(cpu) {
2085                if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2086                        /* add queue to CPU maps */
2087                        int pos = 0;
2088
2089                        map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2090                        while ((pos < map->len) && (map->queues[pos] != index))
2091                                pos++;
2092
2093                        if (pos == map->len)
2094                                map->queues[map->len++] = index;
2095#ifdef CONFIG_NUMA
2096                        if (numa_node_id == -2)
2097                                numa_node_id = cpu_to_node(cpu);
2098                        else if (numa_node_id != cpu_to_node(cpu))
2099                                numa_node_id = -1;
2100#endif
2101                } else if (dev_maps) {
2102                        /* fill in the new device map from the old device map */
2103                        map = xmap_dereference(dev_maps->cpu_map[cpu]);
2104                        RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2105                }
2106
2107        }
2108
2109        rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2110
2111        /* Cleanup old maps */
2112        if (dev_maps) {
2113                for_each_possible_cpu(cpu) {
2114                        new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2115                        map = xmap_dereference(dev_maps->cpu_map[cpu]);
2116                        if (map && map != new_map)
2117                                kfree_rcu(map, rcu);
2118                }
2119
2120                kfree_rcu(dev_maps, rcu);
2121        }
2122
2123        dev_maps = new_dev_maps;
2124        active = true;
2125
2126out_no_new_maps:
2127        /* update Tx queue numa node */
2128        netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2129                                     (numa_node_id >= 0) ? numa_node_id :
2130                                     NUMA_NO_NODE);
2131
2132        if (!dev_maps)
2133                goto out_no_maps;
2134
2135        /* removes queue from unused CPUs */
2136        for_each_possible_cpu(cpu) {
2137                if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2138                        continue;
2139
2140                if (remove_xps_queue(dev_maps, cpu, index))
2141                        active = true;
2142        }
2143
2144        /* free map if not active */
2145        if (!active) {
2146                RCU_INIT_POINTER(dev->xps_maps, NULL);
2147                kfree_rcu(dev_maps, rcu);
2148        }
2149
2150out_no_maps:
2151        mutex_unlock(&xps_map_mutex);
2152
2153        return 0;
2154error:
2155        /* remove any maps that we added */
2156        for_each_possible_cpu(cpu) {
2157                new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2158                map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2159                                 NULL;
2160                if (new_map && new_map != map)
2161                        kfree(new_map);
2162        }
2163
2164        mutex_unlock(&xps_map_mutex);
2165
2166        kfree(new_dev_maps);
2167        return -ENOMEM;
2168}
2169EXPORT_SYMBOL(netif_set_xps_queue);
2170
2171#endif
2172/*
2173 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2174 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2175 */
2176int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2177{
2178        int rc;
2179
2180        if (txq < 1 || txq > dev->num_tx_queues)
2181                return -EINVAL;
2182
2183        if (dev->reg_state == NETREG_REGISTERED ||
2184            dev->reg_state == NETREG_UNREGISTERING) {
2185                ASSERT_RTNL();
2186
2187                rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2188                                                  txq);
2189                if (rc)
2190                        return rc;
2191
2192                if (dev->num_tc)
2193                        netif_setup_tc(dev, txq);
2194
2195                if (txq < dev->real_num_tx_queues) {
2196                        qdisc_reset_all_tx_gt(dev, txq);
2197#ifdef CONFIG_XPS
2198                        netif_reset_xps_queues_gt(dev, txq);
2199#endif
2200                }
2201        }
2202
2203        dev->real_num_tx_queues = txq;
2204        return 0;
2205}
2206EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2207
2208#ifdef CONFIG_SYSFS
2209/**
2210 *      netif_set_real_num_rx_queues - set actual number of RX queues used
2211 *      @dev: Network device
2212 *      @rxq: Actual number of RX queues
2213 *
2214 *      This must be called either with the rtnl_lock held or before
2215 *      registration of the net device.  Returns 0 on success, or a
2216 *      negative error code.  If called before registration, it always
2217 *      succeeds.
2218 */
2219int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2220{
2221        int rc;
2222
2223        if (rxq < 1 || rxq > dev->num_rx_queues)
2224                return -EINVAL;
2225
2226        if (dev->reg_state == NETREG_REGISTERED) {
2227                ASSERT_RTNL();
2228
2229                rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2230                                                  rxq);
2231                if (rc)
2232                        return rc;
2233        }
2234
2235        dev->real_num_rx_queues = rxq;
2236        return 0;
2237}
2238EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2239#endif
2240
2241/**
2242 * netif_get_num_default_rss_queues - default number of RSS queues
2243 *
2244 * This routine should set an upper limit on the number of RSS queues
2245 * used by default by multiqueue devices.
2246 */
2247int netif_get_num_default_rss_queues(void)
2248{
2249        return is_kdump_kernel() ?
2250                1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2251}
2252EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2253
2254static void __netif_reschedule(struct Qdisc *q)
2255{
2256        struct softnet_data *sd;
2257        unsigned long flags;
2258
2259        local_irq_save(flags);
2260        sd = this_cpu_ptr(&softnet_data);
2261        q->next_sched = NULL;
2262        *sd->output_queue_tailp = q;
2263        sd->output_queue_tailp = &q->next_sched;
2264        raise_softirq_irqoff(NET_TX_SOFTIRQ);
2265        local_irq_restore(flags);
2266}
2267
2268void __netif_schedule(struct Qdisc *q)
2269{
2270        if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2271                __netif_reschedule(q);
2272}
2273EXPORT_SYMBOL(__netif_schedule);
2274
2275struct dev_kfree_skb_cb {
2276        enum skb_free_reason reason;
2277};
2278
2279static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2280{
2281        return (struct dev_kfree_skb_cb *)skb->cb;
2282}
2283
2284void netif_schedule_queue(struct netdev_queue *txq)
2285{
2286        rcu_read_lock();
2287        if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2288                struct Qdisc *q = rcu_dereference(txq->qdisc);
2289
2290                __netif_schedule(q);
2291        }
2292        rcu_read_unlock();
2293}
2294EXPORT_SYMBOL(netif_schedule_queue);
2295
2296/**
2297 *      netif_wake_subqueue - allow sending packets on subqueue
2298 *      @dev: network device
2299 *      @queue_index: sub queue index
2300 *
2301 * Resume individual transmit queue of a device with multiple transmit queues.
2302 */
2303void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2304{
2305        struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2306
2307        if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2308                struct Qdisc *q;
2309
2310                rcu_read_lock();
2311                q = rcu_dereference(txq->qdisc);
2312                __netif_schedule(q);
2313                rcu_read_unlock();
2314        }
2315}
2316EXPORT_SYMBOL(netif_wake_subqueue);
2317
2318void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2319{
2320        if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2321                struct Qdisc *q;
2322
2323                rcu_read_lock();
2324                q = rcu_dereference(dev_queue->qdisc);
2325                __netif_schedule(q);
2326                rcu_read_unlock();
2327        }
2328}
2329EXPORT_SYMBOL(netif_tx_wake_queue);
2330
2331void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2332{
2333        unsigned long flags;
2334
2335        if (likely(atomic_read(&skb->users) == 1)) {
2336                smp_rmb();
2337                atomic_set(&skb->users, 0);
2338        } else if (likely(!atomic_dec_and_test(&skb->users))) {
2339                return;
2340        }
2341        get_kfree_skb_cb(skb)->reason = reason;
2342        local_irq_save(flags);
2343        skb->next = __this_cpu_read(softnet_data.completion_queue);
2344        __this_cpu_write(softnet_data.completion_queue, skb);
2345        raise_softirq_irqoff(NET_TX_SOFTIRQ);
2346        local_irq_restore(flags);
2347}
2348EXPORT_SYMBOL(__dev_kfree_skb_irq);
2349
2350void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2351{
2352        if (in_irq() || irqs_disabled())
2353                __dev_kfree_skb_irq(skb, reason);
2354        else
2355                dev_kfree_skb(skb);
2356}
2357EXPORT_SYMBOL(__dev_kfree_skb_any);
2358
2359
2360/**
2361 * netif_device_detach - mark device as removed
2362 * @dev: network device
2363 *
2364 * Mark device as removed from system and therefore no longer available.
2365 */
2366void netif_device_detach(struct net_device *dev)
2367{
2368        if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2369            netif_running(dev)) {
2370                netif_tx_stop_all_queues(dev);
2371        }
2372}
2373EXPORT_SYMBOL(netif_device_detach);
2374
2375/**
2376 * netif_device_attach - mark device as attached
2377 * @dev: network device
2378 *
2379 * Mark device as attached from system and restart if needed.
2380 */
2381void netif_device_attach(struct net_device *dev)
2382{
2383        if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2384            netif_running(dev)) {
2385                netif_tx_wake_all_queues(dev);
2386                __netdev_watchdog_up(dev);
2387        }
2388}
2389EXPORT_SYMBOL(netif_device_attach);
2390
2391/*
2392 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2393 * to be used as a distribution range.
2394 */
2395u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2396                  unsigned int num_tx_queues)
2397{
2398        u32 hash;
2399        u16 qoffset = 0;
2400        u16 qcount = num_tx_queues;
2401
2402        if (skb_rx_queue_recorded(skb)) {
2403                hash = skb_get_rx_queue(skb);
2404                while (unlikely(hash >= num_tx_queues))
2405                        hash -= num_tx_queues;
2406                return hash;
2407        }
2408
2409        if (dev->num_tc) {
2410                u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2411                qoffset = dev->tc_to_txq[tc].offset;
2412                qcount = dev->tc_to_txq[tc].count;
2413        }
2414
2415        return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2416}
2417EXPORT_SYMBOL(__skb_tx_hash);
2418
2419static void skb_warn_bad_offload(const struct sk_buff *skb)
2420{
2421        static const netdev_features_t null_features;
2422        struct net_device *dev = skb->dev;
2423        const char *name = "";
2424
2425        if (!net_ratelimit())
2426                return;
2427
2428        if (dev) {
2429                if (dev->dev.parent)
2430                        name = dev_driver_string(dev->dev.parent);
2431                else
2432                        name = netdev_name(dev);
2433        }
2434        WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2435             "gso_type=%d ip_summed=%d\n",
2436             name, dev ? &dev->features : &null_features,
2437             skb->sk ? &skb->sk->sk_route_caps : &null_features,
2438             skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2439             skb_shinfo(skb)->gso_type, skb->ip_summed);
2440}
2441
2442/*
2443 * Invalidate hardware checksum when packet is to be mangled, and
2444 * complete checksum manually on outgoing path.
2445 */
2446int skb_checksum_help(struct sk_buff *skb)
2447{
2448        __wsum csum;
2449        int ret = 0, offset;
2450
2451        if (skb->ip_summed == CHECKSUM_COMPLETE)
2452                goto out_set_summed;
2453
2454        if (unlikely(skb_shinfo(skb)->gso_size)) {
2455                skb_warn_bad_offload(skb);
2456                return -EINVAL;
2457        }
2458
2459        /* Before computing a checksum, we should make sure no frag could
2460         * be modified by an external entity : checksum could be wrong.
2461         */
2462        if (skb_has_shared_frag(skb)) {
2463                ret = __skb_linearize(skb);
2464                if (ret)
2465                        goto out;
2466        }
2467
2468        offset = skb_checksum_start_offset(skb);
2469        BUG_ON(offset >= skb_headlen(skb));
2470        csum = skb_checksum(skb, offset, skb->len - offset, 0);
2471
2472        offset += skb->csum_offset;
2473        BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2474
2475        if (skb_cloned(skb) &&
2476            !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2477                ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2478                if (ret)
2479                        goto out;
2480        }
2481
2482        *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2483out_set_summed:
2484        skb->ip_summed = CHECKSUM_NONE;
2485out:
2486        return ret;
2487}
2488EXPORT_SYMBOL(skb_checksum_help);
2489
2490/* skb_csum_offload_check - Driver helper function to determine if a device
2491 * with limited checksum offload capabilities is able to offload the checksum
2492 * for a given packet.
2493 *
2494 * Arguments:
2495 *   skb - sk_buff for the packet in question
2496 *   spec - contains the description of what device can offload
2497 *   csum_encapped - returns true if the checksum being offloaded is
2498 *            encpasulated. That is it is checksum for the transport header
2499 *            in the inner headers.
2500 *   checksum_help - when set indicates that helper function should
2501 *            call skb_checksum_help if offload checks fail
2502 *
2503 * Returns:
2504 *   true: Packet has passed the checksum checks and should be offloadable to
2505 *         the device (a driver may still need to check for additional
2506 *         restrictions of its device)
2507 *   false: Checksum is not offloadable. If checksum_help was set then
2508 *         skb_checksum_help was called to resolve checksum for non-GSO
2509 *         packets and when IP protocol is not SCTP
2510 */
2511bool __skb_csum_offload_chk(struct sk_buff *skb,
2512                            const struct skb_csum_offl_spec *spec,
2513                            bool *csum_encapped,
2514                            bool csum_help)
2515{
2516        struct iphdr *iph;
2517        struct ipv6hdr *ipv6;
2518        void *nhdr;
2519        int protocol;
2520        u8 ip_proto;
2521
2522        if (skb->protocol == htons(ETH_P_8021Q) ||
2523            skb->protocol == htons(ETH_P_8021AD)) {
2524                if (!spec->vlan_okay)
2525                        goto need_help;
2526        }
2527
2528        /* We check whether the checksum refers to a transport layer checksum in
2529         * the outermost header or an encapsulated transport layer checksum that
2530         * corresponds to the inner headers of the skb. If the checksum is for
2531         * something else in the packet we need help.
2532         */
2533        if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) {
2534                /* Non-encapsulated checksum */
2535                protocol = eproto_to_ipproto(vlan_get_protocol(skb));
2536                nhdr = skb_network_header(skb);
2537                *csum_encapped = false;
2538                if (spec->no_not_encapped)
2539                        goto need_help;
2540        } else if (skb->encapsulation && spec->encap_okay &&
2541                   skb_checksum_start_offset(skb) ==
2542                   skb_inner_transport_offset(skb)) {
2543                /* Encapsulated checksum */
2544                *csum_encapped = true;
2545                switch (skb->inner_protocol_type) {
2546                case ENCAP_TYPE_ETHER:
2547                        protocol = eproto_to_ipproto(skb->inner_protocol);
2548                        break;
2549                case ENCAP_TYPE_IPPROTO:
2550                        protocol = skb->inner_protocol;
2551                        break;
2552                }
2553                nhdr = skb_inner_network_header(skb);
2554        } else {
2555                goto need_help;
2556        }
2557
2558        switch (protocol) {
2559        case IPPROTO_IP:
2560                if (!spec->ipv4_okay)
2561                        goto need_help;
2562                iph = nhdr;
2563                ip_proto = iph->protocol;
2564                if (iph->ihl != 5 && !spec->ip_options_okay)
2565                        goto need_help;
2566                break;
2567        case IPPROTO_IPV6:
2568                if (!spec->ipv6_okay)
2569                        goto need_help;
2570                if (spec->no_encapped_ipv6 && *csum_encapped)
2571                        goto need_help;
2572                ipv6 = nhdr;
2573                nhdr += sizeof(*ipv6);
2574                ip_proto = ipv6->nexthdr;
2575                break;
2576        default:
2577                goto need_help;
2578        }
2579
2580ip_proto_again:
2581        switch (ip_proto) {
2582        case IPPROTO_TCP:
2583                if (!spec->tcp_okay ||
2584                    skb->csum_offset != offsetof(struct tcphdr, check))
2585                        goto need_help;
2586                break;
2587        case IPPROTO_UDP:
2588                if (!spec->udp_okay ||
2589                    skb->csum_offset != offsetof(struct udphdr, check))
2590                        goto need_help;
2591                break;
2592        case IPPROTO_SCTP:
2593                if (!spec->sctp_okay ||
2594                    skb->csum_offset != offsetof(struct sctphdr, checksum))
2595                        goto cant_help;
2596                break;
2597        case NEXTHDR_HOP:
2598        case NEXTHDR_ROUTING:
2599        case NEXTHDR_DEST: {
2600                u8 *opthdr = nhdr;
2601
2602                if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay)
2603                        goto need_help;
2604
2605                ip_proto = opthdr[0];
2606                nhdr += (opthdr[1] + 1) << 3;
2607
2608                goto ip_proto_again;
2609        }
2610        default:
2611                goto need_help;
2612        }
2613
2614        /* Passed the tests for offloading checksum */
2615        return true;
2616
2617need_help:
2618        if (csum_help && !skb_shinfo(skb)->gso_size)
2619                skb_checksum_help(skb);
2620cant_help:
2621        return false;
2622}
2623EXPORT_SYMBOL(__skb_csum_offload_chk);
2624
2625__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2626{
2627        __be16 type = skb->protocol;
2628
2629        /* Tunnel gso handlers can set protocol to ethernet. */
2630        if (type == htons(ETH_P_TEB)) {
2631                struct ethhdr *eth;
2632
2633                if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2634                        return 0;
2635
2636                eth = (struct ethhdr *)skb_mac_header(skb);
2637                type = eth->h_proto;
2638        }
2639
2640        return __vlan_get_protocol(skb, type, depth);
2641}
2642
2643/**
2644 *      skb_mac_gso_segment - mac layer segmentation handler.
2645 *      @skb: buffer to segment
2646 *      @features: features for the output path (see dev->features)
2647 */
2648struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2649                                    netdev_features_t features)
2650{
2651        struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2652        struct packet_offload *ptype;
2653        int vlan_depth = skb->mac_len;
2654        __be16 type = skb_network_protocol(skb, &vlan_depth);
2655
2656        if (unlikely(!type))
2657                return ERR_PTR(-EINVAL);
2658
2659        __skb_pull(skb, vlan_depth);
2660
2661        rcu_read_lock();
2662        list_for_each_entry_rcu(ptype, &offload_base, list) {
2663                if (ptype->type == type && ptype->callbacks.gso_segment) {
2664                        segs = ptype->callbacks.gso_segment(skb, features);
2665                        break;
2666                }
2667        }
2668        rcu_read_unlock();
2669
2670        __skb_push(skb, skb->data - skb_mac_header(skb));
2671
2672        return segs;
2673}
2674EXPORT_SYMBOL(skb_mac_gso_segment);
2675
2676
2677/* openvswitch calls this on rx path, so we need a different check.
2678 */
2679static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2680{
2681        if (tx_path)
2682                return skb->ip_summed != CHECKSUM_PARTIAL;
2683        else
2684                return skb->ip_summed == CHECKSUM_NONE;
2685}
2686
2687/**
2688 *      __skb_gso_segment - Perform segmentation on skb.
2689 *      @skb: buffer to segment
2690 *      @features: features for the output path (see dev->features)
2691 *      @tx_path: whether it is called in TX path
2692 *
2693 *      This function segments the given skb and returns a list of segments.
2694 *
2695 *      It may return NULL if the skb requires no segmentation.  This is
2696 *      only possible when GSO is used for verifying header integrity.
2697 *
2698 *      Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2699 */
2700struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2701                                  netdev_features_t features, bool tx_path)
2702{
2703        if (unlikely(skb_needs_check(skb, tx_path))) {
2704                int err;
2705
2706                skb_warn_bad_offload(skb);
2707
2708                err = skb_cow_head(skb, 0);
2709                if (err < 0)
2710                        return ERR_PTR(err);
2711        }
2712
2713        /* Only report GSO partial support if it will enable us to
2714         * support segmentation on this frame without needing additional
2715         * work.
2716         */
2717        if (features & NETIF_F_GSO_PARTIAL) {
2718                netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2719                struct net_device *dev = skb->dev;
2720
2721                partial_features |= dev->features & dev->gso_partial_features;
2722                if (!skb_gso_ok(skb, features | partial_features))
2723                        features &= ~NETIF_F_GSO_PARTIAL;
2724        }
2725
2726        BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2727                     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2728
2729        SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2730        SKB_GSO_CB(skb)->encap_level = 0;
2731
2732        skb_reset_mac_header(skb);
2733        skb_reset_mac_len(skb);
2734
2735        return skb_mac_gso_segment(skb, features);
2736}
2737EXPORT_SYMBOL(__skb_gso_segment);
2738
2739/* Take action when hardware reception checksum errors are detected. */
2740#ifdef CONFIG_BUG
2741void netdev_rx_csum_fault(struct net_device *dev)
2742{
2743        if (net_ratelimit()) {
2744                pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2745                dump_stack();
2746        }
2747}
2748EXPORT_SYMBOL(netdev_rx_csum_fault);
2749#endif
2750
2751/* Actually, we should eliminate this check as soon as we know, that:
2752 * 1. IOMMU is present and allows to map all the memory.
2753 * 2. No high memory really exists on this machine.
2754 */
2755
2756static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2757{
2758#ifdef CONFIG_HIGHMEM
2759        int i;
2760        if (!(dev->features & NETIF_F_HIGHDMA)) {
2761                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2762                        skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2763                        if (PageHighMem(skb_frag_page(frag)))
2764                                return 1;
2765                }
2766        }
2767
2768        if (PCI_DMA_BUS_IS_PHYS) {
2769                struct device *pdev = dev->dev.parent;
2770
2771                if (!pdev)
2772                        return 0;
2773                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2774                        skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2775                        dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2776                        if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2777                                return 1;
2778                }
2779        }
2780#endif
2781        return 0;
2782}
2783
2784/* If MPLS offload request, verify we are testing hardware MPLS features
2785 * instead of standard features for the netdev.
2786 */
2787#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2788static netdev_features_t net_mpls_features(struct sk_buff *skb,
2789                                           netdev_features_t features,
2790                                           __be16 type)
2791{
2792        if (eth_p_mpls(type))
2793                features &= skb->dev->mpls_features;
2794
2795        return features;
2796}
2797#else
2798static netdev_features_t net_mpls_features(struct sk_buff *skb,
2799                                           netdev_features_t features,
2800                                           __be16 type)
2801{
2802        return features;
2803}
2804#endif
2805
2806static netdev_features_t harmonize_features(struct sk_buff *skb,
2807        netdev_features_t features)
2808{
2809        int tmp;
2810        __be16 type;
2811
2812        type = skb_network_protocol(skb, &tmp);
2813        features = net_mpls_features(skb, features, type);
2814
2815        if (skb->ip_summed != CHECKSUM_NONE &&
2816            !can_checksum_protocol(features, type)) {
2817                features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2818        } else if (illegal_highdma(skb->dev, skb)) {
2819                features &= ~NETIF_F_SG;
2820        }
2821
2822        return features;
2823}
2824
2825netdev_features_t passthru_features_check(struct sk_buff *skb,
2826                                          struct net_device *dev,
2827                                          netdev_features_t features)
2828{
2829        return features;
2830}
2831EXPORT_SYMBOL(passthru_features_check);
2832
2833static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2834                                             struct net_device *dev,
2835                                             netdev_features_t features)
2836{
2837        return vlan_features_check(skb, features);
2838}
2839
2840static netdev_features_t gso_features_check(const struct sk_buff *skb,
2841                                            struct net_device *dev,
2842                                            netdev_features_t features)
2843{
2844        u16 gso_segs = skb_shinfo(skb)->gso_segs;
2845
2846        if (gso_segs > dev->gso_max_segs)
2847                return features & ~NETIF_F_GSO_MASK;
2848
2849        /* Support for GSO partial features requires software
2850         * intervention before we can actually process the packets
2851         * so we need to strip support for any partial features now
2852         * and we can pull them back in after we have partially
2853         * segmented the frame.
2854         */
2855        if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2856                features &= ~dev->gso_partial_features;
2857
2858        /* Make sure to clear the IPv4 ID mangling feature if the
2859         * IPv4 header has the potential to be fragmented.
2860         */
2861        if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2862                struct iphdr *iph = skb->encapsulation ?
2863                                    inner_ip_hdr(skb) : ip_hdr(skb);
2864
2865                if (!(iph->frag_off & htons(IP_DF)))
2866                        features &= ~NETIF_F_TSO_MANGLEID;
2867        }
2868
2869        return features;
2870}
2871
2872netdev_features_t netif_skb_features(struct sk_buff *skb)
2873{
2874        struct net_device *dev = skb->dev;
2875        netdev_features_t features = dev->features;
2876
2877        if (skb_is_gso(skb))
2878                features = gso_features_check(skb, dev, features);
2879
2880        /* If encapsulation offload request, verify we are testing
2881         * hardware encapsulation features instead of standard
2882         * features for the netdev
2883         */
2884        if (skb->encapsulation)
2885                features &= dev->hw_enc_features;
2886
2887        if (skb_vlan_tagged(skb))
2888                features = netdev_intersect_features(features,
2889                                                     dev->vlan_features |
2890                                                     NETIF_F_HW_VLAN_CTAG_TX |
2891                                                     NETIF_F_HW_VLAN_STAG_TX);
2892
2893        if (dev->netdev_ops->ndo_features_check)
2894                features &= dev->netdev_ops->ndo_features_check(skb, dev,
2895                                                                features);
2896        else
2897                features &= dflt_features_check(skb, dev, features);
2898
2899        return harmonize_features(skb, features);
2900}
2901EXPORT_SYMBOL(netif_skb_features);
2902
2903static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2904                    struct netdev_queue *txq, bool more)
2905{
2906        unsigned int len;
2907        int rc;
2908
2909        if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2910                dev_queue_xmit_nit(skb, dev);
2911
2912        len = skb->len;
2913        trace_net_dev_start_xmit(skb, dev);
2914        rc = netdev_start_xmit(skb, dev, txq, more);
2915        trace_net_dev_xmit(skb, rc, dev, len);
2916
2917        return rc;
2918}
2919
2920struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2921                                    struct netdev_queue *txq, int *ret)
2922{
2923        struct sk_buff *skb = first;
2924        int rc = NETDEV_TX_OK;
2925
2926        while (skb) {
2927                struct sk_buff *next = skb->next;
2928
2929                skb->next = NULL;
2930                rc = xmit_one(skb, dev, txq, next != NULL);
2931                if (unlikely(!dev_xmit_complete(rc))) {
2932                        skb->next = next;
2933                        goto out;
2934                }
2935
2936                skb = next;
2937                if (netif_xmit_stopped(txq) && skb) {
2938                        rc = NETDEV_TX_BUSY;
2939                        break;
2940                }
2941        }
2942
2943out:
2944        *ret = rc;
2945        return skb;
2946}
2947
2948static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2949                                          netdev_features_t features)
2950{
2951        if (skb_vlan_tag_present(skb) &&
2952            !vlan_hw_offload_capable(features, skb->vlan_proto))
2953                skb = __vlan_hwaccel_push_inside(skb);
2954        return skb;
2955}
2956
2957static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2958{
2959        netdev_features_t features;
2960
2961        features = netif_skb_features(skb);
2962        skb = validate_xmit_vlan(skb, features);
2963        if (unlikely(!skb))
2964                goto out_null;
2965
2966        if (netif_needs_gso(skb, features)) {
2967                struct sk_buff *segs;
2968
2969                segs = skb_gso_segment(skb, features);
2970                if (IS_ERR(segs)) {
2971                        goto out_kfree_skb;
2972                } else if (segs) {
2973                        consume_skb(skb);
2974                        skb = segs;
2975                }
2976        } else {
2977                if (skb_needs_linearize(skb, features) &&
2978                    __skb_linearize(skb))
2979                        goto out_kfree_skb;
2980
2981                /* If packet is not checksummed and device does not
2982                 * support checksumming for this protocol, complete
2983                 * checksumming here.
2984                 */
2985                if (skb->ip_summed == CHECKSUM_PARTIAL) {
2986                        if (skb->encapsulation)
2987                                skb_set_inner_transport_header(skb,
2988                                                               skb_checksum_start_offset(skb));
2989                        else
2990                                skb_set_transport_header(skb,
2991                                                         skb_checksum_start_offset(skb));
2992                        if (!(features & NETIF_F_CSUM_MASK) &&
2993                            skb_checksum_help(skb))
2994                                goto out_kfree_skb;
2995                }
2996        }
2997
2998        return skb;
2999
3000out_kfree_skb:
3001        kfree_skb(skb);
3002out_null:
3003        atomic_long_inc(&dev->tx_dropped);
3004        return NULL;
3005}
3006
3007struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3008{
3009        struct sk_buff *next, *head = NULL, *tail;
3010
3011        for (; skb != NULL; skb = next) {
3012                next = skb->next;
3013                skb->next = NULL;
3014
3015                /* in case skb wont be segmented, point to itself */
3016                skb->prev = skb;
3017
3018                skb = validate_xmit_skb(skb, dev);
3019                if (!skb)
3020                        continue;
3021
3022                if (!head)
3023                        head = skb;
3024                else
3025                        tail->next = skb;
3026                /* If skb was segmented, skb->prev points to
3027                 * the last segment. If not, it still contains skb.
3028                 */
3029                tail = skb->prev;
3030        }
3031        return head;
3032}
3033EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3034
3035static void qdisc_pkt_len_init(struct sk_buff *skb)
3036{
3037        const struct skb_shared_info *shinfo = skb_shinfo(skb);
3038
3039        qdisc_skb_cb(skb)->pkt_len = skb->len;
3040
3041        /* To get more precise estimation of bytes sent on wire,
3042         * we add to pkt_len the headers size of all segments
3043         */
3044        if (shinfo->gso_size)  {
3045                unsigned int hdr_len;
3046                u16 gso_segs = shinfo->gso_segs;
3047
3048                /* mac layer + network layer */
3049                hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3050
3051                /* + transport layer */
3052                if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3053                        hdr_len += tcp_hdrlen(skb);
3054                else
3055                        hdr_len += sizeof(struct udphdr);
3056
3057                if (shinfo->gso_type & SKB_GSO_DODGY)
3058                        gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3059                                                shinfo->gso_size);
3060
3061                qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3062        }
3063}
3064
3065static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3066                                 struct net_device *dev,
3067                                 struct netdev_queue *txq)
3068{
3069        spinlock_t *root_lock = qdisc_lock(q);
3070        struct sk_buff *to_free = NULL;
3071        bool contended;
3072        int rc;
3073
3074        qdisc_calculate_pkt_len(skb, q);
3075        /*
3076         * Heuristic to force contended enqueues to serialize on a
3077         * separate lock before trying to get qdisc main lock.
3078         * This permits qdisc->running owner to get the lock more
3079         * often and dequeue packets faster.
3080         */
3081        contended = qdisc_is_running(q);
3082        if (unlikely(contended))
3083                spin_lock(&q->busylock);
3084
3085        spin_lock(root_lock);
3086        if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3087                __qdisc_drop(skb, &to_free);
3088                rc = NET_XMIT_DROP;
3089        } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3090                   qdisc_run_begin(q)) {
3091                /*
3092                 * This is a work-conserving queue; there are no old skbs
3093                 * waiting to be sent out; and the qdisc is not running -
3094                 * xmit the skb directly.
3095                 */
3096
3097                qdisc_bstats_update(q, skb);
3098
3099                if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3100                        if (unlikely(contended)) {
3101                                spin_unlock(&q->busylock);
3102                                contended = false;
3103                        }
3104                        __qdisc_run(q);
3105                } else
3106                        qdisc_run_end(q);
3107
3108                rc = NET_XMIT_SUCCESS;
3109        } else {
3110                rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3111                if (qdisc_run_begin(q)) {
3112                        if (unlikely(contended)) {
3113                                spin_unlock(&q->busylock);
3114                                contended = false;
3115                        }
3116                        __qdisc_run(q);
3117                }
3118        }
3119        spin_unlock(root_lock);
3120        if (unlikely(to_free))
3121                kfree_skb_list(to_free);
3122        if (unlikely(contended))
3123                spin_unlock(&q->busylock);
3124        return rc;
3125}
3126
3127#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3128static void skb_update_prio(struct sk_buff *skb)
3129{
3130        struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3131
3132        if (!skb->priority && skb->sk && map) {
3133                unsigned int prioidx =
3134                        sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3135
3136                if (prioidx < map->priomap_len)
3137                        skb->priority = map->priomap[prioidx];
3138        }
3139}
3140#else
3141#define skb_update_prio(skb)
3142#endif
3143
3144DEFINE_PER_CPU(int, xmit_recursion);
3145EXPORT_SYMBOL(xmit_recursion);
3146
3147/**
3148 *      dev_loopback_xmit - loop back @skb
3149 *      @net: network namespace this loopback is happening in
3150 *      @sk:  sk needed to be a netfilter okfn
3151 *      @skb: buffer to transmit
3152 */
3153int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3154{
3155        skb_reset_mac_header(skb);
3156        __skb_pull(skb, skb_network_offset(skb));
3157        skb->pkt_type = PACKET_LOOPBACK;
3158        skb->ip_summed = CHECKSUM_UNNECESSARY;
3159        WARN_ON(!skb_dst(skb));
3160        skb_dst_force(skb);
3161        netif_rx_ni(skb);
3162        return 0;
3163}
3164EXPORT_SYMBOL(dev_loopback_xmit);
3165
3166#ifdef CONFIG_NET_EGRESS
3167static struct sk_buff *
3168sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3169{
3170        struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3171        struct tcf_result cl_res;
3172
3173        if (!cl)
3174                return skb;
3175
3176        /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3177         * earlier by the caller.
3178         */
3179        qdisc_bstats_cpu_update(cl->q, skb);
3180
3181        switch (tc_classify(skb, cl, &cl_res, false)) {
3182        case TC_ACT_OK:
3183        case TC_ACT_RECLASSIFY:
3184                skb->tc_index = TC_H_MIN(cl_res.classid);
3185                break;
3186        case TC_ACT_SHOT:
3187                qdisc_qstats_cpu_drop(cl->q);
3188                *ret = NET_XMIT_DROP;
3189                kfree_skb(skb);
3190                return NULL;
3191        case TC_ACT_STOLEN:
3192        case TC_ACT_QUEUED:
3193                *ret = NET_XMIT_SUCCESS;
3194                consume_skb(skb);
3195                return NULL;
3196        case TC_ACT_REDIRECT:
3197                /* No need to push/pop skb's mac_header here on egress! */
3198                skb_do_redirect(skb);
3199                *ret = NET_XMIT_SUCCESS;
3200                return NULL;
3201        default:
3202                break;
3203        }
3204
3205        return skb;
3206}
3207#endif /* CONFIG_NET_EGRESS */
3208
3209static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3210{
3211#ifdef CONFIG_XPS
3212        struct xps_dev_maps *dev_maps;
3213        struct xps_map *map;
3214        int queue_index = -1;
3215
3216        rcu_read_lock();
3217        dev_maps = rcu_dereference(dev->xps_maps);
3218        if (dev_maps) {
3219                map = rcu_dereference(
3220                    dev_maps->cpu_map[skb->sender_cpu - 1]);
3221                if (map) {
3222                        if (map->len == 1)
3223                                queue_index = map->queues[0];
3224                        else
3225                                queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3226                                                                           map->len)];
3227                        if (unlikely(queue_index >= dev->real_num_tx_queues))
3228                                queue_index = -1;
3229                }
3230        }
3231        rcu_read_unlock();
3232
3233        return queue_index;
3234#else
3235        return -1;
3236#endif
3237}
3238
3239static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3240{
3241        struct sock *sk = skb->sk;
3242        int queue_index = sk_tx_queue_get(sk);
3243
3244        if (queue_index < 0 || skb->ooo_okay ||
3245            queue_index >= dev->real_num_tx_queues) {
3246                int new_index = get_xps_queue(dev, skb);
3247                if (new_index < 0)
3248                        new_index = skb_tx_hash(dev, skb);
3249
3250                if (queue_index != new_index && sk &&
3251                    sk_fullsock(sk) &&
3252                    rcu_access_pointer(sk->sk_dst_cache))
3253                        sk_tx_queue_set(sk, new_index);
3254
3255                queue_index = new_index;
3256        }
3257
3258        return queue_index;
3259}
3260
3261struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3262                                    struct sk_buff *skb,
3263                                    void *accel_priv)
3264{
3265        int queue_index = 0;
3266
3267#ifdef CONFIG_XPS
3268        u32 sender_cpu = skb->sender_cpu - 1;
3269
3270        if (sender_cpu >= (u32)NR_CPUS)
3271                skb->sender_cpu = raw_smp_processor_id() + 1;
3272#endif
3273
3274        if (dev->real_num_tx_queues != 1) {
3275                const struct net_device_ops *ops = dev->netdev_ops;
3276                if (ops->ndo_select_queue)
3277                        queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3278                                                            __netdev_pick_tx);
3279                else
3280                        queue_index = __netdev_pick_tx(dev, skb);
3281
3282                if (!accel_priv)
3283                        queue_index = netdev_cap_txqueue(dev, queue_index);
3284        }
3285
3286        skb_set_queue_mapping(skb, queue_index);
3287        return netdev_get_tx_queue(dev, queue_index);
3288}
3289
3290/**
3291 *      __dev_queue_xmit - transmit a buffer
3292 *      @skb: buffer to transmit
3293 *      @accel_priv: private data used for L2 forwarding offload
3294 *
3295 *      Queue a buffer for transmission to a network device. The caller must
3296 *      have set the device and priority and built the buffer before calling
3297 *      this function. The function can be called from an interrupt.
3298 *
3299 *      A negative errno code is returned on a failure. A success does not
3300 *      guarantee the frame will be transmitted as it may be dropped due
3301 *      to congestion or traffic shaping.
3302 *
3303 * -----------------------------------------------------------------------------------
3304 *      I notice this method can also return errors from the queue disciplines,
3305 *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3306 *      be positive.
3307 *
3308 *      Regardless of the return value, the skb is consumed, so it is currently
3309 *      difficult to retry a send to this method.  (You can bump the ref count
3310 *      before sending to hold a reference for retry if you are careful.)
3311 *
3312 *      When calling this method, interrupts MUST be enabled.  This is because
3313 *      the BH enable code must have IRQs enabled so that it will not deadlock.
3314 *          --BLG
3315 */
3316static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3317{
3318        struct net_device *dev = skb->dev;
3319        struct netdev_queue *txq;
3320        struct Qdisc *q;
3321        int rc = -ENOMEM;
3322
3323        skb_reset_mac_header(skb);
3324
3325        if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3326                __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3327
3328        /* Disable soft irqs for various locks below. Also
3329         * stops preemption for RCU.
3330         */
3331        rcu_read_lock_bh();
3332
3333        skb_update_prio(skb);
3334
3335        qdisc_pkt_len_init(skb);
3336#ifdef CONFIG_NET_CLS_ACT
3337        skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3338# ifdef CONFIG_NET_EGRESS
3339        if (static_key_false(&egress_needed)) {
3340                skb = sch_handle_egress(skb, &rc, dev);
3341                if (!skb)
3342                        goto out;
3343        }
3344# endif
3345#endif
3346        /* If device/qdisc don't need skb->dst, release it right now while
3347         * its hot in this cpu cache.
3348         */
3349        if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3350                skb_dst_drop(skb);
3351        else
3352                skb_dst_force(skb);
3353
3354        txq = netdev_pick_tx(dev, skb, accel_priv);
3355        q = rcu_dereference_bh(txq->qdisc);
3356
3357        trace_net_dev_queue(skb);
3358        if (q->enqueue) {
3359                rc = __dev_xmit_skb(skb, q, dev, txq);
3360                goto out;
3361        }
3362
3363        /* The device has no queue. Common case for software devices:
3364           loopback, all the sorts of tunnels...
3365
3366           Really, it is unlikely that netif_tx_lock protection is necessary
3367           here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3368           counters.)
3369           However, it is possible, that they rely on protection
3370           made by us here.
3371
3372           Check this and shot the lock. It is not prone from deadlocks.
3373           Either shot noqueue qdisc, it is even simpler 8)
3374         */
3375        if (dev->flags & IFF_UP) {
3376                int cpu = smp_processor_id(); /* ok because BHs are off */
3377
3378                if (txq->xmit_lock_owner != cpu) {
3379                        if (unlikely(__this_cpu_read(xmit_recursion) >
3380                                     XMIT_RECURSION_LIMIT))
3381                                goto recursion_alert;
3382
3383                        skb = validate_xmit_skb(skb, dev);
3384                        if (!skb)
3385                                goto out;
3386
3387                        HARD_TX_LOCK(dev, txq, cpu);
3388
3389                        if (!netif_xmit_stopped(txq)) {
3390                                __this_cpu_inc(xmit_recursion);
3391                                skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3392                                __this_cpu_dec(xmit_recursion);
3393                                if (dev_xmit_complete(rc)) {
3394                                        HARD_TX_UNLOCK(dev, txq);
3395                                        goto out;
3396                                }
3397                        }
3398                        HARD_TX_UNLOCK(dev, txq);
3399                        net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3400                                             dev->name);
3401                } else {
3402                        /* Recursion is detected! It is possible,
3403                         * unfortunately
3404                         */
3405recursion_alert:
3406                        net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3407                                             dev->name);
3408                }
3409        }
3410
3411        rc = -ENETDOWN;
3412        rcu_read_unlock_bh();
3413
3414        atomic_long_inc(&dev->tx_dropped);
3415        kfree_skb_list(skb);
3416        return rc;
3417out:
3418        rcu_read_unlock_bh();
3419        return rc;
3420}
3421
3422int dev_queue_xmit(struct sk_buff *skb)
3423{
3424        return __dev_queue_xmit(skb, NULL);
3425}
3426EXPORT_SYMBOL(dev_queue_xmit);
3427
3428int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3429{
3430        return __dev_queue_xmit(skb, accel_priv);
3431}
3432EXPORT_SYMBOL(dev_queue_xmit_accel);
3433
3434
3435/*=======================================================================
3436                        Receiver routines
3437  =======================================================================*/
3438
3439int netdev_max_backlog __read_mostly = 1000;
3440EXPORT_SYMBOL(netdev_max_backlog);
3441
3442int netdev_tstamp_prequeue __read_mostly = 1;
3443int netdev_budget __read_mostly = 300;
3444int weight_p __read_mostly = 64;            /* old backlog weight */
3445
3446/* Called with irq disabled */
3447static inline void ____napi_schedule(struct softnet_data *sd,
3448                                     struct napi_struct *napi)
3449{
3450        list_add_tail(&napi->poll_list, &sd->poll_list);
3451        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3452}
3453
3454#ifdef CONFIG_RPS
3455
3456/* One global table that all flow-based protocols share. */
3457struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3458EXPORT_SYMBOL(rps_sock_flow_table);
3459u32 rps_cpu_mask __read_mostly;
3460EXPORT_SYMBOL(rps_cpu_mask);
3461
3462struct static_key rps_needed __read_mostly;
3463EXPORT_SYMBOL(rps_needed);
3464
3465static struct rps_dev_flow *
3466set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3467            struct rps_dev_flow *rflow, u16 next_cpu)
3468{
3469        if (next_cpu < nr_cpu_ids) {
3470#ifdef CONFIG_RFS_ACCEL
3471                struct netdev_rx_queue *rxqueue;
3472                struct rps_dev_flow_table *flow_table;
3473                struct rps_dev_flow *old_rflow;
3474                u32 flow_id;
3475                u16 rxq_index;
3476                int rc;
3477
3478                /* Should we steer this flow to a different hardware queue? */
3479                if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3480                    !(dev->features & NETIF_F_NTUPLE))
3481                        goto out;
3482                rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3483                if (rxq_index == skb_get_rx_queue(skb))
3484                        goto out;
3485
3486                rxqueue = dev->_rx + rxq_index;
3487                flow_table = rcu_dereference(rxqueue->rps_flow_table);
3488                if (!flow_table)
3489                        goto out;
3490                flow_id = skb_get_hash(skb) & flow_table->mask;
3491                rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3492                                                        rxq_index, flow_id);
3493                if (rc < 0)
3494                        goto out;
3495                old_rflow = rflow;
3496                rflow = &flow_table->flows[flow_id];
3497                rflow->filter = rc;
3498                if (old_rflow->filter == rflow->filter)
3499                        old_rflow->filter = RPS_NO_FILTER;
3500        out:
3501#endif
3502                rflow->last_qtail =
3503                        per_cpu(softnet_data, next_cpu).input_queue_head;
3504        }
3505
3506        rflow->cpu = next_cpu;
3507        return rflow;
3508}
3509
3510/*
3511 * get_rps_cpu is called from netif_receive_skb and returns the target
3512 * CPU from the RPS map of the receiving queue for a given skb.
3513 * rcu_read_lock must be held on entry.
3514 */
3515static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3516                       struct rps_dev_flow **rflowp)
3517{
3518        const struct rps_sock_flow_table *sock_flow_table;
3519        struct netdev_rx_queue *rxqueue = dev->_rx;
3520        struct rps_dev_flow_table *flow_table;
3521        struct rps_map *map;
3522        int cpu = -1;
3523        u32 tcpu;
3524        u32 hash;
3525
3526        if (skb_rx_queue_recorded(skb)) {
3527                u16 index = skb_get_rx_queue(skb);
3528
3529                if (unlikely(index >= dev->real_num_rx_queues)) {
3530                        WARN_ONCE(dev->real_num_rx_queues > 1,
3531                                  "%s received packet on queue %u, but number "
3532                                  "of RX queues is %u\n",
3533                                  dev->name, index, dev->real_num_rx_queues);
3534                        goto done;
3535                }
3536                rxqueue += index;
3537        }
3538
3539        /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3540
3541        flow_table = rcu_dereference(rxqueue->rps_flow_table);
3542        map = rcu_dereference(rxqueue->rps_map);
3543        if (!flow_table && !map)
3544                goto done;
3545
3546        skb_reset_network_header(skb);
3547        hash = skb_get_hash(skb);
3548        if (!hash)
3549                goto done;
3550
3551        sock_flow_table = rcu_dereference(rps_sock_flow_table);
3552        if (flow_table && sock_flow_table) {
3553                struct rps_dev_flow *rflow;
3554                u32 next_cpu;
3555                u32 ident;
3556
3557                /* First check into global flow table if there is a match */
3558                ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3559                if ((ident ^ hash) & ~rps_cpu_mask)
3560                        goto try_rps;
3561
3562                next_cpu = ident & rps_cpu_mask;
3563
3564                /* OK, now we know there is a match,
3565                 * we can look at the local (per receive queue) flow table
3566                 */
3567                rflow = &flow_table->flows[hash & flow_table->mask];
3568                tcpu = rflow->cpu;
3569
3570                /*
3571                 * If the desired CPU (where last recvmsg was done) is
3572                 * different from current CPU (one in the rx-queue flow
3573                 * table entry), switch if one of the following holds:
3574                 *   - Current CPU is unset (>= nr_cpu_ids).
3575                 *   - Current CPU is offline.
3576                 *   - The current CPU's queue tail has advanced beyond the
3577                 *     last packet that was enqueued using this table entry.
3578                 *     This guarantees that all previous packets for the flow
3579                 *     have been dequeued, thus preserving in order delivery.
3580                 */
3581                if (unlikely(tcpu != next_cpu) &&
3582                    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3583                     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3584                      rflow->last_qtail)) >= 0)) {
3585                        tcpu = next_cpu;
3586                        rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3587                }
3588
3589                if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3590                        *rflowp = rflow;
3591                        cpu = tcpu;
3592                        goto done;
3593                }
3594        }
3595
3596try_rps:
3597
3598        if (map) {
3599                tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3600                if (cpu_online(tcpu)) {
3601                        cpu = tcpu;
3602                        goto done;
3603                }
3604        }
3605
3606done:
3607        return cpu;
3608}
3609
3610#ifdef CONFIG_RFS_ACCEL
3611
3612/**
3613 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3614 * @dev: Device on which the filter was set
3615 * @rxq_index: RX queue index
3616 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3617 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3618 *
3619 * Drivers that implement ndo_rx_flow_steer() should periodically call
3620 * this function for each installed filter and remove the filters for
3621 * which it returns %true.
3622 */
3623bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3624                         u32 flow_id, u16 filter_id)
3625{
3626        struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3627        struct rps_dev_flow_table *flow_table;
3628        struct rps_dev_flow *rflow;
3629        bool expire = true;
3630        unsigned int cpu;
3631
3632        rcu_read_lock();
3633        flow_table = rcu_dereference(rxqueue->rps_flow_table);
3634        if (flow_table && flow_id <= flow_table->mask) {
3635                rflow = &flow_table->flows[flow_id];
3636                cpu = ACCESS_ONCE(rflow->cpu);
3637                if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3638                    ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3639                           rflow->last_qtail) <
3640                     (int)(10 * flow_table->mask)))
3641                        expire = false;
3642        }
3643        rcu_read_unlock();
3644        return expire;
3645}
3646EXPORT_SYMBOL(rps_may_expire_flow);
3647
3648#endif /* CONFIG_RFS_ACCEL */
3649
3650/* Called from hardirq (IPI) context */
3651static void rps_trigger_softirq(void *data)
3652{
3653        struct softnet_data *sd = data;
3654
3655        ____napi_schedule(sd, &sd->backlog);
3656        sd->received_rps++;
3657}
3658
3659#endif /* CONFIG_RPS */
3660
3661/*
3662 * Check if this softnet_data structure is another cpu one
3663 * If yes, queue it to our IPI list and return 1
3664 * If no, return 0
3665 */
3666static int rps_ipi_queued(struct softnet_data *sd)
3667{
3668#ifdef CONFIG_RPS
3669        struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3670
3671        if (sd != mysd) {
3672                sd->rps_ipi_next = mysd->rps_ipi_list;
3673                mysd->rps_ipi_list = sd;
3674
3675                __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3676                return 1;
3677        }
3678#endif /* CONFIG_RPS */
3679        return 0;
3680}
3681
3682#ifdef CONFIG_NET_FLOW_LIMIT
3683int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3684#endif
3685
3686static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3687{
3688#ifdef CONFIG_NET_FLOW_LIMIT
3689        struct sd_flow_limit *fl;
3690        struct softnet_data *sd;
3691        unsigned int old_flow, new_flow;
3692
3693        if (qlen < (netdev_max_backlog >> 1))
3694                return false;
3695
3696        sd = this_cpu_ptr(&softnet_data);
3697
3698        rcu_read_lock();
3699        fl = rcu_dereference(sd->flow_limit);
3700        if (fl) {
3701                new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3702                old_flow = fl->history[fl->history_head];
3703                fl->history[fl->history_head] = new_flow;
3704
3705                fl->history_head++;
3706                fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3707
3708                if (likely(fl->buckets[old_flow]))
3709                        fl->buckets[old_flow]--;
3710
3711                if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3712                        fl->count++;
3713                        rcu_read_unlock();
3714                        return true;
3715                }
3716        }
3717        rcu_read_unlock();
3718#endif
3719        return false;
3720}
3721
3722/*
3723 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3724 * queue (may be a remote CPU queue).
3725 */
3726static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3727                              unsigned int *qtail)
3728{
3729        struct softnet_data *sd;
3730        unsigned long flags;
3731        unsigned int qlen;
3732
3733        sd = &per_cpu(softnet_data, cpu);
3734
3735        local_irq_save(flags);
3736
3737        rps_lock(sd);
3738        if (!netif_running(skb->dev))
3739                goto drop;
3740        qlen = skb_queue_len(&sd->input_pkt_queue);
3741        if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3742                if (qlen) {
3743enqueue:
3744                        __skb_queue_tail(&sd->input_pkt_queue, skb);
3745                        input_queue_tail_incr_save(sd, qtail);
3746                        rps_unlock(sd);
3747                        local_irq_restore(flags);
3748                        return NET_RX_SUCCESS;
3749                }
3750
3751                /* Schedule NAPI for backlog device
3752                 * We can use non atomic operation since we own the queue lock
3753                 */
3754                if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3755                        if (!rps_ipi_queued(sd))
3756                                ____napi_schedule(sd, &sd->backlog);
3757                }
3758                goto enqueue;
3759        }
3760
3761drop:
3762        sd->dropped++;
3763        rps_unlock(sd);
3764
3765        local_irq_restore(flags);
3766
3767        atomic_long_inc(&skb->dev->rx_dropped);
3768        kfree_skb(skb);
3769        return NET_RX_DROP;
3770}
3771
3772static int netif_rx_internal(struct sk_buff *skb)
3773{
3774        int ret;
3775
3776        net_timestamp_check(netdev_tstamp_prequeue, skb);
3777
3778        trace_netif_rx(skb);
3779#ifdef CONFIG_RPS
3780        if (static_key_false(&rps_needed)) {
3781                struct rps_dev_flow voidflow, *rflow = &voidflow;
3782                int cpu;
3783
3784                preempt_disable();
3785                rcu_read_lock();
3786
3787                cpu = get_rps_cpu(skb->dev, skb, &rflow);
3788                if (cpu < 0)
3789                        cpu = smp_processor_id();
3790
3791                ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3792
3793                rcu_read_unlock();
3794                preempt_enable();
3795        } else
3796#endif
3797        {
3798                unsigned int qtail;
3799                ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3800                put_cpu();
3801        }
3802        return ret;
3803}
3804
3805/**
3806 *      netif_rx        -       post buffer to the network code
3807 *      @skb: buffer to post
3808 *
3809 *      This function receives a packet from a device driver and queues it for
3810 *      the upper (protocol) levels to process.  It always succeeds. The buffer
3811 *      may be dropped during processing for congestion control or by the
3812 *      protocol layers.
3813 *
3814 *      return values:
3815 *      NET_RX_SUCCESS  (no congestion)
3816 *      NET_RX_DROP     (packet was dropped)
3817 *
3818 */
3819
3820int netif_rx(struct sk_buff *skb)
3821{
3822        trace_netif_rx_entry(skb);
3823
3824        return netif_rx_internal(skb);
3825}
3826EXPORT_SYMBOL(netif_rx);
3827
3828int netif_rx_ni(struct sk_buff *skb)
3829{
3830        int err;
3831
3832        trace_netif_rx_ni_entry(skb);
3833
3834        preempt_disable();
3835        err = netif_rx_internal(skb);
3836        if (local_softirq_pending())
3837                do_softirq();
3838        preempt_enable();
3839
3840        return err;
3841}
3842EXPORT_SYMBOL(netif_rx_ni);
3843
3844static __latent_entropy void net_tx_action(struct softirq_action *h)
3845{
3846        struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3847
3848        if (sd->completion_queue) {
3849                struct sk_buff *clist;
3850
3851                local_irq_disable();
3852                clist = sd->completion_queue;
3853                sd->completion_queue = NULL;
3854                local_irq_enable();
3855
3856                while (clist) {
3857                        struct sk_buff *skb = clist;
3858                        clist = clist->next;
3859
3860                        WARN_ON(atomic_read(&skb->users));
3861                        if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3862                                trace_consume_skb(skb);
3863                        else
3864                                trace_kfree_skb(skb, net_tx_action);
3865
3866                        if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3867                                __kfree_skb(skb);
3868                        else
3869                                __kfree_skb_defer(skb);
3870                }
3871
3872                __kfree_skb_flush();
3873        }
3874
3875        if (sd->output_queue) {
3876                struct Qdisc *head;
3877
3878                local_irq_disable();
3879                head = sd->output_queue;
3880                sd->output_queue = NULL;
3881                sd->output_queue_tailp = &sd->output_queue;
3882                local_irq_enable();
3883
3884                while (head) {
3885                        struct Qdisc *q = head;
3886                        spinlock_t *root_lock;
3887
3888                        head = head->next_sched;
3889
3890                        root_lock = qdisc_lock(q);
3891                        spin_lock(root_lock);
3892                        /* We need to make sure head->next_sched is read
3893                         * before clearing __QDISC_STATE_SCHED
3894                         */
3895                        smp_mb__before_atomic();
3896                        clear_bit(__QDISC_STATE_SCHED, &q->state);
3897                        qdisc_run(q);
3898                        spin_unlock(root_lock);
3899                }
3900        }
3901}
3902
3903#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
3904/* This hook is defined here for ATM LANE */
3905int (*br_fdb_test_addr_hook)(struct net_device *dev,
3906                             unsigned char *addr) __read_mostly;
3907EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3908#endif
3909
3910static inline struct sk_buff *
3911sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3912                   struct net_device *orig_dev)
3913{
3914#ifdef CONFIG_NET_CLS_ACT
3915        struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3916        struct tcf_result cl_res;
3917
3918        /* If there's at least one ingress present somewhere (so
3919         * we get here via enabled static key), remaining devices
3920         * that are not configured with an ingress qdisc will bail
3921         * out here.
3922         */
3923        if (!cl)
3924                return skb;
3925        if (*pt_prev) {
3926                *ret = deliver_skb(skb, *pt_prev, orig_dev);
3927                *pt_prev = NULL;
3928        }
3929
3930        qdisc_skb_cb(skb)->pkt_len = skb->len;
3931        skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3932        qdisc_bstats_cpu_update(cl->q, skb);
3933
3934        switch (tc_classify(skb, cl, &cl_res, false)) {
3935        case TC_ACT_OK:
3936        case TC_ACT_RECLASSIFY:
3937                skb->tc_index = TC_H_MIN(cl_res.classid);
3938                break;
3939        case TC_ACT_SHOT:
3940                qdisc_qstats_cpu_drop(cl->q);
3941                kfree_skb(skb);
3942                return NULL;
3943        case TC_ACT_STOLEN:
3944        case TC_ACT_QUEUED:
3945                consume_skb(skb);
3946                return NULL;
3947        case TC_ACT_REDIRECT:
3948                /* skb_mac_header check was done by cls/act_bpf, so
3949                 * we can safely push the L2 header back before
3950                 * redirecting to another netdev
3951                 */
3952                __skb_push(skb, skb->mac_len);
3953                skb_do_redirect(skb);
3954                return NULL;
3955        default:
3956                break;
3957        }
3958#endif /* CONFIG_NET_CLS_ACT */
3959        return skb;
3960}
3961
3962/**
3963 *      netdev_is_rx_handler_busy - check if receive handler is registered
3964 *      @dev: device to check
3965 *
3966 *      Check if a receive handler is already registered for a given device.
3967 *      Return true if there one.
3968 *
3969 *      The caller must hold the rtnl_mutex.
3970 */
3971bool netdev_is_rx_handler_busy(struct net_device *dev)
3972{
3973        ASSERT_RTNL();
3974        return dev && rtnl_dereference(dev->rx_handler);
3975}
3976EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
3977
3978/**
3979 *      netdev_rx_handler_register - register receive handler
3980 *      @dev: device to register a handler for
3981 *      @rx_handler: receive handler to register
3982 *      @rx_handler_data: data pointer that is used by rx handler
3983 *
3984 *      Register a receive handler for a device. This handler will then be
3985 *      called from __netif_receive_skb. A negative errno code is returned
3986 *      on a failure.
3987 *
3988 *      The caller must hold the rtnl_mutex.
3989 *
3990 *      For a general description of rx_handler, see enum rx_handler_result.
3991 */
3992int netdev_rx_handler_register(struct net_device *dev,
3993                               rx_handler_func_t *rx_handler,
3994                               void *rx_handler_data)
3995{
3996        ASSERT_RTNL();
3997
3998        if (dev->rx_handler)
3999                return -EBUSY;
4000
4001        /* Note: rx_handler_data must be set before rx_handler */
4002        rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4003        rcu_assign_pointer(dev->rx_handler, rx_handler);
4004
4005        return 0;
4006}
4007EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4008
4009/**
4010 *      netdev_rx_handler_unregister - unregister receive handler
4011 *      @dev: device to unregister a handler from
4012 *
4013 *      Unregister a receive handler from a device.
4014 *
4015 *      The caller must hold the rtnl_mutex.
4016 */
4017void netdev_rx_handler_unregister(struct net_device *dev)
4018{
4019
4020        ASSERT_RTNL();
4021        RCU_INIT_POINTER(dev->rx_handler, NULL);
4022        /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4023         * section has a guarantee to see a non NULL rx_handler_data
4024         * as well.
4025         */
4026        synchronize_net();
4027        RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4028}
4029EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4030
4031/*
4032 * Limit the use of PFMEMALLOC reserves to those protocols that implement
4033 * the special handling of PFMEMALLOC skbs.
4034 */
4035static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4036{
4037        switch (skb->protocol) {
4038        case htons(ETH_P_ARP):
4039        case htons(ETH_P_IP):
4040        case htons(ETH_P_IPV6):
4041        case htons(ETH_P_8021Q):
4042        case htons(ETH_P_8021AD):
4043                return true;
4044        default:
4045                return false;
4046        }
4047}
4048
4049static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4050                             int *ret, struct net_device *orig_dev)
4051{
4052#ifdef CONFIG_NETFILTER_INGRESS
4053        if (nf_hook_ingress_active(skb)) {
4054                int ingress_retval;
4055
4056                if (*pt_prev) {
4057                        *ret = deliver_skb(skb, *pt_prev, orig_dev);
4058                        *pt_prev = NULL;
4059                }
4060
4061                rcu_read_lock();
4062                ingress_retval = nf_hook_ingress(skb);
4063                rcu_read_unlock();
4064                return ingress_retval;
4065        }
4066#endif /* CONFIG_NETFILTER_INGRESS */
4067        return 0;
4068}
4069
4070static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4071{
4072        struct packet_type *ptype, *pt_prev;
4073        rx_handler_func_t *rx_handler;
4074        struct net_device *orig_dev;
4075        bool deliver_exact = false;
4076        int ret = NET_RX_DROP;
4077        __be16 type;
4078
4079        net_timestamp_check(!netdev_tstamp_prequeue, skb);
4080
4081        trace_netif_receive_skb(skb);
4082
4083        orig_dev = skb->dev;
4084
4085        skb_reset_network_header(skb);
4086        if (!skb_transport_header_was_set(skb))
4087                skb_reset_transport_header(skb);
4088        skb_reset_mac_len(skb);
4089
4090        pt_prev = NULL;
4091
4092another_round:
4093        skb->skb_iif = skb->dev->ifindex;
4094
4095        __this_cpu_inc(softnet_data.processed);
4096
4097        if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4098            skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4099                skb = skb_vlan_untag(skb);
4100                if (unlikely(!skb))
4101                        goto out;
4102        }
4103
4104#ifdef CONFIG_NET_CLS_ACT
4105        if (skb->tc_verd & TC_NCLS) {
4106                skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
4107                goto ncls;
4108        }
4109#endif
4110
4111        if (pfmemalloc)
4112                goto skip_taps;
4113
4114        list_for_each_entry_rcu(ptype, &ptype_all, list) {
4115                if (pt_prev)
4116                        ret = deliver_skb(skb, pt_prev, orig_dev);
4117                pt_prev = ptype;
4118        }
4119
4120        list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4121                if (pt_prev)
4122                        ret = deliver_skb(skb, pt_prev, orig_dev);
4123                pt_prev = ptype;
4124        }
4125
4126skip_taps:
4127#ifdef CONFIG_NET_INGRESS
4128        if (static_key_false(&ingress_needed)) {
4129                skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4130                if (!skb)
4131                        goto out;
4132
4133                if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4134                        goto out;
4135        }
4136#endif
4137#ifdef CONFIG_NET_CLS_ACT
4138        skb->tc_verd = 0;
4139ncls:
4140#endif
4141        if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4142                goto drop;
4143
4144        if (skb_vlan_tag_present(skb)) {
4145                if (pt_prev) {
4146                        ret = deliver_skb(skb, pt_prev, orig_dev);
4147                        pt_prev = NULL;
4148                }
4149                if (vlan_do_receive(&skb))
4150                        goto another_round;
4151                else if (unlikely(!skb))
4152                        goto out;
4153        }
4154
4155        rx_handler = rcu_dereference(skb->dev->rx_handler);
4156        if (rx_handler) {
4157                if (pt_prev) {
4158                        ret = deliver_skb(skb, pt_prev, orig_dev);
4159                        pt_prev = NULL;
4160                }
4161                switch (rx_handler(&skb)) {
4162                case RX_HANDLER_CONSUMED:
4163                        ret = NET_RX_SUCCESS;
4164                        goto out;
4165                case RX_HANDLER_ANOTHER:
4166                        goto another_round;
4167                case RX_HANDLER_EXACT:
4168                        deliver_exact = true;
4169                case RX_HANDLER_PASS:
4170                        break;
4171                default:
4172                        BUG();
4173                }
4174        }
4175
4176        if (unlikely(skb_vlan_tag_present(skb))) {
4177                if (skb_vlan_tag_get_id(skb))
4178                        skb->pkt_type = PACKET_OTHERHOST;
4179                /* Note: we might in the future use prio bits
4180                 * and set skb->priority like in vlan_do_receive()
4181                 * For the time being, just ignore Priority Code Point
4182                 */
4183                skb->vlan_tci = 0;
4184        }
4185
4186        type = skb->protocol;
4187
4188        /* deliver only exact match when indicated */
4189        if (likely(!deliver_exact)) {
4190                deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4191                                       &ptype_base[ntohs(type) &
4192                                                   PTYPE_HASH_MASK]);
4193        }
4194
4195        deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4196                               &orig_dev->ptype_specific);
4197
4198        if (unlikely(skb->dev != orig_dev)) {
4199                deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4200                                       &skb->dev->ptype_specific);
4201        }
4202
4203        if (pt_prev) {
4204                if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4205                        goto drop;
4206                else
4207                        ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4208        } else {
4209drop:
4210                if (!deliver_exact)
4211                        atomic_long_inc(&skb->dev->rx_dropped);
4212                else
4213                        atomic_long_inc(&skb->dev->rx_nohandler);
4214                kfree_skb(skb);
4215                /* Jamal, now you will not able to escape explaining
4216                 * me how you were going to use this. :-)
4217                 */
4218                ret = NET_RX_DROP;
4219        }
4220
4221out:
4222        return ret;
4223}
4224
4225static int __netif_receive_skb(struct sk_buff *skb)
4226{
4227        int ret;
4228
4229        if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4230                unsigned long pflags = current->flags;
4231
4232                /*
4233                 * PFMEMALLOC skbs are special, they should
4234                 * - be delivered to SOCK_MEMALLOC sockets only
4235                 * - stay away from userspace
4236                 * - have bounded memory usage
4237                 *
4238                 * Use PF_MEMALLOC as this saves us from propagating the allocation
4239                 * context down to all allocation sites.
4240                 */
4241                current->flags |= PF_MEMALLOC;
4242                ret = __netif_receive_skb_core(skb, true);
4243                tsk_restore_flags(current, pflags, PF_MEMALLOC);
4244        } else
4245                ret = __netif_receive_skb_core(skb, false);
4246
4247        return ret;
4248}
4249
4250static int netif_receive_skb_internal(struct sk_buff *skb)
4251{
4252        int ret;
4253
4254        net_timestamp_check(netdev_tstamp_prequeue, skb);
4255
4256        if (skb_defer_rx_timestamp(skb))
4257                return NET_RX_SUCCESS;
4258
4259        rcu_read_lock();
4260
4261#ifdef CONFIG_RPS
4262        if (static_key_false(&rps_needed)) {
4263                struct rps_dev_flow voidflow, *rflow = &voidflow;
4264                int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4265
4266                if (cpu >= 0) {
4267                        ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4268                        rcu_read_unlock();
4269                        return ret;
4270                }
4271        }
4272#endif
4273        ret = __netif_receive_skb(skb);
4274        rcu_read_unlock();
4275        return ret;
4276}
4277
4278/**
4279 *      netif_receive_skb - process receive buffer from network
4280 *      @skb: buffer to process
4281 *
4282 *      netif_receive_skb() is the main receive data processing function.
4283 *      It always succeeds. The buffer may be dropped during processing
4284 *      for congestion control or by the protocol layers.
4285 *
4286 *      This function may only be called from softirq context and interrupts
4287 *      should be enabled.
4288 *
4289 *      Return values (usually ignored):
4290 *      NET_RX_SUCCESS: no congestion
4291 *      NET_RX_DROP: packet was dropped
4292 */
4293int netif_receive_skb(struct sk_buff *skb)
4294{
4295        trace_netif_receive_skb_entry(skb);
4296
4297        return netif_receive_skb_internal(skb);
4298}
4299EXPORT_SYMBOL(netif_receive_skb);
4300
4301DEFINE_PER_CPU(struct work_struct, flush_works);
4302
4303/* Network device is going away, flush any packets still pending */
4304static void flush_backlog(struct work_struct *work)
4305{
4306        struct sk_buff *skb, *tmp;
4307        struct softnet_data *sd;
4308
4309        local_bh_disable();
4310        sd = this_cpu_ptr(&softnet_data);
4311
4312        local_irq_disable();
4313        rps_lock(sd);
4314        skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4315                if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4316                        __skb_unlink(skb, &sd->input_pkt_queue);
4317                        kfree_skb(skb);
4318                        input_queue_head_incr(sd);
4319                }
4320        }
4321        rps_unlock(sd);
4322        local_irq_enable();
4323
4324        skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4325                if (skb->dev->reg_state == NETREG_UNREGISTERING) {
4326                        __skb_unlink(skb, &sd->process_queue);
4327                        kfree_skb(skb);
4328                        input_queue_head_incr(sd);
4329                }
4330        }
4331        local_bh_enable();
4332}
4333
4334static void flush_all_backlogs(void)
4335{
4336        unsigned int cpu;
4337
4338        get_online_cpus();
4339
4340        for_each_online_cpu(cpu)
4341                queue_work_on(cpu, system_highpri_wq,
4342                              per_cpu_ptr(&flush_works, cpu));
4343
4344        for_each_online_cpu(cpu)
4345                flush_work(per_cpu_ptr(&flush_works, cpu));
4346
4347        put_online_cpus();
4348}
4349
4350static int napi_gro_complete(struct sk_buff *skb)
4351{
4352        struct packet_offload *ptype;
4353        __be16 type = skb->protocol;
4354        struct list_head *head = &offload_base;
4355        int err = -ENOENT;
4356
4357        BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4358
4359        if (NAPI_GRO_CB(skb)->count == 1) {
4360                skb_shinfo(skb)->gso_size = 0;
4361                goto out;
4362        }
4363
4364        rcu_read_lock();
4365        list_for_each_entry_rcu(ptype, head, list) {
4366                if (ptype->type != type || !ptype->callbacks.gro_complete)
4367                        continue;
4368
4369                err = ptype->callbacks.gro_complete(skb, 0);
4370                break;
4371        }
4372        rcu_read_unlock();
4373
4374        if (err) {
4375                WARN_ON(&ptype->list == head);
4376                kfree_skb(skb);
4377                return NET_RX_SUCCESS;
4378        }
4379
4380out:
4381        return netif_receive_skb_internal(skb);
4382}
4383
4384/* napi->gro_list contains packets ordered by age.
4385 * youngest packets at the head of it.
4386 * Complete skbs in reverse order to reduce latencies.
4387 */
4388void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4389{
4390        struct sk_buff *skb, *prev = NULL;
4391
4392        /* scan list and build reverse chain */
4393        for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4394                skb->prev = prev;
4395                prev = skb;
4396        }
4397
4398        for (skb = prev; skb; skb = prev) {
4399                skb->next = NULL;
4400
4401                if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4402                        return;
4403
4404                prev = skb->prev;
4405                napi_gro_complete(skb);
4406                napi->gro_count--;
4407        }
4408
4409        napi->gro_list = NULL;
4410}
4411EXPORT_SYMBOL(napi_gro_flush);
4412
4413static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4414{
4415        struct sk_buff *p;
4416        unsigned int maclen = skb->dev->hard_header_len;
4417        u32 hash = skb_get_hash_raw(skb);
4418
4419        for (p = napi->gro_list; p; p = p->next) {
4420                unsigned long diffs;
4421
4422                NAPI_GRO_CB(p)->flush = 0;
4423
4424                if (hash != skb_get_hash_raw(p)) {
4425                        NAPI_GRO_CB(p)->same_flow = 0;
4426                        continue;
4427                }
4428
4429                diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4430                diffs |= p->vlan_tci ^ skb->vlan_tci;
4431                diffs |= skb_metadata_dst_cmp(p, skb);
4432                if (maclen == ETH_HLEN)
4433                        diffs |= compare_ether_header(skb_mac_header(p),
4434                                                      skb_mac_header(skb));
4435                else if (!diffs)
4436                        diffs = memcmp(skb_mac_header(p),
4437                                       skb_mac_header(skb),
4438                                       maclen);
4439                NAPI_GRO_CB(p)->same_flow = !diffs;
4440        }
4441}
4442
4443static void skb_gro_reset_offset(struct sk_buff *skb)
4444{
4445        const struct skb_shared_info *pinfo = skb_shinfo(skb);
4446        const skb_frag_t *frag0 = &pinfo->frags[0];
4447
4448        NAPI_GRO_CB(skb)->data_offset = 0;
4449        NAPI_GRO_CB(skb)->frag0 = NULL;
4450        NAPI_GRO_CB(skb)->frag0_len = 0;
4451
4452        if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4453            pinfo->nr_frags &&
4454            !PageHighMem(skb_frag_page(frag0))) {
4455                NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4456                NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
4457                                                    skb_frag_size(frag0),
4458                                                    skb->end - skb->tail);
4459        }
4460}
4461
4462static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4463{
4464        struct skb_shared_info *pinfo = skb_shinfo(skb);
4465
4466        BUG_ON(skb->end - skb->tail < grow);
4467
4468        memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4469
4470        skb->data_len -= grow;
4471        skb->tail += grow;
4472
4473        pinfo->frags[0].page_offset += grow;
4474        skb_frag_size_sub(&pinfo->frags[0], grow);
4475
4476        if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4477                skb_frag_unref(skb, 0);
4478                memmove(pinfo->frags, pinfo->frags + 1,
4479                        --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4480        }
4481}
4482
4483static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4484{
4485        struct sk_buff **pp = NULL;
4486        struct packet_offload *ptype;
4487        __be16 type = skb->protocol;
4488        struct list_head *head = &offload_base;
4489        int same_flow;
4490        enum gro_result ret;
4491        int grow;
4492
4493        if (!(skb->dev->features & NETIF_F_GRO))
4494                goto normal;
4495
4496        if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
4497                goto normal;
4498
4499        gro_list_prepare(napi, skb);
4500
4501        rcu_read_lock();
4502        list_for_each_entry_rcu(ptype, head, list) {
4503                if (ptype->type != type || !ptype->callbacks.gro_receive)
4504                        continue;
4505
4506                skb_set_network_header(skb, skb_gro_offset(skb));
4507                skb_reset_mac_len(skb);
4508                NAPI_GRO_CB(skb)->same_flow = 0;
4509                NAPI_GRO_CB(skb)->flush = 0;
4510                NAPI_GRO_CB(skb)->free = 0;
4511                NAPI_GRO_CB(skb)->encap_mark = 0;
4512                NAPI_GRO_CB(skb)->recursion_counter = 0;
4513                NAPI_GRO_CB(skb)->is_fou = 0;
4514                NAPI_GRO_CB(skb)->is_atomic = 1;
4515                NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4516
4517                /* Setup for GRO checksum validation */
4518                switch (skb->ip_summed) {
4519                case CHECKSUM_COMPLETE:
4520                        NAPI_GRO_CB(skb)->csum = skb->csum;
4521                        NAPI_GRO_CB(skb)->csum_valid = 1;
4522                        NAPI_GRO_CB(skb)->csum_cnt = 0;
4523                        break;
4524                case CHECKSUM_UNNECESSARY:
4525                        NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4526                        NAPI_GRO_CB(skb)->csum_valid = 0;
4527                        break;
4528                default:
4529                        NAPI_GRO_CB(skb)->csum_cnt = 0;
4530                        NAPI_GRO_CB(skb)->csum_valid = 0;
4531                }
4532
4533                pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4534                break;
4535        }
4536        rcu_read_unlock();
4537
4538        if (&ptype->list == head)
4539                goto normal;
4540
4541        same_flow = NAPI_GRO_CB(skb)->same_flow;
4542        ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4543
4544        if (pp) {
4545                struct sk_buff *nskb = *pp;
4546
4547                *pp = nskb->next;
4548                nskb->next = NULL;
4549                napi_gro_complete(nskb);
4550                napi->gro_count--;
4551        }
4552
4553        if (same_flow)
4554                goto ok;
4555
4556        if (NAPI_GRO_CB(skb)->flush)
4557                goto normal;
4558
4559        if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4560                struct sk_buff *nskb = napi->gro_list;
4561
4562                /* locate the end of the list to select the 'oldest' flow */
4563                while (nskb->next) {
4564                        pp = &nskb->next;
4565                        nskb = *pp;
4566                }
4567                *pp = NULL;
4568                nskb->next = NULL;
4569                napi_gro_complete(nskb);
4570        } else {
4571                napi->gro_count++;
4572        }
4573        NAPI_GRO_CB(skb)->count = 1;
4574        NAPI_GRO_CB(skb)->age = jiffies;
4575        NAPI_GRO_CB(skb)->last = skb;
4576        skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4577        skb->next = napi->gro_list;
4578        napi->gro_list = skb;
4579        ret = GRO_HELD;
4580
4581pull:
4582        grow = skb_gro_offset(skb) - skb_headlen(skb);
4583        if (grow > 0)
4584                gro_pull_from_frag0(skb, grow);
4585ok:
4586        return ret;
4587
4588normal:
4589        ret = GRO_NORMAL;
4590        goto pull;
4591}
4592
4593struct packet_offload *gro_find_receive_by_type(__be16 type)
4594{
4595        struct list_head *offload_head = &offload_base;
4596        struct packet_offload *ptype;
4597
4598        list_for_each_entry_rcu(ptype, offload_head, list) {
4599                if (ptype->type != type || !ptype->callbacks.gro_receive)
4600                        continue;
4601                return ptype;
4602        }
4603        return NULL;
4604}
4605EXPORT_SYMBOL(gro_find_receive_by_type);
4606
4607struct packet_offload *gro_find_complete_by_type(__be16 type)
4608{
4609        struct list_head *offload_head = &offload_base;
4610        struct packet_offload *ptype;
4611
4612        list_for_each_entry_rcu(ptype, offload_head, list) {
4613                if (ptype->type != type || !ptype->callbacks.gro_complete)
4614                        continue;
4615                return ptype;
4616        }
4617        return NULL;
4618}
4619EXPORT_SYMBOL(gro_find_complete_by_type);
4620
4621static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4622{
4623        switch (ret) {
4624        case GRO_NORMAL:
4625                if (netif_receive_skb_internal(skb))
4626                        ret = GRO_DROP;
4627                break;
4628
4629        case GRO_DROP:
4630                kfree_skb(skb);
4631                break;
4632
4633        case GRO_MERGED_FREE:
4634                if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4635                        skb_dst_drop(skb);
4636                        kmem_cache_free(skbuff_head_cache, skb);
4637                } else {
4638                        __kfree_skb(skb);
4639                }
4640                break;
4641
4642        case GRO_HELD:
4643        case GRO_MERGED:
4644                break;
4645        }
4646
4647        return ret;
4648}
4649
4650gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4651{
4652        skb_mark_napi_id(skb, napi);
4653        trace_napi_gro_receive_entry(skb);
4654
4655        skb_gro_reset_offset(skb);
4656
4657        return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4658}
4659EXPORT_SYMBOL(napi_gro_receive);
4660
4661static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4662{
4663        if (unlikely(skb->pfmemalloc)) {
4664                consume_skb(skb);
4665                return;
4666        }
4667        __skb_pull(skb, skb_headlen(skb));
4668        /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4669        skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4670        skb->vlan_tci = 0;
4671        skb->dev = napi->dev;
4672        skb->skb_iif = 0;
4673        skb->encapsulation = 0;
4674        skb_shinfo(skb)->gso_type = 0;
4675        skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4676
4677        napi->skb = skb;
4678}
4679
4680struct sk_buff *napi_get_frags(struct napi_struct *napi)
4681{
4682        struct sk_buff *skb = napi->skb;
4683
4684        if (!skb) {
4685                skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4686                if (skb) {
4687                        napi->skb = skb;
4688                        skb_mark_napi_id(skb, napi);
4689                }
4690        }
4691        return skb;
4692}
4693EXPORT_SYMBOL(napi_get_frags);
4694
4695static gro_result_t napi_frags_finish(struct napi_struct *napi,
4696                                      struct sk_buff *skb,
4697                                      gro_result_t ret)
4698{
4699        switch (ret) {
4700        case GRO_NORMAL:
4701        case GRO_HELD:
4702                __skb_push(skb, ETH_HLEN);
4703                skb->protocol = eth_type_trans(skb, skb->dev);
4704                if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4705                        ret = GRO_DROP;
4706                break;
4707
4708        case GRO_DROP:
4709        case GRO_MERGED_FREE:
4710                napi_reuse_skb(napi, skb);
4711                break;
4712
4713        case GRO_MERGED:
4714                break;
4715        }
4716
4717        return ret;
4718}
4719
4720/* Upper GRO stack assumes network header starts at gro_offset=0
4721 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4722 * We copy ethernet header into skb->data to have a common layout.
4723 */
4724static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4725{
4726        struct sk_buff *skb = napi->skb;
4727        const struct ethhdr *eth;
4728        unsigned int hlen = sizeof(*eth);
4729
4730        napi->skb = NULL;
4731
4732        skb_reset_mac_header(skb);
4733        skb_gro_reset_offset(skb);
4734
4735        eth = skb_gro_header_fast(skb, 0);
4736        if (unlikely(skb_gro_header_hard(skb, hlen))) {
4737                eth = skb_gro_header_slow(skb, hlen, 0);
4738                if (unlikely(!eth)) {
4739                        net_warn_ratelimited("%s: dropping impossible skb from %s\n",
4740                                             __func__, napi->dev->name);
4741                        napi_reuse_skb(napi, skb);
4742                        return NULL;
4743                }
4744        } else {
4745                gro_pull_from_frag0(skb, hlen);
4746                NAPI_GRO_CB(skb)->frag0 += hlen;
4747                NAPI_GRO_CB(skb)->frag0_len -= hlen;
4748        }
4749        __skb_pull(skb, hlen);
4750
4751        /*
4752         * This works because the only protocols we care about don't require
4753         * special handling.
4754         * We'll fix it up properly in napi_frags_finish()
4755         */
4756        skb->protocol = eth->h_proto;
4757
4758        return skb;
4759}
4760
4761gro_result_t napi_gro_frags(struct napi_struct *napi)
4762{
4763        struct sk_buff *skb = napi_frags_skb(napi);
4764
4765        if (!skb)
4766                return GRO_DROP;
4767
4768        trace_napi_gro_frags_entry(skb);
4769
4770        return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4771}
4772EXPORT_SYMBOL(napi_gro_frags);
4773
4774/* Compute the checksum from gro_offset and return the folded value
4775 * after adding in any pseudo checksum.
4776 */
4777__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4778{
4779        __wsum wsum;
4780        __sum16 sum;
4781
4782        wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4783
4784        /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4785        sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4786        if (likely(!sum)) {
4787                if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4788                    !skb->csum_complete_sw)
4789                        netdev_rx_csum_fault(skb->dev);
4790        }
4791
4792        NAPI_GRO_CB(skb)->csum = wsum;
4793        NAPI_GRO_CB(skb)->csum_valid = 1;
4794
4795        return sum;
4796}
4797EXPORT_SYMBOL(__skb_gro_checksum_complete);
4798
4799/*
4800 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4801 * Note: called with local irq disabled, but exits with local irq enabled.
4802 */
4803static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4804{
4805#ifdef CONFIG_RPS
4806        struct softnet_data *remsd = sd->rps_ipi_list;
4807
4808        if (remsd) {
4809                sd->rps_ipi_list = NULL;
4810
4811                local_irq_enable();
4812
4813                /* Send pending IPI's to kick RPS processing on remote cpus. */
4814                while (remsd) {
4815                        struct softnet_data *next = remsd->rps_ipi_next;
4816
4817                        if (cpu_online(remsd->cpu))
4818                                smp_call_function_single_async(remsd->cpu,
4819                                                           &remsd->csd);
4820                        remsd = next;
4821                }
4822        } else
4823#endif
4824                local_irq_enable();
4825}
4826
4827static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4828{
4829#ifdef CONFIG_RPS
4830        return sd->rps_ipi_list != NULL;
4831#else
4832        return false;
4833#endif
4834}
4835
4836static int process_backlog(struct napi_struct *napi, int quota)
4837{
4838        struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4839        bool again = true;
4840        int work = 0;
4841
4842        /* Check if we have pending ipi, its better to send them now,
4843         * not waiting net_rx_action() end.
4844         */
4845        if (sd_has_rps_ipi_waiting(sd)) {
4846                local_irq_disable();
4847                net_rps_action_and_irq_enable(sd);
4848        }
4849
4850        napi->weight = weight_p;
4851        while (again) {
4852                struct sk_buff *skb;
4853
4854                while ((skb = __skb_dequeue(&sd->process_queue))) {
4855                        rcu_read_lock();
4856                        __netif_receive_skb(skb);
4857                        rcu_read_unlock();
4858                        input_queue_head_incr(sd);
4859                        if (++work >= quota)
4860                                return work;
4861
4862                }
4863
4864                local_irq_disable();
4865                rps_lock(sd);
4866                if (skb_queue_empty(&sd->input_pkt_queue)) {
4867                        /*
4868                         * Inline a custom version of __napi_complete().
4869                         * only current cpu owns and manipulates this napi,
4870                         * and NAPI_STATE_SCHED is the only possible flag set
4871                         * on backlog.
4872                         * We can use a plain write instead of clear_bit(),
4873                         * and we dont need an smp_mb() memory barrier.
4874                         */
4875                        napi->state = 0;
4876                        again = false;
4877                } else {
4878                        skb_queue_splice_tail_init(&sd->input_pkt_queue,
4879                                                   &sd->process_queue);
4880                }
4881                rps_unlock(sd);
4882                local_irq_enable();
4883        }
4884
4885        return work;
4886}
4887
4888/**
4889 * __napi_schedule - schedule for receive
4890 * @n: entry to schedule
4891 *
4892 * The entry's receive function will be scheduled to run.
4893 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4894 */
4895void __napi_schedule(struct napi_struct *n)
4896{
4897        unsigned long flags;
4898
4899        local_irq_save(flags);
4900        ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4901        local_irq_restore(flags);
4902}
4903EXPORT_SYMBOL(__napi_schedule);
4904
4905/**
4906 * __napi_schedule_irqoff - schedule for receive
4907 * @n: entry to schedule
4908 *
4909 * Variant of __napi_schedule() assuming hard irqs are masked
4910 */
4911void __napi_schedule_irqoff(struct napi_struct *n)
4912{
4913        ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4914}
4915EXPORT_SYMBOL(__napi_schedule_irqoff);
4916
4917void __napi_complete(struct napi_struct *n)
4918{
4919        BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4920
4921        list_del_init(&n->poll_list);
4922        smp_mb__before_atomic();
4923        clear_bit(NAPI_STATE_SCHED, &n->state);
4924}
4925EXPORT_SYMBOL(__napi_complete);
4926
4927void napi_complete_done(struct napi_struct *n, int work_done)
4928{
4929        unsigned long flags;
4930
4931        /*
4932         * don't let napi dequeue from the cpu poll list
4933         * just in case its running on a different cpu
4934         */
4935        if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4936                return;
4937
4938        if (n->gro_list) {
4939                unsigned long timeout = 0;
4940
4941                if (work_done)
4942                        timeout = n->dev->gro_flush_timeout;
4943
4944                if (timeout)
4945                        hrtimer_start(&n->timer, ns_to_ktime(timeout),
4946                                      HRTIMER_MODE_REL_PINNED);
4947                else
4948                        napi_gro_flush(n, false);
4949        }
4950        if (likely(list_empty(&n->poll_list))) {
4951                WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4952        } else {
4953                /* If n->poll_list is not empty, we need to mask irqs */
4954                local_irq_save(flags);
4955                __napi_complete(n);
4956                local_irq_restore(flags);
4957        }
4958}
4959EXPORT_SYMBOL(napi_complete_done);
4960
4961/* must be called under rcu_read_lock(), as we dont take a reference */
4962static struct napi_struct *napi_by_id(unsigned int napi_id)
4963{
4964        unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4965        struct napi_struct *napi;
4966
4967        hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4968                if (napi->napi_id == napi_id)
4969                        return napi;
4970
4971        return NULL;
4972}
4973
4974#if defined(CONFIG_NET_RX_BUSY_POLL)
4975#define BUSY_POLL_BUDGET 8
4976bool sk_busy_loop(struct sock *sk, int nonblock)
4977{
4978        unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
4979        int (*busy_poll)(struct napi_struct *dev);
4980        struct napi_struct *napi;
4981        int rc = false;
4982
4983        rcu_read_lock();
4984
4985        napi = napi_by_id(sk->sk_napi_id);
4986        if (!napi)
4987                goto out;
4988
4989        /* Note: ndo_busy_poll method is optional in linux-4.5 */
4990        busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
4991
4992        do {
4993                rc = 0;
4994                local_bh_disable();
4995                if (busy_poll) {
4996                        rc = busy_poll(napi);
4997                } else if (napi_schedule_prep(napi)) {
4998                        void *have = netpoll_poll_lock(napi);
4999
5000                        if (test_bit(NAPI_STATE_SCHED, &napi->state)) {
5001                                rc = napi->poll(napi, BUSY_POLL_BUDGET);
5002                                trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
5003                                if (rc == BUSY_POLL_BUDGET) {
5004                                        napi_complete_done(napi, rc);
5005                                        napi_schedule(napi);
5006                                }
5007                        }
5008                        netpoll_poll_unlock(have);
5009                }
5010                if (rc > 0)
5011                        __NET_ADD_STATS(sock_net(sk),
5012                                        LINUX_MIB_BUSYPOLLRXPACKETS, rc);
5013                local_bh_enable();
5014
5015                if (rc == LL_FLUSH_FAILED)
5016                        break; /* permanent failure */
5017
5018                cpu_relax();
5019        } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) &&
5020                 !need_resched() && !busy_loop_timeout(end_time));
5021
5022        rc = !skb_queue_empty(&sk->sk_receive_queue);
5023out:
5024        rcu_read_unlock();
5025        return rc;
5026}
5027EXPORT_SYMBOL(sk_busy_loop);
5028
5029#endif /* CONFIG_NET_RX_BUSY_POLL */
5030
5031void napi_hash_add(struct napi_struct *napi)
5032{
5033        if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
5034            test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
5035                return;
5036
5037        spin_lock(&napi_hash_lock);
5038
5039        /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
5040        do {
5041                if (unlikely(++napi_gen_id < NR_CPUS + 1))
5042                        napi_gen_id = NR_CPUS + 1;
5043        } while (napi_by_id(napi_gen_id));
5044        napi->napi_id = napi_gen_id;
5045
5046        hlist_add_head_rcu(&napi->napi_hash_node,
5047                           &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
5048
5049        spin_unlock(&napi_hash_lock);
5050}
5051EXPORT_SYMBOL_GPL(napi_hash_add);
5052
5053/* Warning : caller is responsible to make sure rcu grace period
5054 * is respected before freeing memory containing @napi
5055 */
5056bool napi_hash_del(struct napi_struct *napi)
5057{
5058        bool rcu_sync_needed = false;
5059
5060        spin_lock(&napi_hash_lock);
5061
5062        if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
5063                rcu_sync_needed = true;
5064                hlist_del_rcu(&napi->napi_hash_node);
5065        }
5066        spin_unlock(&napi_hash_lock);
5067        return rcu_sync_needed;
5068}
5069EXPORT_SYMBOL_GPL(napi_hash_del);
5070
5071static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5072{
5073        struct napi_struct *napi;
5074
5075        napi = container_of(timer, struct napi_struct, timer);
5076        if (napi->gro_list)
5077                napi_schedule(napi);
5078
5079        return HRTIMER_NORESTART;
5080}
5081
5082void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5083                    int (*poll)(struct napi_struct *, int), int weight)
5084{
5085        INIT_LIST_HEAD(&napi->poll_list);
5086        hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5087        napi->timer.function = napi_watchdog;
5088        napi->gro_count = 0;
5089        napi->gro_list = NULL;
5090        napi->skb = NULL;
5091        napi->poll = poll;
5092        if (weight > NAPI_POLL_WEIGHT)
5093                pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5094                            weight, dev->name);
5095        napi->weight = weight;
5096        list_add(&napi->dev_list, &dev->napi_list);
5097        napi->dev = dev;
5098#ifdef CONFIG_NETPOLL
5099        spin_lock_init(&napi->poll_lock);
5100        napi->poll_owner = -1;
5101#endif
5102        set_bit(NAPI_STATE_SCHED, &napi->state);
5103        napi_hash_add(napi);
5104}
5105EXPORT_SYMBOL(netif_napi_add);
5106
5107void napi_disable(struct napi_struct *n)
5108{
5109        might_sleep();
5110        set_bit(NAPI_STATE_DISABLE, &n->state);
5111
5112        while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5113                msleep(1);
5114        while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5115                msleep(1);
5116
5117        hrtimer_cancel(&n->timer);
5118
5119        clear_bit(NAPI_STATE_DISABLE, &n->state);
5120}
5121EXPORT_SYMBOL(napi_disable);
5122
5123/* Must be called in process context */
5124void netif_napi_del(struct napi_struct *napi)
5125{
5126        might_sleep();
5127        if (napi_hash_del(napi))
5128                synchronize_net();
5129        list_del_init(&napi->dev_list);
5130        napi_free_frags(napi);
5131
5132        kfree_skb_list(napi->gro_list);
5133        napi->gro_list = NULL;
5134        napi->gro_count = 0;
5135}
5136EXPORT_SYMBOL(netif_napi_del);
5137
5138static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5139{
5140        void *have;
5141        int work, weight;
5142
5143        list_del_init(&n->poll_list);
5144
5145        have = netpoll_poll_lock(n);
5146
5147        weight = n->weight;
5148
5149        /* This NAPI_STATE_SCHED test is for avoiding a race
5150         * with netpoll's poll_napi().  Only the entity which
5151         * obtains the lock and sees NAPI_STATE_SCHED set will
5152         * actually make the ->poll() call.  Therefore we avoid
5153         * accidentally calling ->poll() when NAPI is not scheduled.
5154         */
5155        work = 0;
5156        if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5157                work = n->poll(n, weight);
5158                trace_napi_poll(n, work, weight);
5159        }
5160
5161        WARN_ON_ONCE(work > weight);
5162
5163        if (likely(work < weight))
5164                goto out_unlock;
5165
5166        /* Drivers must not modify the NAPI state if they
5167         * consume the entire weight.  In such cases this code
5168         * still "owns" the NAPI instance and therefore can
5169         * move the instance around on the list at-will.
5170         */
5171        if (unlikely(napi_disable_pending(n))) {
5172                napi_complete(n);
5173                goto out_unlock;
5174        }
5175
5176        if (n->gro_list) {
5177                /* flush too old packets
5178                 * If HZ < 1000, flush all packets.
5179                 */
5180                napi_gro_flush(n, HZ >= 1000);
5181        }
5182
5183        /* Some drivers may have called napi_schedule
5184         * prior to exhausting their budget.
5185         */
5186        if (unlikely(!list_empty(&n->poll_list))) {
5187                pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5188                             n->dev ? n->dev->name : "backlog");
5189                goto out_unlock;
5190        }
5191
5192        list_add_tail(&n->poll_list, repoll);
5193
5194out_unlock:
5195        netpoll_poll_unlock(have);
5196
5197        return work;
5198}
5199
5200static __latent_entropy void net_rx_action(struct softirq_action *h)
5201{
5202        struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5203        unsigned long time_limit = jiffies + 2;
5204        int budget = netdev_budget;
5205        LIST_HEAD(list);
5206        LIST_HEAD(repoll);
5207
5208        local_irq_disable();
5209        list_splice_init(&sd->poll_list, &list);
5210        local_irq_enable();
5211
5212        for (;;) {
5213                struct napi_struct *n;
5214
5215                if (list_empty(&list)) {
5216                        if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5217                                return;
5218                        break;
5219                }
5220
5221                n = list_first_entry(&list, struct napi_struct, poll_list);
5222                budget -= napi_poll(n, &repoll);
5223
5224                /* If softirq window is exhausted then punt.
5225                 * Allow this to run for 2 jiffies since which will allow
5226                 * an average latency of 1.5/HZ.
5227                 */
5228                if (unlikely(budget <= 0 ||
5229                             time_after_eq(jiffies, time_limit))) {
5230                        sd->time_squeeze++;
5231                        break;
5232                }
5233        }
5234
5235        __kfree_skb_flush();
5236        local_irq_disable();
5237
5238        list_splice_tail_init(&sd->poll_list, &list);
5239        list_splice_tail(&repoll, &list);
5240        list_splice(&list, &sd->poll_list);
5241        if (!list_empty(&sd->poll_list))
5242                __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5243
5244        net_rps_action_and_irq_enable(sd);
5245}
5246
5247struct netdev_adjacent {
5248        struct net_device *dev;
5249
5250        /* upper master flag, there can only be one master device per list */
5251        bool master;
5252
5253        /* counter for the number of times this device was added to us */
5254        u16 ref_nr;
5255
5256        /* private field for the users */
5257        void *private;
5258
5259        struct list_head list;
5260        struct rcu_head rcu;
5261};
5262
5263static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5264                                                 struct list_head *adj_list)
5265{
5266        struct netdev_adjacent *adj;
5267
5268        list_for_each_entry(adj, adj_list, list) {
5269                if (adj->dev == adj_dev)
5270                        return adj;
5271        }
5272        return NULL;
5273}
5274
5275/**
5276 * netdev_has_upper_dev - Check if device is linked to an upper device
5277 * @dev: device
5278 * @upper_dev: upper device to check
5279 *
5280 * Find out if a device is linked to specified upper device and return true
5281 * in case it is. Note that this checks only immediate upper device,
5282 * not through a complete stack of devices. The caller must hold the RTNL lock.
5283 */
5284bool netdev_has_upper_dev(struct net_device *dev,
5285                          struct net_device *upper_dev)
5286{
5287        ASSERT_RTNL();
5288
5289        return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
5290}
5291EXPORT_SYMBOL(netdev_has_upper_dev);
5292
5293/**
5294 * netdev_has_any_upper_dev - Check if device is linked to some device
5295 * @dev: device
5296 *
5297 * Find out if a device is linked to an upper device and return true in case
5298 * it is. The caller must hold the RTNL lock.
5299 */
5300static bool netdev_has_any_upper_dev(struct net_device *dev)
5301{
5302        ASSERT_RTNL();
5303
5304        return !list_empty(&dev->all_adj_list.upper);
5305}
5306
5307/**
5308 * netdev_master_upper_dev_get - Get master upper device
5309 * @dev: device
5310 *
5311 * Find a master upper device and return pointer to it or NULL in case
5312 * it's not there. The caller must hold the RTNL lock.
5313 */
5314struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5315{
5316        struct netdev_adjacent *upper;
5317
5318        ASSERT_RTNL();
5319
5320        if (list_empty(&dev->adj_list.upper))
5321                return NULL;
5322
5323        upper = list_first_entry(&dev->adj_list.upper,
5324                                 struct netdev_adjacent, list);
5325        if (likely(upper->master))
5326                return upper->dev;
5327        return NULL;
5328}
5329EXPORT_SYMBOL(netdev_master_upper_dev_get);
5330
5331void *netdev_adjacent_get_private(struct list_head *adj_list)
5332{
5333        struct netdev_adjacent *adj;
5334
5335        adj = list_entry(adj_list, struct netdev_adjacent, list);
5336
5337        return adj->private;
5338}
5339EXPORT_SYMBOL(netdev_adjacent_get_private);
5340
5341/**
5342 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5343 * @dev: device
5344 * @iter: list_head ** of the current position
5345 *
5346 * Gets the next device from the dev's upper list, starting from iter
5347 * position. The caller must hold RCU read lock.
5348 */
5349struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5350                                                 struct list_head **iter)
5351{
5352        struct netdev_adjacent *upper;
5353
5354        WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5355
5356        upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5357
5358        if (&upper->list == &dev->adj_list.upper)
5359                return NULL;
5360
5361        *iter = &upper->list;
5362
5363        return upper->dev;
5364}
5365EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5366
5367/**
5368 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
5369 * @dev: device
5370 * @iter: list_head ** of the current position
5371 *
5372 * Gets the next device from the dev's upper list, starting from iter
5373 * position. The caller must hold RCU read lock.
5374 */
5375struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5376                                                     struct list_head **iter)
5377{
5378        struct netdev_adjacent *upper;
5379
5380        WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5381
5382        upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5383
5384        if (&upper->list == &dev->all_adj_list.upper)
5385                return NULL;
5386
5387        *iter = &upper->list;
5388
5389        return upper->dev;
5390}
5391EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
5392
5393/**
5394 * netdev_lower_get_next_private - Get the next ->private from the
5395 *                                 lower neighbour list
5396 * @dev: device
5397 * @iter: list_head ** of the current position
5398 *
5399 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5400 * list, starting from iter position. The caller must hold either hold the
5401 * RTNL lock or its own locking that guarantees that the neighbour lower
5402 * list will remain unchanged.
5403 */
5404void *netdev_lower_get_next_private(struct net_device *dev,
5405                                    struct list_head **iter)
5406{
5407        struct netdev_adjacent *lower;
5408
5409        lower = list_entry(*iter, struct netdev_adjacent, list);
5410
5411        if (&lower->list == &dev->adj_list.lower)
5412                return NULL;
5413
5414        *iter = lower->list.next;
5415
5416        return lower->private;
5417}
5418EXPORT_SYMBOL(netdev_lower_get_next_private);
5419
5420/**
5421 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5422 *                                     lower neighbour list, RCU
5423 *                                     variant
5424 * @dev: device
5425 * @iter: list_head ** of the current position
5426 *
5427 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5428 * list, starting from iter position. The caller must hold RCU read lock.
5429 */
5430void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5431                                        struct list_head **iter)
5432{
5433        struct netdev_adjacent *lower;
5434
5435        WARN_ON_ONCE(!rcu_read_lock_held());
5436
5437        lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5438
5439        if (&lower->list == &dev->adj_list.lower)
5440                return NULL;
5441
5442        *iter = &lower->list;
5443
5444        return lower->private;
5445}
5446EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5447
5448/**
5449 * netdev_lower_get_next - Get the next device from the lower neighbour
5450 *                         list
5451 * @dev: device
5452 * @iter: list_head ** of the current position
5453 *
5454 * Gets the next netdev_adjacent from the dev's lower neighbour
5455 * list, starting from iter position. The caller must hold RTNL lock or
5456 * its own locking that guarantees that the neighbour lower
5457 * list will remain unchanged.
5458 */
5459void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5460{
5461        struct netdev_adjacent *lower;
5462
5463        lower = list_entry(*iter, struct netdev_adjacent, list);
5464
5465        if (&lower->list == &dev->adj_list.lower)
5466                return NULL;
5467
5468        *iter = lower->list.next;
5469
5470        return lower->dev;
5471}
5472EXPORT_SYMBOL(netdev_lower_get_next);
5473
5474/**
5475 * netdev_all_lower_get_next - Get the next device from all lower neighbour list
5476 * @dev: device
5477 * @iter: list_head ** of the current position
5478 *
5479 * Gets the next netdev_adjacent from the dev's all lower neighbour
5480 * list, starting from iter position. The caller must hold RTNL lock or
5481 * its own locking that guarantees that the neighbour all lower
5482 * list will remain unchanged.
5483 */
5484struct net_device *netdev_all_lower_get_next(struct net_device *dev, struct list_head **iter)
5485{
5486        struct netdev_adjacent *lower;
5487
5488        lower = list_entry(*iter, struct netdev_adjacent, list);
5489
5490        if (&lower->list == &dev->all_adj_list.lower)
5491                return NULL;
5492
5493        *iter = lower->list.next;
5494
5495        return lower->dev;
5496}
5497EXPORT_SYMBOL(netdev_all_lower_get_next);
5498
5499/**
5500 * netdev_all_lower_get_next_rcu - Get the next device from all
5501 *                                 lower neighbour list, RCU variant
5502 * @dev: device
5503 * @iter: list_head ** of the current position
5504 *
5505 * Gets the next netdev_adjacent from the dev's all lower neighbour
5506 * list, starting from iter position. The caller must hold RCU read lock.
5507 */
5508struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev,
5509                                                 struct list_head **iter)
5510{
5511        struct netdev_adjacent *lower;
5512
5513        lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5514
5515        if (&lower->list == &dev->all_adj_list.lower)
5516                return NULL;
5517
5518        *iter = &lower->list;
5519
5520        return lower->dev;
5521}
5522EXPORT_SYMBOL(netdev_all_lower_get_next_rcu);
5523
5524/**
5525 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5526 *                                     lower neighbour list, RCU
5527 *                                     variant
5528 * @dev: device
5529 *
5530 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5531 * list. The caller must hold RCU read lock.
5532 */
5533void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5534{
5535        struct netdev_adjacent *lower;
5536
5537        lower = list_first_or_null_rcu(&dev->adj_list.lower,
5538                        struct netdev_adjacent, list);
5539        if (lower)
5540                return lower->private;
5541        return NULL;
5542}
5543EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5544
5545/**
5546 * netdev_master_upper_dev_get_rcu - Get master upper device
5547 * @dev: device
5548 *
5549 * Find a master upper device and return pointer to it or NULL in case
5550 * it's not there. The caller must hold the RCU read lock.
5551 */
5552struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5553{
5554        struct netdev_adjacent *upper;
5555
5556        upper = list_first_or_null_rcu(&dev->adj_list.upper,
5557                                       struct netdev_adjacent, list);
5558        if (upper && likely(upper->master))
5559                return upper->dev;
5560        return NULL;
5561}
5562EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5563
5564static int netdev_adjacent_sysfs_add(struct net_device *dev,
5565                              struct net_device *adj_dev,
5566                              struct list_head *dev_list)
5567{
5568        char linkname[IFNAMSIZ+7];
5569        sprintf(linkname, dev_list == &dev->adj_list.upper ?
5570                "upper_%s" : "lower_%s", adj_dev->name);
5571        return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5572                                 linkname);
5573}
5574static void netdev_adjacent_sysfs_del(struct net_device *dev,
5575                               char *name,
5576                               struct list_head *dev_list)
5577{
5578        char linkname[IFNAMSIZ+7];
5579        sprintf(linkname, dev_list == &dev->adj_list.upper ?
5580                "upper_%s" : "lower_%s", name);
5581        sysfs_remove_link(&(dev->dev.kobj), linkname);
5582}
5583
5584static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5585                                                 struct net_device *adj_dev,
5586                                                 struct list_head *dev_list)
5587{
5588        return (dev_list == &dev->adj_list.upper ||
5589                dev_list == &dev->adj_list.lower) &&
5590                net_eq(dev_net(dev), dev_net(adj_dev));
5591}
5592
5593static int __netdev_adjacent_dev_insert(struct net_device *dev,
5594                                        struct net_device *adj_dev,
5595                                        u16 ref_nr,
5596                                        struct list_head *dev_list,
5597                                        void *private, bool master)
5598{
5599        struct netdev_adjacent *adj;
5600        int ret;
5601
5602        adj = __netdev_find_adj(adj_dev, dev_list);
5603
5604        if (adj) {
5605                adj->ref_nr += ref_nr;
5606                return 0;
5607        }
5608
5609        adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5610        if (!adj)
5611                return -ENOMEM;
5612
5613        adj->dev = adj_dev;
5614        adj->master = master;
5615        adj->ref_nr = ref_nr;
5616        adj->private = private;
5617        dev_hold(adj_dev);
5618
5619        pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5620                 adj_dev->name, dev->name, adj_dev->name);
5621
5622        if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5623                ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5624                if (ret)
5625                        goto free_adj;
5626        }
5627
5628        /* Ensure that master link is always the first item in list. */
5629        if (master) {
5630                ret = sysfs_create_link(&(dev->dev.kobj),
5631                                        &(adj_dev->dev.kobj), "master");
5632                if (ret)
5633                        goto remove_symlinks;
5634
5635                list_add_rcu(&adj->list, dev_list);
5636        } else {
5637                list_add_tail_rcu(&adj->list, dev_list);
5638        }
5639
5640        return 0;
5641
5642remove_symlinks:
5643        if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5644                netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5645free_adj:
5646        kfree(adj);
5647        dev_put(adj_dev);
5648
5649        return ret;
5650}
5651
5652static void __netdev_adjacent_dev_remove(struct net_device *dev,
5653                                         struct