2 * NET3 Protocol independent device support routines.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Derived from the non IP parts of dev.c 1.0.19
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
51 * Rudi Cilibrasi : Pass the right thing to
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
75 #include <asm/uaccess.h>
76 #include <asm/system.h>
77 #include <linux/bitops.h>
78 #include <linux/capability.h>
79 #include <linux/cpu.h>
80 #include <linux/types.h>
81 #include <linux/kernel.h>
82 #include <linux/hash.h>
83 #include <linux/slab.h>
84 #include <linux/sched.h>
85 #include <linux/mutex.h>
86 #include <linux/string.h>
88 #include <linux/socket.h>
89 #include <linux/sockios.h>
90 #include <linux/errno.h>
91 #include <linux/interrupt.h>
92 #include <linux/if_ether.h>
93 #include <linux/netdevice.h>
94 #include <linux/etherdevice.h>
95 #include <linux/ethtool.h>
96 #include <linux/notifier.h>
97 #include <linux/skbuff.h>
98 #include <net/net_namespace.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/proc_fs.h>
102 #include <linux/seq_file.h>
103 #include <linux/stat.h>
105 #include <net/pkt_sched.h>
106 #include <net/checksum.h>
107 #include <net/xfrm.h>
108 #include <linux/highmem.h>
109 #include <linux/init.h>
110 #include <linux/kmod.h>
111 #include <linux/module.h>
112 #include <linux/netpoll.h>
113 #include <linux/rcupdate.h>
114 #include <linux/delay.h>
115 #include <net/wext.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
126 #include <linux/ipv6.h>
127 #include <linux/in.h>
128 #include <linux/jhash.h>
129 #include <linux/random.h>
130 #include <trace/events/napi.h>
131 #include <trace/events/net.h>
132 #include <trace/events/skb.h>
133 #include <linux/pci.h>
134 #include <linux/inetdevice.h>
136 #include "net-sysfs.h"
138 /* Instead of increasing this, you should create a hash table. */
139 #define MAX_GRO_SKBS 8
141 /* This should be increased if a protocol with a bigger head is added. */
142 #define GRO_MAX_HEAD (MAX_HEADER + 128)
145 * The list of packet types we will receive (as opposed to discard)
146 * and the routines to invoke.
148 * Why 16. Because with 16 the only overlap we get on a hash of the
149 * low nibble of the protocol value is RARP/SNAP/X.25.
151 * NOTE: That is no longer true with the addition of VLAN tags. Not
152 * sure which should go first, but I bet it won't make much
153 * difference if we are running VLANs. The good news is that
154 * this protocol won't be in the list unless compiled in, so
155 * the average user (w/out VLANs) will not be adversely affected.
172 #define PTYPE_HASH_SIZE (16)
173 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
175 static DEFINE_SPINLOCK(ptype_lock);
176 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
177 static struct list_head ptype_all __read_mostly; /* Taps */
180 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
183 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
185 * Writers must hold the rtnl semaphore while they loop through the
186 * dev_base_head list, and hold dev_base_lock for writing when they do the
187 * actual updates. This allows pure readers to access the list even
188 * while a writer is preparing to update it.
190 * To put it another way, dev_base_lock is held for writing only to
191 * protect against pure readers; the rtnl semaphore provides the
192 * protection against other writers.
194 * See, for example usages, register_netdevice() and
195 * unregister_netdevice(), which must be called with the rtnl
198 DEFINE_RWLOCK(dev_base_lock);
199 EXPORT_SYMBOL(dev_base_lock);
201 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
203 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
204 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
207 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
209 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
212 static inline void rps_lock(struct softnet_data *sd)
215 spin_lock(&sd->input_pkt_queue.lock);
219 static inline void rps_unlock(struct softnet_data *sd)
222 spin_unlock(&sd->input_pkt_queue.lock);
226 /* Device list insertion */
227 static int list_netdevice(struct net_device *dev)
229 struct net *net = dev_net(dev);
233 write_lock_bh(&dev_base_lock);
234 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
235 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
236 hlist_add_head_rcu(&dev->index_hlist,
237 dev_index_hash(net, dev->ifindex));
238 write_unlock_bh(&dev_base_lock);
242 /* Device list removal
243 * caller must respect a RCU grace period before freeing/reusing dev
245 static void unlist_netdevice(struct net_device *dev)
249 /* Unlink dev from the device chain */
250 write_lock_bh(&dev_base_lock);
251 list_del_rcu(&dev->dev_list);
252 hlist_del_rcu(&dev->name_hlist);
253 hlist_del_rcu(&dev->index_hlist);
254 write_unlock_bh(&dev_base_lock);
261 static RAW_NOTIFIER_HEAD(netdev_chain);
264 * Device drivers call our routines to queue packets here. We empty the
265 * queue in the local softnet handler.
268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
269 EXPORT_PER_CPU_SYMBOL(softnet_data);
271 #ifdef CONFIG_LOCKDEP
273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
274 * according to dev->type
276 static const unsigned short netdev_lock_type[] =
277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
290 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
291 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
292 ARPHRD_VOID, ARPHRD_NONE};
294 static const char *const netdev_lock_name[] =
295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
307 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
308 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
309 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
310 "_xmit_VOID", "_xmit_NONE"};
312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
315 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
320 if (netdev_lock_type[i] == dev_type)
322 /* the last key is used by default */
323 return ARRAY_SIZE(netdev_lock_type) - 1;
326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
327 unsigned short dev_type)
331 i = netdev_lock_pos(dev_type);
332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
333 netdev_lock_name[i]);
336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
340 i = netdev_lock_pos(dev->type);
341 lockdep_set_class_and_name(&dev->addr_list_lock,
342 &netdev_addr_lock_key[i],
343 netdev_lock_name[i]);
346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
347 unsigned short dev_type)
350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
355 /*******************************************************************************
357 Protocol management and registration routines
359 *******************************************************************************/
362 * Add a protocol ID to the list. Now that the input handler is
363 * smarter we can dispense with all the messy stuff that used to be
366 * BEWARE!!! Protocol handlers, mangling input packets,
367 * MUST BE last in hash buckets and checking protocol handlers
368 * MUST start from promiscuous ptype_all chain in net_bh.
369 * It is true now, do not change it.
370 * Explanation follows: if protocol handler, mangling packet, will
371 * be the first on list, it is not able to sense, that packet
372 * is cloned and should be copied-on-write, so that it will
373 * change it and subsequent readers will get broken packet.
377 static inline struct list_head *ptype_head(const struct packet_type *pt)
379 if (pt->type == htons(ETH_P_ALL))
382 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
386 * dev_add_pack - add packet handler
387 * @pt: packet type declaration
389 * Add a protocol handler to the networking stack. The passed &packet_type
390 * is linked into kernel lists and may not be freed until it has been
391 * removed from the kernel lists.
393 * This call does not sleep therefore it can not
394 * guarantee all CPU's that are in middle of receiving packets
395 * will see the new packet type (until the next received packet).
398 void dev_add_pack(struct packet_type *pt)
400 struct list_head *head = ptype_head(pt);
402 spin_lock(&ptype_lock);
403 list_add_rcu(&pt->list, head);
404 spin_unlock(&ptype_lock);
406 EXPORT_SYMBOL(dev_add_pack);
409 * __dev_remove_pack - remove packet handler
410 * @pt: packet type declaration
412 * Remove a protocol handler that was previously added to the kernel
413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
414 * from the kernel lists and can be freed or reused once this function
417 * The packet type might still be in use by receivers
418 * and must not be freed until after all the CPU's have gone
419 * through a quiescent state.
421 void __dev_remove_pack(struct packet_type *pt)
423 struct list_head *head = ptype_head(pt);
424 struct packet_type *pt1;
426 spin_lock(&ptype_lock);
428 list_for_each_entry(pt1, head, list) {
430 list_del_rcu(&pt->list);
435 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
437 spin_unlock(&ptype_lock);
439 EXPORT_SYMBOL(__dev_remove_pack);
442 * dev_remove_pack - remove packet handler
443 * @pt: packet type declaration
445 * Remove a protocol handler that was previously added to the kernel
446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
447 * from the kernel lists and can be freed or reused once this function
450 * This call sleeps to guarantee that no CPU is looking at the packet
453 void dev_remove_pack(struct packet_type *pt)
455 __dev_remove_pack(pt);
459 EXPORT_SYMBOL(dev_remove_pack);
461 /******************************************************************************
463 Device Boot-time Settings Routines
465 *******************************************************************************/
467 /* Boot time configuration table */
468 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
471 * netdev_boot_setup_add - add new setup entry
472 * @name: name of the device
473 * @map: configured settings for the device
475 * Adds new setup entry to the dev_boot_setup list. The function
476 * returns 0 on error and 1 on success. This is a generic routine to
479 static int netdev_boot_setup_add(char *name, struct ifmap *map)
481 struct netdev_boot_setup *s;
485 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
486 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
487 memset(s[i].name, 0, sizeof(s[i].name));
488 strlcpy(s[i].name, name, IFNAMSIZ);
489 memcpy(&s[i].map, map, sizeof(s[i].map));
494 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
498 * netdev_boot_setup_check - check boot time settings
499 * @dev: the netdevice
501 * Check boot time settings for the device.
502 * The found settings are set for the device to be used
503 * later in the device probing.
504 * Returns 0 if no settings found, 1 if they are.
506 int netdev_boot_setup_check(struct net_device *dev)
508 struct netdev_boot_setup *s = dev_boot_setup;
511 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
512 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
513 !strcmp(dev->name, s[i].name)) {
514 dev->irq = s[i].map.irq;
515 dev->base_addr = s[i].map.base_addr;
516 dev->mem_start = s[i].map.mem_start;
517 dev->mem_end = s[i].map.mem_end;
523 EXPORT_SYMBOL(netdev_boot_setup_check);
527 * netdev_boot_base - get address from boot time settings
528 * @prefix: prefix for network device
529 * @unit: id for network device
531 * Check boot time settings for the base address of device.
532 * The found settings are set for the device to be used
533 * later in the device probing.
534 * Returns 0 if no settings found.
536 unsigned long netdev_boot_base(const char *prefix, int unit)
538 const struct netdev_boot_setup *s = dev_boot_setup;
542 sprintf(name, "%s%d", prefix, unit);
545 * If device already registered then return base of 1
546 * to indicate not to probe for this interface
548 if (__dev_get_by_name(&init_net, name))
551 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
552 if (!strcmp(name, s[i].name))
553 return s[i].map.base_addr;
558 * Saves at boot time configured settings for any netdevice.
560 int __init netdev_boot_setup(char *str)
565 str = get_options(str, ARRAY_SIZE(ints), ints);
570 memset(&map, 0, sizeof(map));
574 map.base_addr = ints[2];
576 map.mem_start = ints[3];
578 map.mem_end = ints[4];
580 /* Add new entry to the list */
581 return netdev_boot_setup_add(str, &map);
584 __setup("netdev=", netdev_boot_setup);
586 /*******************************************************************************
588 Device Interface Subroutines
590 *******************************************************************************/
593 * __dev_get_by_name - find a device by its name
594 * @net: the applicable net namespace
595 * @name: name to find
597 * Find an interface by name. Must be called under RTNL semaphore
598 * or @dev_base_lock. If the name is found a pointer to the device
599 * is returned. If the name is not found then %NULL is returned. The
600 * reference counters are not incremented so the caller must be
601 * careful with locks.
604 struct net_device *__dev_get_by_name(struct net *net, const char *name)
606 struct hlist_node *p;
607 struct net_device *dev;
608 struct hlist_head *head = dev_name_hash(net, name);
610 hlist_for_each_entry(dev, p, head, name_hlist)
611 if (!strncmp(dev->name, name, IFNAMSIZ))
616 EXPORT_SYMBOL(__dev_get_by_name);
619 * dev_get_by_name_rcu - find a device by its name
620 * @net: the applicable net namespace
621 * @name: name to find
623 * Find an interface by name.
624 * If the name is found a pointer to the device is returned.
625 * If the name is not found then %NULL is returned.
626 * The reference counters are not incremented so the caller must be
627 * careful with locks. The caller must hold RCU lock.
630 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
632 struct hlist_node *p;
633 struct net_device *dev;
634 struct hlist_head *head = dev_name_hash(net, name);
636 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
637 if (!strncmp(dev->name, name, IFNAMSIZ))
642 EXPORT_SYMBOL(dev_get_by_name_rcu);
645 * dev_get_by_name - find a device by its name
646 * @net: the applicable net namespace
647 * @name: name to find
649 * Find an interface by name. This can be called from any
650 * context and does its own locking. The returned handle has
651 * the usage count incremented and the caller must use dev_put() to
652 * release it when it is no longer needed. %NULL is returned if no
653 * matching device is found.
656 struct net_device *dev_get_by_name(struct net *net, const char *name)
658 struct net_device *dev;
661 dev = dev_get_by_name_rcu(net, name);
667 EXPORT_SYMBOL(dev_get_by_name);
670 * __dev_get_by_index - find a device by its ifindex
671 * @net: the applicable net namespace
672 * @ifindex: index of device
674 * Search for an interface by index. Returns %NULL if the device
675 * is not found or a pointer to the device. The device has not
676 * had its reference counter increased so the caller must be careful
677 * about locking. The caller must hold either the RTNL semaphore
681 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
683 struct hlist_node *p;
684 struct net_device *dev;
685 struct hlist_head *head = dev_index_hash(net, ifindex);
687 hlist_for_each_entry(dev, p, head, index_hlist)
688 if (dev->ifindex == ifindex)
693 EXPORT_SYMBOL(__dev_get_by_index);
696 * dev_get_by_index_rcu - find a device by its ifindex
697 * @net: the applicable net namespace
698 * @ifindex: index of device
700 * Search for an interface by index. Returns %NULL if the device
701 * is not found or a pointer to the device. The device has not
702 * had its reference counter increased so the caller must be careful
703 * about locking. The caller must hold RCU lock.
706 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
708 struct hlist_node *p;
709 struct net_device *dev;
710 struct hlist_head *head = dev_index_hash(net, ifindex);
712 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
713 if (dev->ifindex == ifindex)
718 EXPORT_SYMBOL(dev_get_by_index_rcu);
722 * dev_get_by_index - find a device by its ifindex
723 * @net: the applicable net namespace
724 * @ifindex: index of device
726 * Search for an interface by index. Returns NULL if the device
727 * is not found or a pointer to the device. The device returned has
728 * had a reference added and the pointer is safe until the user calls
729 * dev_put to indicate they have finished with it.
732 struct net_device *dev_get_by_index(struct net *net, int ifindex)
734 struct net_device *dev;
737 dev = dev_get_by_index_rcu(net, ifindex);
743 EXPORT_SYMBOL(dev_get_by_index);
746 * dev_getbyhwaddr_rcu - find a device by its hardware address
747 * @net: the applicable net namespace
748 * @type: media type of device
749 * @ha: hardware address
751 * Search for an interface by MAC address. Returns NULL if the device
752 * is not found or a pointer to the device.
753 * The caller must hold RCU or RTNL.
754 * The returned device has not had its ref count increased
755 * and the caller must therefore be careful about locking
759 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
762 struct net_device *dev;
764 for_each_netdev_rcu(net, dev)
765 if (dev->type == type &&
766 !memcmp(dev->dev_addr, ha, dev->addr_len))
771 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
773 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
775 struct net_device *dev;
778 for_each_netdev(net, dev)
779 if (dev->type == type)
784 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
786 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
788 struct net_device *dev, *ret = NULL;
791 for_each_netdev_rcu(net, dev)
792 if (dev->type == type) {
800 EXPORT_SYMBOL(dev_getfirstbyhwtype);
803 * dev_get_by_flags_rcu - find any device with given flags
804 * @net: the applicable net namespace
805 * @if_flags: IFF_* values
806 * @mask: bitmask of bits in if_flags to check
808 * Search for any interface with the given flags. Returns NULL if a device
809 * is not found or a pointer to the device. Must be called inside
810 * rcu_read_lock(), and result refcount is unchanged.
813 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
816 struct net_device *dev, *ret;
819 for_each_netdev_rcu(net, dev) {
820 if (((dev->flags ^ if_flags) & mask) == 0) {
827 EXPORT_SYMBOL(dev_get_by_flags_rcu);
830 * dev_valid_name - check if name is okay for network device
833 * Network device names need to be valid file names to
834 * to allow sysfs to work. We also disallow any kind of
837 int dev_valid_name(const char *name)
841 if (strlen(name) >= IFNAMSIZ)
843 if (!strcmp(name, ".") || !strcmp(name, ".."))
847 if (*name == '/' || isspace(*name))
853 EXPORT_SYMBOL(dev_valid_name);
856 * __dev_alloc_name - allocate a name for a device
857 * @net: network namespace to allocate the device name in
858 * @name: name format string
859 * @buf: scratch buffer and result name string
861 * Passed a format string - eg "lt%d" it will try and find a suitable
862 * id. It scans list of devices to build up a free map, then chooses
863 * the first empty slot. The caller must hold the dev_base or rtnl lock
864 * while allocating the name and adding the device in order to avoid
866 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
867 * Returns the number of the unit assigned or a negative errno code.
870 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
874 const int max_netdevices = 8*PAGE_SIZE;
875 unsigned long *inuse;
876 struct net_device *d;
878 p = strnchr(name, IFNAMSIZ-1, '%');
881 * Verify the string as this thing may have come from
882 * the user. There must be either one "%d" and no other "%"
885 if (p[1] != 'd' || strchr(p + 2, '%'))
888 /* Use one page as a bit array of possible slots */
889 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
893 for_each_netdev(net, d) {
894 if (!sscanf(d->name, name, &i))
896 if (i < 0 || i >= max_netdevices)
899 /* avoid cases where sscanf is not exact inverse of printf */
900 snprintf(buf, IFNAMSIZ, name, i);
901 if (!strncmp(buf, d->name, IFNAMSIZ))
905 i = find_first_zero_bit(inuse, max_netdevices);
906 free_page((unsigned long) inuse);
910 snprintf(buf, IFNAMSIZ, name, i);
911 if (!__dev_get_by_name(net, buf))
914 /* It is possible to run out of possible slots
915 * when the name is long and there isn't enough space left
916 * for the digits, or if all bits are used.
922 * dev_alloc_name - allocate a name for a device
924 * @name: name format string
926 * Passed a format string - eg "lt%d" it will try and find a suitable
927 * id. It scans list of devices to build up a free map, then chooses
928 * the first empty slot. The caller must hold the dev_base or rtnl lock
929 * while allocating the name and adding the device in order to avoid
931 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
932 * Returns the number of the unit assigned or a negative errno code.
935 int dev_alloc_name(struct net_device *dev, const char *name)
941 BUG_ON(!dev_net(dev));
943 ret = __dev_alloc_name(net, name, buf);
945 strlcpy(dev->name, buf, IFNAMSIZ);
948 EXPORT_SYMBOL(dev_alloc_name);
950 static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
954 BUG_ON(!dev_net(dev));
957 if (!dev_valid_name(name))
960 if (fmt && strchr(name, '%'))
961 return dev_alloc_name(dev, name);
962 else if (__dev_get_by_name(net, name))
964 else if (dev->name != name)
965 strlcpy(dev->name, name, IFNAMSIZ);
971 * dev_change_name - change name of a device
973 * @newname: name (or format string) must be at least IFNAMSIZ
975 * Change name of a device, can pass format strings "eth%d".
978 int dev_change_name(struct net_device *dev, const char *newname)
980 char oldname[IFNAMSIZ];
986 BUG_ON(!dev_net(dev));
989 if (dev->flags & IFF_UP)
992 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
995 memcpy(oldname, dev->name, IFNAMSIZ);
997 err = dev_get_valid_name(dev, newname, 1);
1002 ret = device_rename(&dev->dev, dev->name);
1004 memcpy(dev->name, oldname, IFNAMSIZ);
1008 write_lock_bh(&dev_base_lock);
1009 hlist_del(&dev->name_hlist);
1010 write_unlock_bh(&dev_base_lock);
1014 write_lock_bh(&dev_base_lock);
1015 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1016 write_unlock_bh(&dev_base_lock);
1018 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1019 ret = notifier_to_errno(ret);
1022 /* err >= 0 after dev_alloc_name() or stores the first errno */
1025 memcpy(dev->name, oldname, IFNAMSIZ);
1029 "%s: name change rollback failed: %d.\n",
1038 * dev_set_alias - change ifalias of a device
1040 * @alias: name up to IFALIASZ
1041 * @len: limit of bytes to copy from info
1043 * Set ifalias for a device,
1045 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1049 if (len >= IFALIASZ)
1054 kfree(dev->ifalias);
1055 dev->ifalias = NULL;
1060 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1064 strlcpy(dev->ifalias, alias, len+1);
1070 * netdev_features_change - device changes features
1071 * @dev: device to cause notification
1073 * Called to indicate a device has changed features.
1075 void netdev_features_change(struct net_device *dev)
1077 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1079 EXPORT_SYMBOL(netdev_features_change);
1082 * netdev_state_change - device changes state
1083 * @dev: device to cause notification
1085 * Called to indicate a device has changed state. This function calls
1086 * the notifier chains for netdev_chain and sends a NEWLINK message
1087 * to the routing socket.
1089 void netdev_state_change(struct net_device *dev)
1091 if (dev->flags & IFF_UP) {
1092 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1093 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1096 EXPORT_SYMBOL(netdev_state_change);
1098 int netdev_bonding_change(struct net_device *dev, unsigned long event)
1100 return call_netdevice_notifiers(event, dev);
1102 EXPORT_SYMBOL(netdev_bonding_change);
1105 * dev_load - load a network module
1106 * @net: the applicable net namespace
1107 * @name: name of interface
1109 * If a network interface is not present and the process has suitable
1110 * privileges this function loads the module. If module loading is not
1111 * available in this kernel then it becomes a nop.
1114 void dev_load(struct net *net, const char *name)
1116 struct net_device *dev;
1120 dev = dev_get_by_name_rcu(net, name);
1124 if (no_module && capable(CAP_NET_ADMIN))
1125 no_module = request_module("netdev-%s", name);
1126 if (no_module && capable(CAP_SYS_MODULE)) {
1127 if (!request_module("%s", name))
1128 pr_err("Loading kernel module for a network device "
1129 "with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s "
1133 EXPORT_SYMBOL(dev_load);
1135 static int __dev_open(struct net_device *dev)
1137 const struct net_device_ops *ops = dev->netdev_ops;
1143 * Is it even present?
1145 if (!netif_device_present(dev))
1148 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1149 ret = notifier_to_errno(ret);
1154 * Call device private open method
1156 set_bit(__LINK_STATE_START, &dev->state);
1158 if (ops->ndo_validate_addr)
1159 ret = ops->ndo_validate_addr(dev);
1161 if (!ret && ops->ndo_open)
1162 ret = ops->ndo_open(dev);
1165 * If it went open OK then:
1169 clear_bit(__LINK_STATE_START, &dev->state);
1174 dev->flags |= IFF_UP;
1179 net_dmaengine_get();
1182 * Initialize multicasting status
1184 dev_set_rx_mode(dev);
1187 * Wakeup transmit queue engine
1196 * dev_open - prepare an interface for use.
1197 * @dev: device to open
1199 * Takes a device from down to up state. The device's private open
1200 * function is invoked and then the multicast lists are loaded. Finally
1201 * the device is moved into the up state and a %NETDEV_UP message is
1202 * sent to the netdev notifier chain.
1204 * Calling this function on an active interface is a nop. On a failure
1205 * a negative errno code is returned.
1207 int dev_open(struct net_device *dev)
1214 if (dev->flags & IFF_UP)
1220 ret = __dev_open(dev);
1225 * ... and announce new interface.
1227 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1228 call_netdevice_notifiers(NETDEV_UP, dev);
1232 EXPORT_SYMBOL(dev_open);
1234 static int __dev_close_many(struct list_head *head)
1236 struct net_device *dev;
1241 list_for_each_entry(dev, head, unreg_list) {
1243 * Tell people we are going down, so that they can
1244 * prepare to death, when device is still operating.
1246 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1248 clear_bit(__LINK_STATE_START, &dev->state);
1250 /* Synchronize to scheduled poll. We cannot touch poll list, it
1251 * can be even on different cpu. So just clear netif_running().
1253 * dev->stop() will invoke napi_disable() on all of it's
1254 * napi_struct instances on this device.
1256 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1259 dev_deactivate_many(head);
1261 list_for_each_entry(dev, head, unreg_list) {
1262 const struct net_device_ops *ops = dev->netdev_ops;
1265 * Call the device specific close. This cannot fail.
1266 * Only if device is UP
1268 * We allow it to be called even after a DETACH hot-plug
1275 * Device is now down.
1278 dev->flags &= ~IFF_UP;
1283 net_dmaengine_put();
1289 static int __dev_close(struct net_device *dev)
1294 list_add(&dev->unreg_list, &single);
1295 retval = __dev_close_many(&single);
1300 int dev_close_many(struct list_head *head)
1302 struct net_device *dev, *tmp;
1303 LIST_HEAD(tmp_list);
1305 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1306 if (!(dev->flags & IFF_UP))
1307 list_move(&dev->unreg_list, &tmp_list);
1309 __dev_close_many(head);
1312 * Tell people we are down
1314 list_for_each_entry(dev, head, unreg_list) {
1315 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1316 call_netdevice_notifiers(NETDEV_DOWN, dev);
1319 /* rollback_registered_many needs the complete original list */
1320 list_splice(&tmp_list, head);
1325 * dev_close - shutdown an interface.
1326 * @dev: device to shutdown
1328 * This function moves an active device into down state. A
1329 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1330 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1333 int dev_close(struct net_device *dev)
1335 if (dev->flags & IFF_UP) {
1338 list_add(&dev->unreg_list, &single);
1339 dev_close_many(&single);
1344 EXPORT_SYMBOL(dev_close);
1348 * dev_disable_lro - disable Large Receive Offload on a device
1351 * Disable Large Receive Offload (LRO) on a net device. Must be
1352 * called under RTNL. This is needed if received packets may be
1353 * forwarded to another interface.
1355 void dev_disable_lro(struct net_device *dev)
1357 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1358 dev->ethtool_ops->set_flags) {
1359 u32 flags = dev->ethtool_ops->get_flags(dev);
1360 if (flags & ETH_FLAG_LRO) {
1361 flags &= ~ETH_FLAG_LRO;
1362 dev->ethtool_ops->set_flags(dev, flags);
1365 WARN_ON(dev->features & NETIF_F_LRO);
1367 EXPORT_SYMBOL(dev_disable_lro);
1370 static int dev_boot_phase = 1;
1373 * Device change register/unregister. These are not inline or static
1374 * as we export them to the world.
1378 * register_netdevice_notifier - register a network notifier block
1381 * Register a notifier to be called when network device events occur.
1382 * The notifier passed is linked into the kernel structures and must
1383 * not be reused until it has been unregistered. A negative errno code
1384 * is returned on a failure.
1386 * When registered all registration and up events are replayed
1387 * to the new notifier to allow device to have a race free
1388 * view of the network device list.
1391 int register_netdevice_notifier(struct notifier_block *nb)
1393 struct net_device *dev;
1394 struct net_device *last;
1399 err = raw_notifier_chain_register(&netdev_chain, nb);
1405 for_each_netdev(net, dev) {
1406 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1407 err = notifier_to_errno(err);
1411 if (!(dev->flags & IFF_UP))
1414 nb->notifier_call(nb, NETDEV_UP, dev);
1425 for_each_netdev(net, dev) {
1429 if (dev->flags & IFF_UP) {
1430 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1431 nb->notifier_call(nb, NETDEV_DOWN, dev);
1433 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1434 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1438 raw_notifier_chain_unregister(&netdev_chain, nb);
1441 EXPORT_SYMBOL(register_netdevice_notifier);
1444 * unregister_netdevice_notifier - unregister a network notifier block
1447 * Unregister a notifier previously registered by
1448 * register_netdevice_notifier(). The notifier is unlinked into the
1449 * kernel structures and may then be reused. A negative errno code
1450 * is returned on a failure.
1453 int unregister_netdevice_notifier(struct notifier_block *nb)
1458 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1462 EXPORT_SYMBOL(unregister_netdevice_notifier);
1465 * call_netdevice_notifiers - call all network notifier blocks
1466 * @val: value passed unmodified to notifier function
1467 * @dev: net_device pointer passed unmodified to notifier function
1469 * Call all network notifier blocks. Parameters and return value
1470 * are as for raw_notifier_call_chain().
1473 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1476 return raw_notifier_call_chain(&netdev_chain, val, dev);
1479 /* When > 0 there are consumers of rx skb time stamps */
1480 static atomic_t netstamp_needed = ATOMIC_INIT(0);
1482 void net_enable_timestamp(void)
1484 atomic_inc(&netstamp_needed);
1486 EXPORT_SYMBOL(net_enable_timestamp);
1488 void net_disable_timestamp(void)
1490 atomic_dec(&netstamp_needed);
1492 EXPORT_SYMBOL(net_disable_timestamp);
1494 static inline void net_timestamp_set(struct sk_buff *skb)
1496 if (atomic_read(&netstamp_needed))
1497 __net_timestamp(skb);
1499 skb->tstamp.tv64 = 0;
1502 static inline void net_timestamp_check(struct sk_buff *skb)
1504 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1505 __net_timestamp(skb);
1509 * dev_forward_skb - loopback an skb to another netif
1511 * @dev: destination network device
1512 * @skb: buffer to forward
1515 * NET_RX_SUCCESS (no congestion)
1516 * NET_RX_DROP (packet was dropped, but freed)
1518 * dev_forward_skb can be used for injecting an skb from the
1519 * start_xmit function of one device into the receive queue
1520 * of another device.
1522 * The receiving device may be in another namespace, so
1523 * we have to clear all information in the skb that could
1524 * impact namespace isolation.
1526 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1531 if (unlikely(!(dev->flags & IFF_UP) ||
1532 (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN)))) {
1533 atomic_long_inc(&dev->rx_dropped);
1537 skb_set_dev(skb, dev);
1538 skb->tstamp.tv64 = 0;
1539 skb->pkt_type = PACKET_HOST;
1540 skb->protocol = eth_type_trans(skb, dev);
1541 return netif_rx(skb);
1543 EXPORT_SYMBOL_GPL(dev_forward_skb);
1545 static inline int deliver_skb(struct sk_buff *skb,
1546 struct packet_type *pt_prev,
1547 struct net_device *orig_dev)
1549 atomic_inc(&skb->users);
1550 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1554 * Support routine. Sends outgoing frames to any network
1555 * taps currently in use.
1558 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1560 struct packet_type *ptype;
1561 struct sk_buff *skb2 = NULL;
1562 struct packet_type *pt_prev = NULL;
1565 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1566 /* Never send packets back to the socket
1567 * they originated from - MvS (miquels@drinkel.ow.org)
1569 if ((ptype->dev == dev || !ptype->dev) &&
1570 (ptype->af_packet_priv == NULL ||
1571 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1573 deliver_skb(skb2, pt_prev, skb->dev);
1578 skb2 = skb_clone(skb, GFP_ATOMIC);
1582 net_timestamp_set(skb2);
1584 /* skb->nh should be correctly
1585 set by sender, so that the second statement is
1586 just protection against buggy protocols.
1588 skb_reset_mac_header(skb2);
1590 if (skb_network_header(skb2) < skb2->data ||
1591 skb2->network_header > skb2->tail) {
1592 if (net_ratelimit())
1593 printk(KERN_CRIT "protocol %04x is "
1595 ntohs(skb2->protocol),
1597 skb_reset_network_header(skb2);
1600 skb2->transport_header = skb2->network_header;
1601 skb2->pkt_type = PACKET_OUTGOING;
1606 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1611 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1612 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1614 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1618 if (txq < 1 || txq > dev->num_tx_queues)
1621 if (dev->reg_state == NETREG_REGISTERED) {
1624 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1629 if (txq < dev->real_num_tx_queues)
1630 qdisc_reset_all_tx_gt(dev, txq);
1633 dev->real_num_tx_queues = txq;
1636 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
1640 * netif_set_real_num_rx_queues - set actual number of RX queues used
1641 * @dev: Network device
1642 * @rxq: Actual number of RX queues
1644 * This must be called either with the rtnl_lock held or before
1645 * registration of the net device. Returns 0 on success, or a
1646 * negative error code. If called before registration, it always
1649 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1653 if (rxq < 1 || rxq > dev->num_rx_queues)
1656 if (dev->reg_state == NETREG_REGISTERED) {
1659 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1665 dev->real_num_rx_queues = rxq;
1668 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1671 static inline void __netif_reschedule(struct Qdisc *q)
1673 struct softnet_data *sd;
1674 unsigned long flags;
1676 local_irq_save(flags);
1677 sd = &__get_cpu_var(softnet_data);
1678 q->next_sched = NULL;
1679 *sd->output_queue_tailp = q;
1680 sd->output_queue_tailp = &q->next_sched;
1681 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1682 local_irq_restore(flags);
1685 void __netif_schedule(struct Qdisc *q)
1687 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1688 __netif_reschedule(q);
1690 EXPORT_SYMBOL(__netif_schedule);
1692 void dev_kfree_skb_irq(struct sk_buff *skb)
1694 if (atomic_dec_and_test(&skb->users)) {
1695 struct softnet_data *sd;
1696 unsigned long flags;
1698 local_irq_save(flags);
1699 sd = &__get_cpu_var(softnet_data);
1700 skb->next = sd->completion_queue;
1701 sd->completion_queue = skb;
1702 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1703 local_irq_restore(flags);
1706 EXPORT_SYMBOL(dev_kfree_skb_irq);
1708 void dev_kfree_skb_any(struct sk_buff *skb)
1710 if (in_irq() || irqs_disabled())
1711 dev_kfree_skb_irq(skb);
1715 EXPORT_SYMBOL(dev_kfree_skb_any);
1719 * netif_device_detach - mark device as removed
1720 * @dev: network device
1722 * Mark device as removed from system and therefore no longer available.
1724 void netif_device_detach(struct net_device *dev)
1726 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1727 netif_running(dev)) {
1728 netif_tx_stop_all_queues(dev);
1731 EXPORT_SYMBOL(netif_device_detach);
1734 * netif_device_attach - mark device as attached
1735 * @dev: network device
1737 * Mark device as attached from system and restart if needed.
1739 void netif_device_attach(struct net_device *dev)
1741 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1742 netif_running(dev)) {
1743 netif_tx_wake_all_queues(dev);
1744 __netdev_watchdog_up(dev);
1747 EXPORT_SYMBOL(netif_device_attach);
1750 * skb_dev_set -- assign a new device to a buffer
1751 * @skb: buffer for the new device
1752 * @dev: network device
1754 * If an skb is owned by a device already, we have to reset
1755 * all data private to the namespace a device belongs to
1756 * before assigning it a new device.
1758 #ifdef CONFIG_NET_NS
1759 void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1762 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1765 skb_init_secmark(skb);
1769 skb->ipvs_property = 0;
1770 #ifdef CONFIG_NET_SCHED
1776 EXPORT_SYMBOL(skb_set_dev);
1777 #endif /* CONFIG_NET_NS */
1780 * Invalidate hardware checksum when packet is to be mangled, and
1781 * complete checksum manually on outgoing path.
1783 int skb_checksum_help(struct sk_buff *skb)
1786 int ret = 0, offset;
1788 if (skb->ip_summed == CHECKSUM_COMPLETE)
1789 goto out_set_summed;
1791 if (unlikely(skb_shinfo(skb)->gso_size)) {
1792 /* Let GSO fix up the checksum. */
1793 goto out_set_summed;
1796 offset = skb_checksum_start_offset(skb);
1797 BUG_ON(offset >= skb_headlen(skb));
1798 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1800 offset += skb->csum_offset;
1801 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1803 if (skb_cloned(skb) &&
1804 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
1805 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1810 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
1812 skb->ip_summed = CHECKSUM_NONE;
1816 EXPORT_SYMBOL(skb_checksum_help);
1819 * skb_gso_segment - Perform segmentation on skb.
1820 * @skb: buffer to segment
1821 * @features: features for the output path (see dev->features)
1823 * This function segments the given skb and returns a list of segments.
1825 * It may return NULL if the skb requires no segmentation. This is
1826 * only possible when GSO is used for verifying header integrity.
1828 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1830 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1831 struct packet_type *ptype;
1832 __be16 type = skb->protocol;
1833 int vlan_depth = ETH_HLEN;
1836 while (type == htons(ETH_P_8021Q)) {
1837 struct vlan_hdr *vh;
1839 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
1840 return ERR_PTR(-EINVAL);
1842 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1843 type = vh->h_vlan_encapsulated_proto;
1844 vlan_depth += VLAN_HLEN;
1847 skb_reset_mac_header(skb);
1848 skb->mac_len = skb->network_header - skb->mac_header;
1849 __skb_pull(skb, skb->mac_len);
1851 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1852 struct net_device *dev = skb->dev;
1853 struct ethtool_drvinfo info = {};
1855 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1856 dev->ethtool_ops->get_drvinfo(dev, &info);
1858 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
1859 info.driver, dev ? dev->features : 0L,
1860 skb->sk ? skb->sk->sk_route_caps : 0L,
1861 skb->len, skb->data_len, skb->ip_summed);
1863 if (skb_header_cloned(skb) &&
1864 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1865 return ERR_PTR(err);
1869 list_for_each_entry_rcu(ptype,
1870 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
1871 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
1872 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1873 err = ptype->gso_send_check(skb);
1874 segs = ERR_PTR(err);
1875 if (err || skb_gso_ok(skb, features))
1877 __skb_push(skb, (skb->data -
1878 skb_network_header(skb)));
1880 segs = ptype->gso_segment(skb, features);
1886 __skb_push(skb, skb->data - skb_mac_header(skb));
1890 EXPORT_SYMBOL(skb_gso_segment);
1892 /* Take action when hardware reception checksum errors are detected. */
1894 void netdev_rx_csum_fault(struct net_device *dev)
1896 if (net_ratelimit()) {
1897 printk(KERN_ERR "%s: hw csum failure.\n",
1898 dev ? dev->name : "<unknown>");
1902 EXPORT_SYMBOL(netdev_rx_csum_fault);
1905 /* Actually, we should eliminate this check as soon as we know, that:
1906 * 1. IOMMU is present and allows to map all the memory.
1907 * 2. No high memory really exists on this machine.
1910 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1912 #ifdef CONFIG_HIGHMEM
1914 if (!(dev->features & NETIF_F_HIGHDMA)) {
1915 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1916 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1920 if (PCI_DMA_BUS_IS_PHYS) {
1921 struct device *pdev = dev->dev.parent;
1925 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1926 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1927 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1936 void (*destructor)(struct sk_buff *skb);
1939 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1941 static void dev_gso_skb_destructor(struct sk_buff *skb)
1943 struct dev_gso_cb *cb;
1946 struct sk_buff *nskb = skb->next;
1948 skb->next = nskb->next;
1951 } while (skb->next);
1953 cb = DEV_GSO_CB(skb);
1955 cb->destructor(skb);
1959 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1960 * @skb: buffer to segment
1961 * @features: device features as applicable to this skb
1963 * This function segments the given skb and stores the list of segments
1966 static int dev_gso_segment(struct sk_buff *skb, int features)
1968 struct sk_buff *segs;
1970 segs = skb_gso_segment(skb, features);
1972 /* Verifying header integrity only. */
1977 return PTR_ERR(segs);
1980 DEV_GSO_CB(skb)->destructor = skb->destructor;
1981 skb->destructor = dev_gso_skb_destructor;
1987 * Try to orphan skb early, right before transmission by the device.
1988 * We cannot orphan skb if tx timestamp is requested or the sk-reference
1989 * is needed on driver level for other reasons, e.g. see net/can/raw.c
1991 static inline void skb_orphan_try(struct sk_buff *skb)
1993 struct sock *sk = skb->sk;
1995 if (sk && !skb_shinfo(skb)->tx_flags) {
1996 /* skb_tx_hash() wont be able to get sk.
1997 * We copy sk_hash into skb->rxhash
2000 skb->rxhash = sk->sk_hash;
2005 static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2007 return ((features & NETIF_F_GEN_CSUM) ||
2008 ((features & NETIF_F_V4_CSUM) &&
2009 protocol == htons(ETH_P_IP)) ||
2010 ((features & NETIF_F_V6_CSUM) &&
2011 protocol == htons(ETH_P_IPV6)) ||
2012 ((features & NETIF_F_FCOE_CRC) &&
2013 protocol == htons(ETH_P_FCOE)));
2016 static int harmonize_features(struct sk_buff *skb, __be16 protocol, int features)
2018 if (!can_checksum_protocol(features, protocol)) {
2019 features &= ~NETIF_F_ALL_CSUM;
2020 features &= ~NETIF_F_SG;
2021 } else if (illegal_highdma(skb->dev, skb)) {
2022 features &= ~NETIF_F_SG;
2028 int netif_skb_features(struct sk_buff *skb)
2030 __be16 protocol = skb->protocol;
2031 int features = skb->dev->features;
2033 if (protocol == htons(ETH_P_8021Q)) {
2034 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2035 protocol = veh->h_vlan_encapsulated_proto;
2036 } else if (!vlan_tx_tag_present(skb)) {
2037 return harmonize_features(skb, protocol, features);
2040 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
2042 if (protocol != htons(ETH_P_8021Q)) {
2043 return harmonize_features(skb, protocol, features);
2045 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
2046 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
2047 return harmonize_features(skb, protocol, features);
2050 EXPORT_SYMBOL(netif_skb_features);
2053 * Returns true if either:
2054 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2055 * 2. skb is fragmented and the device does not support SG, or if
2056 * at least one of fragments is in highmem and device does not
2057 * support DMA from it.
2059 static inline int skb_needs_linearize(struct sk_buff *skb,
2062 return skb_is_nonlinear(skb) &&
2063 ((skb_has_frag_list(skb) &&
2064 !(features & NETIF_F_FRAGLIST)) ||
2065 (skb_shinfo(skb)->nr_frags &&
2066 !(features & NETIF_F_SG)));
2069 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2070 struct netdev_queue *txq)
2072 const struct net_device_ops *ops = dev->netdev_ops;
2073 int rc = NETDEV_TX_OK;
2075 if (likely(!skb->next)) {
2079 * If device doesnt need skb->dst, release it right now while
2080 * its hot in this cpu cache
2082 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2085 if (!list_empty(&ptype_all))
2086 dev_queue_xmit_nit(skb, dev);
2088 skb_orphan_try(skb);
2090 features = netif_skb_features(skb);
2092 if (vlan_tx_tag_present(skb) &&
2093 !(features & NETIF_F_HW_VLAN_TX)) {
2094 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2101 if (netif_needs_gso(skb, features)) {
2102 if (unlikely(dev_gso_segment(skb, features)))
2107 if (skb_needs_linearize(skb, features) &&
2108 __skb_linearize(skb))
2111 /* If packet is not checksummed and device does not
2112 * support checksumming for this protocol, complete
2113 * checksumming here.
2115 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2116 skb_set_transport_header(skb,
2117 skb_checksum_start_offset(skb));
2118 if (!(features & NETIF_F_ALL_CSUM) &&
2119 skb_checksum_help(skb))
2124 rc = ops->ndo_start_xmit(skb, dev);
2125 trace_net_dev_xmit(skb, rc);
2126 if (rc == NETDEV_TX_OK)
2127 txq_trans_update(txq);
2133 struct sk_buff *nskb = skb->next;
2135 skb->next = nskb->next;
2139 * If device doesnt need nskb->dst, release it right now while
2140 * its hot in this cpu cache
2142 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2145 rc = ops->ndo_start_xmit(nskb, dev);
2146 trace_net_dev_xmit(nskb, rc);
2147 if (unlikely(rc != NETDEV_TX_OK)) {
2148 if (rc & ~NETDEV_TX_MASK)
2149 goto out_kfree_gso_skb;
2150 nskb->next = skb->next;
2154 txq_trans_update(txq);
2155 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
2156 return NETDEV_TX_BUSY;
2157 } while (skb->next);
2160 if (likely(skb->next == NULL))
2161 skb->destructor = DEV_GSO_CB(skb)->destructor;
2168 static u32 hashrnd __read_mostly;
2171 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2172 * to be used as a distribution range.
2174 u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2175 unsigned int num_tx_queues)
2179 if (skb_rx_queue_recorded(skb)) {
2180 hash = skb_get_rx_queue(skb);
2181 while (unlikely(hash >= num_tx_queues))
2182 hash -= num_tx_queues;
2186 if (skb->sk && skb->sk->sk_hash)
2187 hash = skb->sk->sk_hash;
2189 hash = (__force u16) skb->protocol ^ skb->rxhash;
2190 hash = jhash_1word(hash, hashrnd);
2192 return (u16) (((u64) hash * num_tx_queues) >> 32);
2194 EXPORT_SYMBOL(__skb_tx_hash);
2196 static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2198 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2199 if (net_ratelimit()) {
2200 pr_warning("%s selects TX queue %d, but "
2201 "real number of TX queues is %d\n",
2202 dev->name, queue_index, dev->real_num_tx_queues);
2209 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2212 struct xps_dev_maps *dev_maps;
2213 struct xps_map *map;
2214 int queue_index = -1;
2217 dev_maps = rcu_dereference(dev->xps_maps);
2219 map = rcu_dereference(
2220 dev_maps->cpu_map[raw_smp_processor_id()]);
2223 queue_index = map->queues[0];
2226 if (skb->sk && skb->sk->sk_hash)
2227 hash = skb->sk->sk_hash;
2229 hash = (__force u16) skb->protocol ^
2231 hash = jhash_1word(hash, hashrnd);
2232 queue_index = map->queues[
2233 ((u64)hash * map->len) >> 32];
2235 if (unlikely(queue_index >= dev->real_num_tx_queues))
2247 static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2248 struct sk_buff *skb)
2251 const struct net_device_ops *ops = dev->netdev_ops;
2253 if (dev->real_num_tx_queues == 1)
2255 else if (ops->ndo_select_queue) {
2256 queue_index = ops->ndo_select_queue(dev, skb);
2257 queue_index = dev_cap_txqueue(dev, queue_index);
2259 struct sock *sk = skb->sk;
2260 queue_index = sk_tx_queue_get(sk);
2262 if (queue_index < 0 || skb->ooo_okay ||
2263 queue_index >= dev->real_num_tx_queues) {
2264 int old_index = queue_index;
2266 queue_index = get_xps_queue(dev, skb);
2267 if (queue_index < 0)
2268 queue_index = skb_tx_hash(dev, skb);
2270 if (queue_index != old_index && sk) {
2271 struct dst_entry *dst =
2272 rcu_dereference_check(sk->sk_dst_cache, 1);
2274 if (dst && skb_dst(skb) == dst)
2275 sk_tx_queue_set(sk, queue_index);
2280 skb_set_queue_mapping(skb, queue_index);
2281 return netdev_get_tx_queue(dev, queue_index);
2284 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2285 struct net_device *dev,
2286 struct netdev_queue *txq)
2288 spinlock_t *root_lock = qdisc_lock(q);
2289 bool contended = qdisc_is_running(q);
2293 * Heuristic to force contended enqueues to serialize on a
2294 * separate lock before trying to get qdisc main lock.
2295 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2296 * and dequeue packets faster.
2298 if (unlikely(contended))
2299 spin_lock(&q->busylock);
2301 spin_lock(root_lock);
2302 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2305 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2306 qdisc_run_begin(q)) {
2308 * This is a work-conserving queue; there are no old skbs
2309 * waiting to be sent out; and the qdisc is not running -
2310 * xmit the skb directly.
2312 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2315 qdisc_skb_cb(skb)->pkt_len = skb->len;
2316 qdisc_bstats_update(q, skb);
2318 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2319 if (unlikely(contended)) {
2320 spin_unlock(&q->busylock);
2327 rc = NET_XMIT_SUCCESS;
2330 rc = qdisc_enqueue_root(skb, q);
2331 if (qdisc_run_begin(q)) {
2332 if (unlikely(contended)) {
2333 spin_unlock(&q->busylock);
2339 spin_unlock(root_lock);
2340 if (unlikely(contended))
2341 spin_unlock(&q->busylock);
2345 static DEFINE_PER_CPU(int, xmit_recursion);
2346 #define RECURSION_LIMIT 10
2349 * dev_queue_xmit - transmit a buffer
2350 * @skb: buffer to transmit
2352 * Queue a buffer for transmission to a network device. The caller must
2353 * have set the device and priority and built the buffer before calling
2354 * this function. The function can be called from an interrupt.
2356 * A negative errno code is returned on a failure. A success does not
2357 * guarantee the frame will be transmitted as it may be dropped due
2358 * to congestion or traffic shaping.
2360 * -----------------------------------------------------------------------------------
2361 * I notice this method can also return errors from the queue disciplines,
2362 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2365 * Regardless of the return value, the skb is consumed, so it is currently
2366 * difficult to retry a send to this method. (You can bump the ref count
2367 * before sending to hold a reference for retry if you are careful.)
2369 * When calling this method, interrupts MUST be enabled. This is because
2370 * the BH enable code must have IRQs enabled so that it will not deadlock.
2373 int dev_queue_xmit(struct sk_buff *skb)
2375 struct net_device *dev = skb->dev;
2376 struct netdev_queue *txq;
2380 /* Disable soft irqs for various locks below. Also
2381 * stops preemption for RCU.
2385 txq = dev_pick_tx(dev, skb);
2386 q = rcu_dereference_bh(txq->qdisc);
2388 #ifdef CONFIG_NET_CLS_ACT
2389 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2391 trace_net_dev_queue(skb);
2393 rc = __dev_xmit_skb(skb, q, dev, txq);
2397 /* The device has no queue. Common case for software devices:
2398 loopback, all the sorts of tunnels...
2400 Really, it is unlikely that netif_tx_lock protection is necessary
2401 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2403 However, it is possible, that they rely on protection
2406 Check this and shot the lock. It is not prone from deadlocks.
2407 Either shot noqueue qdisc, it is even simpler 8)
2409 if (dev->flags & IFF_UP) {
2410 int cpu = smp_processor_id(); /* ok because BHs are off */
2412 if (txq->xmit_lock_owner != cpu) {
2414 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2415 goto recursion_alert;
2417 HARD_TX_LOCK(dev, txq, cpu);
2419 if (!netif_tx_queue_stopped(txq)) {
2420 __this_cpu_inc(xmit_recursion);
2421 rc = dev_hard_start_xmit(skb, dev, txq);
2422 __this_cpu_dec(xmit_recursion);
2423 if (dev_xmit_complete(rc)) {
2424 HARD_TX_UNLOCK(dev, txq);
2428 HARD_TX_UNLOCK(dev, txq);
2429 if (net_ratelimit())
2430 printk(KERN_CRIT "Virtual device %s asks to "
2431 "queue packet!\n", dev->name);
2433 /* Recursion is detected! It is possible,
2437 if (net_ratelimit())
2438 printk(KERN_CRIT "Dead loop on virtual device "
2439 "%s, fix it urgently!\n", dev->name);
2444 rcu_read_unlock_bh();
2449 rcu_read_unlock_bh();
2452 EXPORT_SYMBOL(dev_queue_xmit);
2455 /*=======================================================================
2457 =======================================================================*/
2459 int netdev_max_backlog __read_mostly = 1000;
2460 int netdev_tstamp_prequeue __read_mostly = 1;
2461 int netdev_budget __read_mostly = 300;
2462 int weight_p __read_mostly = 64; /* old backlog weight */
2464 /* Called with irq disabled */
2465 static inline void ____napi_schedule(struct softnet_data *sd,
2466 struct napi_struct *napi)
2468 list_add_tail(&napi->poll_list, &sd->poll_list);
2469 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2473 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2474 * and src/dst port numbers. Returns a non-zero hash number on success
2477 __u32 __skb_get_rxhash(struct sk_buff *skb)
2479 int nhoff, hash = 0, poff;
2480 struct ipv6hdr *ip6;
2483 u32 addr1, addr2, ihl;
2489 nhoff = skb_network_offset(skb);
2491 switch (skb->protocol) {
2492 case __constant_htons(ETH_P_IP):
2493 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2496 ip = (struct iphdr *) (skb->data + nhoff);
2497 if (ip->frag_off & htons(IP_MF | IP_OFFSET))
2500 ip_proto = ip->protocol;
2501 addr1 = (__force u32) ip->saddr;
2502 addr2 = (__force u32) ip->daddr;
2505 case __constant_htons(ETH_P_IPV6):
2506 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2509 ip6 = (struct ipv6hdr *) (skb->data + nhoff);
2510 ip_proto = ip6->nexthdr;
2511 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2512 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2520 poff = proto_ports_offset(ip_proto);
2522 nhoff += ihl * 4 + poff;
2523 if (pskb_may_pull(skb, nhoff + 4)) {
2524 ports.v32 = * (__force u32 *) (skb->data + nhoff);
2525 if (ports.v16[1] < ports.v16[0])
2526 swap(ports.v16[0], ports.v16[1]);
2530 /* get a consistent hash (same value on both flow directions) */
2534 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2541 EXPORT_SYMBOL(__skb_get_rxhash);
2545 /* One global table that all flow-based protocols share. */
2546 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2547 EXPORT_SYMBOL(rps_sock_flow_table);
2550 * get_rps_cpu is called from netif_receive_skb and returns the target
2551 * CPU from the RPS map of the receiving queue for a given skb.
2552 * rcu_read_lock must be held on entry.
2554 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2555 struct rps_dev_flow **rflowp)
2557 struct netdev_rx_queue *rxqueue;
2558 struct rps_map *map;
2559 struct rps_dev_flow_table *flow_table;
2560 struct rps_sock_flow_table *sock_flow_table;
2564 if (skb_rx_queue_recorded(skb)) {
2565 u16 index = skb_get_rx_queue(skb);
2566 if (unlikely(index >= dev->real_num_rx_queues)) {
2567 WARN_ONCE(dev->real_num_rx_queues > 1,
2568 "%s received packet on queue %u, but number "
2569 "of RX queues is %u\n",
2570 dev->name, index, dev->real_num_rx_queues);
2573 rxqueue = dev->_rx + index;
2577 map = rcu_dereference(rxqueue->rps_map);
2579 if (map->len == 1 &&
2580 !rcu_dereference_raw(rxqueue->rps_flow_table)) {
2581 tcpu = map->cpus[0];
2582 if (cpu_online(tcpu))
2586 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
2590 skb_reset_network_header(skb);
2591 if (!skb_get_rxhash(skb))
2594 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2595 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2596 if (flow_table && sock_flow_table) {
2598 struct rps_dev_flow *rflow;
2600 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2603 next_cpu = sock_flow_table->ents[skb->rxhash &
2604 sock_flow_table->mask];
2607 * If the desired CPU (where last recvmsg was done) is
2608 * different from current CPU (one in the rx-queue flow
2609 * table entry), switch if one of the following holds:
2610 * - Current CPU is unset (equal to RPS_NO_CPU).
2611 * - Current CPU is offline.
2612 * - The current CPU's queue tail has advanced beyond the
2613 * last packet that was enqueued using this table entry.
2614 * This guarantees that all previous packets for the flow
2615 * have been dequeued, thus preserving in order delivery.
2617 if (unlikely(tcpu != next_cpu) &&
2618 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2619 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2620 rflow->last_qtail)) >= 0)) {
2621 tcpu = rflow->cpu = next_cpu;
2622 if (tcpu != RPS_NO_CPU)
2623 rflow->last_qtail = per_cpu(softnet_data,
2624 tcpu).input_queue_head;
2626 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2634 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
2636 if (cpu_online(tcpu)) {
2646 /* Called from hardirq (IPI) context */
2647 static void rps_trigger_softirq(void *data)
2649 struct softnet_data *sd = data;
2651 ____napi_schedule(sd, &sd->backlog);
2655 #endif /* CONFIG_RPS */
2658 * Check if this softnet_data structure is another cpu one
2659 * If yes, queue it to our IPI list and return 1
2662 static int rps_ipi_queued(struct softnet_data *sd)
2665 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2668 sd->rps_ipi_next = mysd->rps_ipi_list;
2669 mysd->rps_ipi_list = sd;
2671 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2674 #endif /* CONFIG_RPS */
2679 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2680 * queue (may be a remote CPU queue).
2682 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2683 unsigned int *qtail)
2685 struct softnet_data *sd;
2686 unsigned long flags;
2688 sd = &per_cpu(softnet_data, cpu);
2690 local_irq_save(flags);
2693 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2694 if (skb_queue_len(&sd->input_pkt_queue)) {
2696 __skb_queue_tail(&sd->input_pkt_queue, skb);
2697 input_queue_tail_incr_save(sd, qtail);
2699 local_irq_restore(flags);
2700 return NET_RX_SUCCESS;
2703 /* Schedule NAPI for backlog device
2704 * We can use non atomic operation since we own the queue lock
2706 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
2707 if (!rps_ipi_queued(sd))
2708 ____napi_schedule(sd, &sd->backlog);
2716 local_irq_restore(flags);
2718 atomic_long_inc(&skb->dev->rx_dropped);
2724 * netif_rx - post buffer to the network code
2725 * @skb: buffer to post
2727 * This function receives a packet from a device driver and queues it for
2728 * the upper (protocol) levels to process. It always succeeds. The buffer
2729 * may be dropped during processing for congestion control or by the
2733 * NET_RX_SUCCESS (no congestion)
2734 * NET_RX_DROP (packet was dropped)
2738 int netif_rx(struct sk_buff *skb)
2742 /* if netpoll wants it, pretend we never saw it */
2743 if (netpoll_rx(skb))
2746 if (netdev_tstamp_prequeue)
2747 net_timestamp_check(skb);
2749 trace_netif_rx(skb);
2752 struct rps_dev_flow voidflow, *rflow = &voidflow;
2758 cpu = get_rps_cpu(skb->dev, skb, &rflow);
2760 cpu = smp_processor_id();
2762 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2770 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2776 EXPORT_SYMBOL(netif_rx);
2778 int netif_rx_ni(struct sk_buff *skb)
2783 err = netif_rx(skb);
2784 if (local_softirq_pending())
2790 EXPORT_SYMBOL(netif_rx_ni);
2792 static void net_tx_action(struct softirq_action *h)
2794 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2796 if (sd->completion_queue) {
2797 struct sk_buff *clist;
2799 local_irq_disable();
2800 clist = sd->completion_queue;
2801 sd->completion_queue = NULL;
2805 struct sk_buff *skb = clist;
2806 clist = clist->next;
2808 WARN_ON(atomic_read(&skb->users));
2809 trace_kfree_skb(skb, net_tx_action);
2814 if (sd->output_queue) {
2817 local_irq_disable();
2818 head = sd->output_queue;
2819 sd->output_queue = NULL;
2820 sd->output_queue_tailp = &sd->output_queue;
2824 struct Qdisc *q = head;
2825 spinlock_t *root_lock;
2827 head = head->next_sched;
2829 root_lock = qdisc_lock(q);
2830 if (spin_trylock(root_lock)) {
2831 smp_mb__before_clear_bit();
2832 clear_bit(__QDISC_STATE_SCHED,
2835 spin_unlock(root_lock);
2837 if (!test_bit(__QDISC_STATE_DEACTIVATED,
2839 __netif_reschedule(q);
2841 smp_mb__before_clear_bit();
2842 clear_bit(__QDISC_STATE_SCHED,
2850 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2851 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
2852 /* This hook is defined here for ATM LANE */
2853 int (*br_fdb_test_addr_hook)(struct net_device *dev,
2854 unsigned char *addr) __read_mostly;
2855 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
2858 #ifdef CONFIG_NET_CLS_ACT
2859 /* TODO: Maybe we should just force sch_ingress to be compiled in
2860 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2861 * a compare and 2 stores extra right now if we dont have it on
2862 * but have CONFIG_NET_CLS_ACT
2863 * NOTE: This doesnt stop any functionality; if you dont have
2864 * the ingress scheduler, you just cant add policies on ingress.
2867 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
2869 struct net_device *dev = skb->dev;
2870 u32 ttl = G_TC_RTTL(skb->tc_verd);
2871 int result = TC_ACT_OK;
2874 if (unlikely(MAX_RED_LOOP < ttl++)) {
2875 if (net_ratelimit())
2876 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
2877 skb->skb_iif, dev->ifindex);
2881 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2882 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2885 if (q != &noop_qdisc) {
2886 spin_lock(qdisc_lock(q));
2887 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2888 result = qdisc_enqueue_root(skb, q);
2889 spin_unlock(qdisc_lock(q));
2895 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2896 struct packet_type **pt_prev,
2897 int *ret, struct net_device *orig_dev)
2899 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
2901 if (!rxq || rxq->qdisc == &noop_qdisc)
2905 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2909 switch (ing_filter(skb, rxq)) {
2923 * netdev_rx_handler_register - register receive handler
2924 * @dev: device to register a handler for
2925 * @rx_handler: receive handler to register
2926 * @rx_handler_data: data pointer that is used by rx handler
2928 * Register a receive hander for a device. This handler will then be
2929 * called from __netif_receive_skb. A negative errno code is returned
2932 * The caller must hold the rtnl_mutex.
2934 int netdev_rx_handler_register(struct net_device *dev,
2935 rx_handler_func_t *rx_handler,
2936 void *rx_handler_data)
2940 if (dev->rx_handler)
2943 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
2944 rcu_assign_pointer(dev->rx_handler, rx_handler);
2948 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
2951 * netdev_rx_handler_unregister - unregister receive handler
2952 * @dev: device to unregister a handler from
2954 * Unregister a receive hander from a device.
2956 * The caller must hold the rtnl_mutex.
2958 void netdev_rx_handler_unregister(struct net_device *dev)
2962 rcu_assign_pointer(dev->rx_handler, NULL);
2963 rcu_assign_pointer(dev->rx_handler_data, NULL);
2965 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
2967 static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,
2968 struct net_device *master)
2970 if (skb->pkt_type == PACKET_HOST) {
2971 u16 *dest = (u16 *) eth_hdr(skb)->h_dest;
2973 memcpy(dest, master->dev_addr, ETH_ALEN);
2977 /* On bonding slaves other than the currently active slave, suppress
2978 * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
2979 * ARP on active-backup slaves with arp_validate enabled.
2981 int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)
2983 struct net_device *dev = skb->dev;
2985 if (master->priv_flags & IFF_MASTER_ARPMON)
2986 dev->last_rx = jiffies;
2988 if ((master->priv_flags & IFF_MASTER_ALB) &&
2989 (master->priv_flags & IFF_BRIDGE_PORT)) {
2990 /* Do address unmangle. The local destination address
2991 * will be always the one master has. Provides the right
2992 * functionality in a bridge.
2994 skb_bond_set_mac_by_master(skb, master);
2997 if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
2998 if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&
2999 skb->protocol == __cpu_to_be16(ETH_P_ARP))
3002 if (master->priv_flags & IFF_MASTER_ALB) {
3003 if (skb->pkt_type != PACKET_BROADCAST &&
3004 skb->pkt_type != PACKET_MULTICAST)
3007 if (master->priv_flags & IFF_MASTER_8023AD &&
3008 skb->protocol == __cpu_to_be16(ETH_P_SLOW))
3015 EXPORT_SYMBOL(__skb_bond_should_drop);
3017 static int __netif_receive_skb(struct sk_buff *skb)
3019 struct packet_type *ptype, *pt_prev;
3020 rx_handler_func_t *rx_handler;
3021 struct net_device *orig_dev;
3022 struct net_device *master;
3023 struct net_device *null_or_orig;
3024 struct net_device *orig_or_bond;
3025 int ret = NET_RX_DROP;
3028 if (!netdev_tstamp_prequeue)
3029 net_timestamp_check(skb);
3031 trace_netif_receive_skb(skb);
3033 /* if we've gotten here through NAPI, check netpoll */
3034 if (netpoll_receive_skb(skb))
3038 skb->skb_iif = skb->dev->ifindex;
3041 * bonding note: skbs received on inactive slaves should only
3042 * be delivered to pkt handlers that are exact matches. Also
3043 * the deliver_no_wcard flag will be set. If packet handlers
3044 * are sensitive to duplicate packets these skbs will need to
3045 * be dropped at the handler.
3047 null_or_orig = NULL;
3048 orig_dev = skb->dev;
3049 master = ACCESS_ONCE(orig_dev->master);
3050 if (skb->deliver_no_wcard)
3051 null_or_orig = orig_dev;
3053 if (skb_bond_should_drop(skb, master)) {
3054 skb->deliver_no_wcard = 1;
3055 null_or_orig = orig_dev; /* deliver only exact match */
3060 __this_cpu_inc(softnet_data.processed);
3061 skb_reset_network_header(skb);
3062 skb_reset_transport_header(skb);
3063 skb->mac_len = skb->network_header - skb->mac_header;
3069 #ifdef CONFIG_NET_CLS_ACT
3070 if (skb->tc_verd & TC_NCLS) {
3071 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3076 list_for_each_entry_rcu(ptype, &ptype_all, list) {
3077 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
3078 ptype->dev == orig_dev) {
3080 ret = deliver_skb(skb, pt_prev, orig_dev);
3085 #ifdef CONFIG_NET_CLS_ACT
3086 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3092 /* Handle special case of bridge or macvlan */
3093 rx_handler = rcu_dereference(skb->dev->rx_handler);
3096 ret = deliver_skb(skb, pt_prev, orig_dev);
3099 skb = rx_handler(skb);
3104 if (vlan_tx_tag_present(skb)) {
3106 ret = deliver_skb(skb, pt_prev, orig_dev);
3109 if (vlan_hwaccel_do_receive(&skb)) {
3110 ret = __netif_receive_skb(skb);
3112 } else if (unlikely(!skb))
3117 * Make sure frames received on VLAN interfaces stacked on
3118 * bonding interfaces still make their way to any base bonding
3119 * device that may have registered for a specific ptype. The
3120 * handler may have to adjust skb->dev and orig_dev.
3122 orig_or_bond = orig_dev;
3123 if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
3124 (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
3125 orig_or_bond = vlan_dev_real_dev(skb->dev);
3128 type = skb->protocol;
3129 list_for_each_entry_rcu(ptype,
3130 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3131 if (ptype->type == type && (ptype->dev == null_or_orig ||
3132 ptype->dev == skb->dev || ptype->dev == orig_dev ||
3133 ptype->dev == orig_or_bond)) {
3135 ret = deliver_skb(skb, pt_prev, orig_dev);
3141 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3143 atomic_long_inc(&skb->dev->rx_dropped);
3145 /* Jamal, now you will not able to escape explaining
3146 * me how you were going to use this. :-)
3157 * netif_receive_skb - process receive buffer from network
3158 * @skb: buffer to process
3160 * netif_receive_skb() is the main receive data processing function.
3161 * It always succeeds. The buffer may be dropped during processing
3162 * for congestion control or by the protocol layers.
3164 * This function may only be called from softirq context and interrupts
3165 * should be enabled.
3167 * Return values (usually ignored):
3168 * NET_RX_SUCCESS: no congestion
3169 * NET_RX_DROP: packet was dropped
3171 int netif_receive_skb(struct sk_buff *skb)
3173 if (netdev_tstamp_prequeue)
3174 net_timestamp_check(skb);
3176 if (skb_defer_rx_timestamp(skb))
3177 return NET_RX_SUCCESS;
3181 struct rps_dev_flow voidflow, *rflow = &voidflow;
3186 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3189 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3193 ret = __netif_receive_skb(skb);
3199 return __netif_receive_skb(skb);
3202 EXPORT_SYMBOL(netif_receive_skb);
3204 /* Network device is going away, flush any packets still pending
3205 * Called with irqs disabled.
3207 static void flush_backlog(void *arg)
3209 struct net_device *dev = arg;
3210 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3211 struct sk_buff *skb, *tmp;
3214 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3215 if (skb->dev == dev) {
3216 __skb_unlink(skb, &sd->input_pkt_queue);
3218 input_queue_head_incr(sd);
3223 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3224 if (skb->dev == dev) {
3225 __skb_unlink(skb, &sd->process_queue);
3227 input_queue_head_incr(sd);
3232 static int napi_gro_complete(struct sk_buff *skb)
3234 struct packet_type *ptype;
3235 __be16 type = skb->protocol;
3236 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3239 if (NAPI_GRO_CB(skb)->count == 1) {
3240 skb_shinfo(skb)->gso_size = 0;
3245 list_for_each_entry_rcu(ptype, head, list) {
3246 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3249 err = ptype->gro_complete(skb);
3255 WARN_ON(&ptype->list == head);
3257 return NET_RX_SUCCESS;
3261 return netif_receive_skb(skb);
3264 inline void napi_gro_flush(struct napi_struct *napi)
3266 struct sk_buff *skb, *next;
3268 for (skb = napi->gro_list; skb; skb = next) {
3271 napi_gro_complete(skb);
3274 napi->gro_count = 0;
3275 napi->gro_list = NULL;
3277 EXPORT_SYMBOL(napi_gro_flush);
3279 enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3281 struct sk_buff **pp = NULL;
3282 struct packet_type *ptype;
3283 __be16 type = skb->protocol;
3284 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3287 enum gro_result ret;
3289 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3292 if (skb_is_gso(skb) || skb_has_frag_list(skb))
3296 list_for_each_entry_rcu(ptype, head, list) {
3297 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3300 skb_set_network_header(skb, skb_gro_offset(skb));
3301 mac_len = skb->network_header - skb->mac_header;
3302 skb->mac_len = mac_len;
3303 NAPI_GRO_CB(skb)->same_flow = 0;
3304 NAPI_GRO_CB(skb)->flush = 0;
3305 NAPI_GRO_CB(skb)->free = 0;
3307 pp = ptype->gro_receive(&napi->gro_list, skb);
3312 if (&ptype->list == head)
3315 same_flow = NAPI_GRO_CB(skb)->same_flow;
3316 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3319 struct sk_buff *nskb = *pp;
3323 napi_gro_complete(nskb);
3330 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3334 NAPI_GRO_CB(skb)->count = 1;
3335 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3336 skb->next = napi->gro_list;
3337 napi->gro_list = skb;
3341 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3342 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3344 BUG_ON(skb->end - skb->tail < grow);
3346 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3349 skb->data_len -= grow;
3351 skb_shinfo(skb)->frags[0].page_offset += grow;
3352 skb_shinfo(skb)->frags[0].size -= grow;
3354 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3355 put_page(skb_shinfo(skb)->frags[0].page);
3356 memmove(skb_shinfo(skb)->frags,
3357 skb_shinfo(skb)->frags + 1,
3358 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3369 EXPORT_SYMBOL(dev_gro_receive);
3371 static inline gro_result_t
3372 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3376 for (p = napi->gro_list; p; p = p->next) {
3377 unsigned long diffs;
3379 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3380 diffs |= p->vlan_tci ^ skb->vlan_tci;
3381 diffs |= compare_ether_header(skb_mac_header(p),
3382 skb_gro_mac_header(skb));
3383 NAPI_GRO_CB(p)->same_flow = !diffs;
3384 NAPI_GRO_CB(p)->flush = 0;
3387 return dev_gro_receive(napi, skb);
3390 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3394 if (netif_receive_skb(skb))
3399 case GRO_MERGED_FREE:
3410 EXPORT_SYMBOL(napi_skb_finish);
3412 void skb_gro_reset_offset(struct sk_buff *skb)
3414 NAPI_GRO_CB(skb)->data_offset = 0;
3415 NAPI_GRO_CB(skb)->frag0 = NULL;
3416 NAPI_GRO_CB(skb)->frag0_len = 0;
3418 if (skb->mac_header == skb->tail &&
3419 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
3420 NAPI_GRO_CB(skb)->frag0 =
3421 page_address(skb_shinfo(skb)->frags[0].page) +
3422 skb_shinfo(skb)->frags[0].page_offset;
3423 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3426 EXPORT_SYMBOL(skb_gro_reset_offset);
3428 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3430 skb_gro_reset_offset(skb);
3432 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
3434 EXPORT_SYMBOL(napi_gro_receive);
3436 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3438 __skb_pull(skb, skb_headlen(skb));
3439 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
3441 skb->dev = napi->dev;
3447 struct sk_buff *napi_get_frags(struct napi_struct *napi)
3449 struct sk_buff *skb = napi->skb;
3452 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3458 EXPORT_SYMBOL(napi_get_frags);
3460 gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3466 skb->protocol = eth_type_trans(skb, skb->dev);
3468 if (ret == GRO_HELD)
3469 skb_gro_pull(skb, -ETH_HLEN);
3470 else if (netif_receive_skb(skb))
3475 case GRO_MERGED_FREE:
3476 napi_reuse_skb(napi, skb);
3485 EXPORT_SYMBOL(napi_frags_finish);
3487 struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3489 struct sk_buff *skb = napi->skb;
3496 skb_reset_mac_header(skb);
3497 skb_gro_reset_offset(skb);
3499 off = skb_gro_offset(skb);
3500 hlen = off + sizeof(*eth);
3501 eth = skb_gro_header_fast(skb, off);
3502 if (skb_gro_header_hard(skb, hlen)) {
3503 eth = skb_gro_header_slow(skb, hlen, off);
3504 if (unlikely(!eth)) {
3505 napi_reuse_skb(napi, skb);
3511 skb_gro_pull(skb, sizeof(*eth));
3514 * This works because the only protocols we care about don't require
3515 * special handling. We'll fix it up properly at the end.
3517 skb->protocol = eth->h_proto;
3522 EXPORT_SYMBOL(napi_frags_skb);
3524 gro_result_t napi_gro_frags(struct napi_struct *napi)
3526 struct sk_buff *skb = napi_frags_skb(napi);
3531 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
3533 EXPORT_SYMBOL(napi_gro_frags);
3536 * net_rps_action sends any pending IPI's for rps.
3537 * Note: called with local irq disabled, but exits with local irq enabled.
3539 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3542 struct softnet_data *remsd = sd->rps_ipi_list;
3545 sd->rps_ipi_list = NULL;
3549 /* Send pending IPI's to kick RPS processing on remote cpus. */
3551 struct softnet_data *next = remsd->rps_ipi_next;
3553 if (cpu_online(remsd->cpu))
3554 __smp_call_function_single(remsd->cpu,
3563 static int process_backlog(struct napi_struct *napi, int quota)
3566 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
3569 /* Check if we have pending ipi, its better to send them now,
3570 * not waiting net_rx_action() end.
3572 if (sd->rps_ipi_list) {
3573 local_irq_disable();
3574 net_rps_action_and_irq_enable(sd);
3577 napi->weight = weight_p;
3578 local_irq_disable();
3579 while (work < quota) {
3580 struct sk_buff *skb;
3583 while ((skb = __skb_dequeue(&sd->process_queue))) {
3585 __netif_receive_skb(skb);
3586 local_irq_disable();
3587 input_queue_head_incr(sd);
3588 if (++work >= quota) {
3595 qlen = skb_queue_len(&sd->input_pkt_queue);
3597 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3598 &sd->process_queue);
3600 if (qlen < quota - work) {
3602 * Inline a custom version of __napi_complete().
3603 * only current cpu owns and manipulates this napi,
3604 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3605 * we can use a plain write instead of clear_bit(),
3606 * and we dont need an smp_mb() memory barrier.
3608 list_del(&napi->poll_list);
3611 quota = work + qlen;
3621 * __napi_schedule - schedule for receive
3622 * @n: entry to schedule
3624 * The entry's receive function will be scheduled to run
3626 void __napi_schedule(struct napi_struct *n)
3628 unsigned long flags;
3630 local_irq_save(flags);
3631 ____napi_schedule(&__get_cpu_var(softnet_data), n);
3632 local_irq_restore(flags);
3634 EXPORT_SYMBOL(__napi_schedule);
3636 void __napi_complete(struct napi_struct *n)
3638 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3639 BUG_ON(n->gro_list);
3641 list_del(&n->poll_list);
3642 smp_mb__before_clear_bit();
3643 clear_bit(NAPI_STATE_SCHED, &n->state);
3645 EXPORT_SYMBOL(__napi_complete);
3647 void napi_complete(struct napi_struct *n)
3649 unsigned long flags;
3652 * don't let napi dequeue from the cpu poll list
3653 * just in case its running on a different cpu
3655 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3659 local_irq_save(flags);
3661 local_irq_restore(flags);
3663 EXPORT_SYMBOL(napi_complete);
3665 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3666 int (*poll)(struct napi_struct *, int), int weight)
3668 INIT_LIST_HEAD(&napi->poll_list);
3669 napi->gro_count = 0;
3670 napi->gro_list = NULL;
3673 napi->weight = weight;
3674 list_add(&napi->dev_list, &dev->napi_list);
3676 #ifdef CONFIG_NETPOLL
3677 spin_lock_init(&napi->poll_lock);
3678 napi->poll_owner = -1;
3680 set_bit(NAPI_STATE_SCHED, &napi->state);
3682 EXPORT_SYMBOL(netif_napi_add);
3684 void netif_napi_del(struct napi_struct *napi)
3686 struct sk_buff *skb, *next;
3688 list_del_init(&napi->dev_list);
3689 napi_free_frags(napi);
3691 for (skb = napi->gro_list; skb; skb = next) {
3697 napi->gro_list = NULL;
3698 napi->gro_count = 0;
3700 EXPORT_SYMBOL(netif_napi_del);
3702 static void net_rx_action(struct softirq_action *h)
3704 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3705 unsigned long time_limit = jiffies + 2;
3706 int budget = netdev_budget;
3709 local_irq_disable();
3711 while (!list_empty(&sd->poll_list)) {
3712 struct napi_struct *n;
3715 /* If softirq window is exhuasted then punt.
3716 * Allow this to run for 2 jiffies since which will allow
3717 * an average latency of 1.5/HZ.
3719 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
3724 /* Even though interrupts have been re-enabled, this
3725 * access is safe because interrupts can only add new
3726 * entries to the tail of this list, and only ->poll()
3727 * calls can remove this head entry from the list.
3729 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
3731 have = netpoll_poll_lock(n);
3735 /* This NAPI_STATE_SCHED test is for avoiding a race
3736 * with netpoll's poll_napi(). Only the entity which
3737 * obtains the lock and sees NAPI_STATE_SCHED set will
3738 * actually make the ->poll() call. Therefore we avoid
3739 * accidently calling ->poll() when NAPI is not scheduled.
3742 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
3743 work = n->poll(n, weight);
3747 WARN_ON_ONCE(work > weight);
3751 local_irq_disable();
3753 /* Drivers must not modify the NAPI state if they
3754 * consume the entire weight. In such cases this code
3755 * still "owns" the NAPI instance and therefore can
3756 * move the instance around on the list at-will.
3758 if (unlikely(work == weight)) {
3759 if (unlikely(napi_disable_pending(n))) {
3762 local_irq_disable();
3764 list_move_tail(&n->poll_list, &sd->poll_list);
3767 netpoll_poll_unlock(have);
3770 net_rps_action_and_irq_enable(sd);
3772 #ifdef CONFIG_NET_DMA
3774 * There may not be any more sk_buffs coming right now, so push
3775 * any pending DMA copies to hardware
3777 dma_issue_pending_all();
3784 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3788 static gifconf_func_t *gifconf_list[NPROTO];
3791 * register_gifconf - register a SIOCGIF handler
3792 * @family: Address family
3793 * @gifconf: Function handler
3795 * Register protocol dependent address dumping routines. The handler
3796 * that is passed must not be freed or reused until it has been replaced
3797 * by another handler.
3799 int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
3801 if (family >= NPROTO)
3803 gifconf_list[family] = gifconf;
3806 EXPORT_SYMBOL(register_gifconf);
3810 * Map an interface index to its name (SIOCGIFNAME)
3814 * We need this ioctl for efficient implementation of the
3815 * if_indextoname() function required by the IPv6 API. Without
3816 * it, we would have to search all the interfaces to find a
3820 static int dev_ifname(struct net *net, struct ifreq __user *arg)
3822 struct net_device *dev;
3826 * Fetch the caller's info block.
3829 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3833 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
3839 strcpy(ifr.ifr_name, dev->name);
3842 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3848 * Perform a SIOCGIFCONF call. This structure will change
3849 * size eventually, and there is nothing I can do about it.
3850 * Thus we will need a 'compatibility mode'.
3853 static int dev_ifconf(struct net *net, char __user *arg)
3856 struct net_device *dev;
3863 * Fetch the caller's info block.
3866 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3873 * Loop over the interfaces, and write an info block for each.
3877 for_each_netdev(net, dev) {
3878 for (i = 0; i < NPROTO; i++) {
3879 if (gifconf_list[i]) {
3882 done = gifconf_list[i](dev, NULL, 0);
3884 done = gifconf_list[i](dev, pos + total,
3894 * All done. Write the updated control block back to the caller.
3896 ifc.ifc_len = total;
3899 * Both BSD and Solaris return 0 here, so we do too.
3901 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3904 #ifdef CONFIG_PROC_FS
3906 * This is invoked by the /proc filesystem handler to display a device
3909 void *dev_seq_start(struct seq_file *seq, loff_t *pos)
3912 struct net *net = seq_file_net(seq);
3914 struct net_device *dev;
3918 return SEQ_START_TOKEN;
3921 for_each_netdev_rcu(net, dev)
3928 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3930 struct net_device *dev = (v == SEQ_START_TOKEN) ?
3931 first_net_device(seq_file_net(seq)) :
3932 next_net_device((struct net_device *)v);
3935 return rcu_dereference(dev);
3938 void dev_seq_stop(struct seq_file *seq, void *v)
3944 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
3946 struct rtnl_link_stats64 temp;
3947 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
3949 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
3950 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
3951 dev->name, stats->rx_bytes, stats->rx_packets,
3953 stats->rx_dropped + stats->rx_missed_errors,
3954 stats->rx_fifo_errors,
3955 stats->rx_length_errors + stats->rx_over_errors +
3956 stats->rx_crc_errors + stats->rx_frame_errors,
3957 stats->rx_compressed, stats->multicast,
3958 stats->tx_bytes, stats->tx_packets,
3959 stats->tx_errors, stats->tx_dropped,
3960 stats->tx_fifo_errors, stats->collisions,
3961 stats->tx_carrier_errors +
3962 stats->tx_aborted_errors +
3963 stats->tx_window_errors +
3964 stats->tx_heartbeat_errors,
3965 stats->tx_compressed);
3969 * Called from the PROCfs module. This now uses the new arbitrary sized
3970 * /proc/net interface to create /proc/net/dev
3972 static int dev_seq_show(struct seq_file *seq, void *v)
3974 if (v == SEQ_START_TOKEN)
3975 seq_puts(seq, "Inter-| Receive "
3977 " face |bytes packets errs drop fifo frame "
3978 "compressed multicast|bytes packets errs "
3979 "drop fifo colls carrier compressed\n");
3981 dev_seq_printf_stats(seq, v);
3985 static struct softnet_data *softnet_get_online(loff_t *pos)
3987 struct softnet_data *sd = NULL;
3989 while (*pos < nr_cpu_ids)
3990 if (cpu_online(*pos)) {
3991 sd = &per_cpu(softnet_data, *pos);
3998 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4000 return softnet_get_online(pos);
4003 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4006 return softnet_get_online(pos);
4009 static void softnet_seq_stop(struct seq_file *seq, void *v)
4013 static int softnet_seq_show(struct seq_file *seq, void *v)
4015 struct softnet_data *sd = v;
4017 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
4018 sd->processed, sd->dropped, sd->time_squeeze, 0,
4019 0, 0, 0, 0, /* was fastroute */
4020 sd->cpu_collision, sd->received_rps);
4024 static const struct seq_operations dev_seq_ops = {
4025 .start = dev_seq_start,
4026 .next = dev_seq_next,
4027 .stop = dev_seq_stop,
4028 .show = dev_seq_show,
4031 static int dev_seq_open(struct inode *inode, struct file *file)
4033 return seq_open_net(inode, file, &dev_seq_ops,
4034 sizeof(struct seq_net_private));
4037 static const struct file_operations dev_seq_fops = {
4038 .owner = THIS_MODULE,
4039 .open = dev_seq_open,
4041 .llseek = seq_lseek,
4042 .release = seq_release_net,
4045 static const struct seq_operations softnet_seq_ops = {
4046 .start = softnet_seq_start,
4047 .next = softnet_seq_next,
4048 .stop = softnet_seq_stop,
4049 .show = softnet_seq_show,
4052 static int softnet_seq_open(struct inode *inode, struct file *file)
4054 return seq_open(file, &softnet_seq_ops);
4057 static const struct file_operations softnet_seq_fops = {
4058 .owner = THIS_MODULE,
4059 .open = softnet_seq_open,
4061 .llseek = seq_lseek,
4062 .release = seq_release,
4065 static void *ptype_get_idx(loff_t pos)
4067 struct packet_type *pt = NULL;
4071 list_for_each_entry_rcu(pt, &ptype_all, list) {
4077 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
4078 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4087 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
4091 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4094 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4096 struct packet_type *pt;
4097 struct list_head *nxt;
4101 if (v == SEQ_START_TOKEN)
4102 return ptype_get_idx(0);
4105 nxt = pt->list.next;
4106 if (pt->type == htons(ETH_P_ALL)) {
4107 if (nxt != &ptype_all)
4110 nxt = ptype_base[0].next;
4112 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
4114 while (nxt == &ptype_base[hash]) {
4115 if (++hash >= PTYPE_HASH_SIZE)
4117 nxt = ptype_base[hash].next;
4120 return list_entry(nxt, struct packet_type, list);
4123 static void ptype_seq_stop(struct seq_file *seq, void *v)
4129 static int ptype_seq_show(struct seq_file *seq, void *v)
4131 struct packet_type *pt = v;
4133 if (v == SEQ_START_TOKEN)
4134 seq_puts(seq, "Type Device Function\n");
4135 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
4136 if (pt->type == htons(ETH_P_ALL))
4137 seq_puts(seq, "ALL ");
4139 seq_printf(seq, "%04x", ntohs(pt->type));
4141 seq_printf(seq, " %-8s %pF\n",
4142 pt->dev ? pt->dev->name : "", pt->func);
4148 static const struct seq_operations ptype_seq_ops = {
4149 .start = ptype_seq_start,
4150 .next = ptype_seq_next,
4151 .stop = ptype_seq_stop,
4152 .show = ptype_seq_show,
4155 static int ptype_seq_open(struct inode *inode, struct file *file)
4157 return seq_open_net(inode, file, &ptype_seq_ops,
4158 sizeof(struct seq_net_private));
4161 static const struct file_operations ptype_seq_fops = {
4162 .owner = THIS_MODULE,
4163 .open = ptype_seq_open,
4165 .llseek = seq_lseek,
4166 .release = seq_release_net,
4170 static int __net_init dev_proc_net_init(struct net *net)
4174 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
4176 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
4178 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
4181 if (wext_proc_init(net))
4187 proc_net_remove(net, "ptype");
4189 proc_net_remove(net, "softnet_stat");
4191 proc_net_remove(net, "dev");
4195 static void __net_exit dev_proc_net_exit(struct net *net)
4197 wext_proc_exit(net);
4199 proc_net_remove(net, "ptype");
4200 proc_net_remove(net, "softnet_stat");
4201 proc_net_remove(net, "dev");
4204 static struct pernet_operations __net_initdata dev_proc_ops = {
4205 .init = dev_proc_net_init,
4206 .exit = dev_proc_net_exit,
4209 static int __init dev_proc_init(void)
4211 return register_pernet_subsys(&dev_proc_ops);
4214 #define dev_proc_init() 0
4215 #endif /* CONFIG_PROC_FS */
4219 * netdev_set_master - set up master/slave pair
4220 * @slave: slave device
4221 * @master: new master device
4223 * Changes the master device of the slave. Pass %NULL to break the
4224 * bonding. The caller must hold the RTNL semaphore. On a failure
4225 * a negative errno code is returned. On success the reference counts
4226 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
4227 * function returns zero.
4229 int netdev_set_master(struct net_device *slave, struct net_device *master)
4231 struct net_device *old = slave->master;
4241 slave->master = master;
4248 slave->flags |= IFF_SLAVE;
4250 slave->flags &= ~IFF_SLAVE;
4252 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4255 EXPORT_SYMBOL(netdev_set_master);
4257 static void dev_change_rx_flags(struct net_device *dev, int flags)
4259 const struct net_device_ops *ops = dev->netdev_ops;
4261 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4262 ops->ndo_change_rx_flags(dev, flags);
4265 static int __dev_set_promiscuity(struct net_device *dev, int inc)
4267 unsigned short old_flags = dev->flags;
4273 dev->flags |= IFF_PROMISC;
4274 dev->promiscuity += inc;
4275 if (dev->promiscuity == 0) {
4278 * If inc causes overflow, untouch promisc and return error.
4281 dev->flags &= ~IFF_PROMISC;
4283 dev->promiscuity -= inc;
4284 printk(KERN_WARNING "%s: promiscuity touches roof, "
4285 "set promiscuity failed, promiscuity feature "
4286 "of device might be broken.\n", dev->name);
4290 if (dev->flags != old_flags) {
4291 printk(KERN_INFO "device %s %s promiscuous mode\n",
4292 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4294 if (audit_enabled) {
4295 current_uid_gid(&uid, &gid);
4296 audit_log(current->audit_context, GFP_ATOMIC,
4297 AUDIT_ANOM_PROMISCUOUS,
4298 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4299 dev->name, (dev->flags & IFF_PROMISC),
4300 (old_flags & IFF_PROMISC),
4301 audit_get_loginuid(current),
4303 audit_get_sessionid(current));
4306 dev_change_rx_flags(dev, IFF_PROMISC);
4312 * dev_set_promiscuity - update promiscuity count on a device
4316 * Add or remove promiscuity from a device. While the count in the device
4317 * remains above zero the interface remains promiscuous. Once it hits zero
4318 * the device reverts back to normal filtering operation. A negative inc
4319 * value is used to drop promiscuity on the device.
4320 * Return 0 if successful or a negative errno code on error.
4322 int dev_set_promiscuity(struct net_device *dev, int inc)
4324 unsigned short old_flags = dev->flags;
4327 err = __dev_set_promiscuity(dev, inc);
4330 if (dev->flags != old_flags)
4331 dev_set_rx_mode(dev);
4334 EXPORT_SYMBOL(dev_set_promiscuity);
4337 * dev_set_allmulti - update allmulti count on a device
4341 * Add or remove reception of all multicast frames to a device. While the
4342 * count in the device remains above zero the interface remains listening
4343 * to all interfaces. Once it hits zero the device reverts back to normal
4344 * filtering operation. A negative @inc value is used to drop the counter
4345 * when releasing a resource needing all multicasts.
4346 * Return 0 if successful or a negative errno code on error.
4349 int dev_set_allmulti(struct net_device *dev, int inc)
4351 unsigned short old_flags = dev->flags;
4355 dev->flags |= IFF_ALLMULTI;
4356 dev->allmulti += inc;
4357 if (dev->allmulti == 0) {
4360 * If inc causes overflow, untouch allmulti and return error.
4363 dev->flags &= ~IFF_ALLMULTI;
4365 dev->allmulti -= inc;
4366 printk(KERN_WARNING "%s: allmulti touches roof, "
4367 "set allmulti failed, allmulti feature of "
4368 "device might be broken.\n", dev->name);
4372 if (dev->flags ^ old_flags) {
4373 dev_change_rx_flags(dev, IFF_ALLMULTI);
4374 dev_set_rx_mode(dev);
4378 EXPORT_SYMBOL(dev_set_allmulti);
4381 * Upload unicast and multicast address lists to device and
4382 * configure RX filtering. When the device doesn't support unicast
4383 * filtering it is put in promiscuous mode while unicast addresses
4386 void __dev_set_rx_mode(struct net_device *dev)
4388 const struct net_device_ops *ops = dev->netdev_ops;
4390 /* dev_open will call this function so the list will stay sane. */
4391 if (!(dev->flags&IFF_UP))
4394 if (!netif_device_present(dev))
4397 if (ops->ndo_set_rx_mode)
4398 ops->ndo_set_rx_mode(dev);
4400 /* Unicast addresses changes may only happen under the rtnl,
4401 * therefore calling __dev_set_promiscuity here is safe.
4403 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4404 __dev_set_promiscuity(dev, 1);
4405 dev->uc_promisc = 1;
4406 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4407 __dev_set_promiscuity(dev, -1);
4408 dev->uc_promisc = 0;
4411 if (ops->ndo_set_multicast_list)
4412 ops->ndo_set_multicast_list(dev);
4416 void dev_set_rx_mode(struct net_device *dev)
4418 netif_addr_lock_bh(dev);
4419 __dev_set_rx_mode(dev);
4420 netif_addr_unlock_bh(dev);
4424 * dev_get_flags - get flags reported to userspace
4427 * Get the combination of flag bits exported through APIs to userspace.
4429 unsigned dev_get_flags(const struct net_device *dev)
4433 flags = (dev->flags & ~(IFF_PROMISC |
4438 (dev->gflags & (IFF_PROMISC |
4441 if (netif_running(dev)) {
4442 if (netif_oper_up(dev))
4443 flags |= IFF_RUNNING;
4444 if (netif_carrier_ok(dev))
4445 flags |= IFF_LOWER_UP;
4446 if (netif_dormant(dev))
4447 flags |= IFF_DORMANT;
4452 EXPORT_SYMBOL(dev_get_flags);
4454 int __dev_change_flags(struct net_device *dev, unsigned int flags)
4456 int old_flags = dev->flags;
4462 * Set the flags on our device.
4465 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4466 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4468 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4472 * Load in the correct multicast list now the flags have changed.
4475 if ((old_flags ^ flags) & IFF_MULTICAST)
4476 dev_change_rx_flags(dev, IFF_MULTICAST);
4478 dev_set_rx_mode(dev);
4481 * Have we downed the interface. We handle IFF_UP ourselves
4482 * according to user attempts to set it, rather than blindly
4487 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
4488 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4491 dev_set_rx_mode(dev);
4494 if ((flags ^ dev->gflags) & IFF_PROMISC) {
4495 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4497 dev->gflags ^= IFF_PROMISC;
4498 dev_set_promiscuity(dev, inc);
4501 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4502 is important. Some (broken) drivers set IFF_PROMISC, when
4503 IFF_ALLMULTI is requested not asking us and not reporting.
4505 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4506 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4508 dev->gflags ^= IFF_ALLMULTI;
4509 dev_set_allmulti(dev, inc);
4515 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4517 unsigned int changes = dev->flags ^ old_flags;
4519 if (changes & IFF_UP) {
4520 if (dev->flags & IFF_UP)
4521 call_netdevice_notifiers(NETDEV_UP, dev);
4523 call_netdevice_notifiers(NETDEV_DOWN, dev);
4526 if (dev->flags & IFF_UP &&
4527 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4528 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4532 * dev_change_flags - change device settings
4534 * @flags: device state flags
4536 * Change settings on device based state flags. The flags are
4537 * in the userspace exported format.
4539 int dev_change_flags(struct net_device *dev, unsigned flags)
4542 int old_flags = dev->flags;
4544 ret = __dev_change_flags(dev, flags);
4548 changes = old_flags ^ dev->flags;
4550 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4552 __dev_notify_flags(dev, old_flags);
4555 EXPORT_SYMBOL(dev_change_flags);
4558 * dev_set_mtu - Change maximum transfer unit
4560 * @new_mtu: new transfer unit
4562 * Change the maximum transfer size of the network device.
4564 int dev_set_mtu(struct net_device *dev, int new_mtu)
4566 const struct net_device_ops *ops = dev->netdev_ops;
4569 if (new_mtu == dev->mtu)
4572 /* MTU must be positive. */
4576 if (!netif_device_present(dev))
4580 if (ops->ndo_change_mtu)
4581 err = ops->ndo_change_mtu(dev, new_mtu);
4585 if (!err && dev->flags & IFF_UP)
4586 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4589 EXPORT_SYMBOL(dev_set_mtu);
4592 * dev_set_mac_address - Change Media Access Control Address
4596 * Change the hardware (MAC) address of the device
4598 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4600 const struct net_device_ops *ops = dev->netdev_ops;
4603 if (!ops->ndo_set_mac_address)
4605 if (sa->sa_family != dev->type)
4607 if (!netif_device_present(dev))
4609 err = ops->ndo_set_mac_address(dev, sa);
4611 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4614 EXPORT_SYMBOL(dev_set_mac_address);
4617 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
4619 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
4622 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
4628 case SIOCGIFFLAGS: /* Get interface flags */
4629 ifr->ifr_flags = (short) dev_get_flags(dev);
4632 case SIOCGIFMETRIC: /* Get the metric on the interface
4633 (currently unused) */
4634 ifr->ifr_metric = 0;
4637 case SIOCGIFMTU: /* Get the MTU of a device */
4638 ifr->ifr_mtu = dev->mtu;
4643 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4645 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4646 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4647 ifr->ifr_hwaddr.sa_family = dev->type;
4655 ifr->ifr_map.mem_start = dev->mem_start;
4656 ifr->ifr_map.mem_end = dev->mem_end;
4657 ifr->ifr_map.base_addr = dev->base_addr;
4658 ifr->ifr_map.irq = dev->irq;
4659 ifr->ifr_map.dma = dev->dma;
4660 ifr->ifr_map.port = dev->if_port;
4664 ifr->ifr_ifindex = dev->ifindex;
4668 ifr->ifr_qlen = dev->tx_queue_len;
4672 /* dev_ioctl() should ensure this case
4684 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4686 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4689 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4690 const struct net_device_ops *ops;
4695 ops = dev->netdev_ops;
4698 case SIOCSIFFLAGS: /* Set interface flags */
4699 return dev_change_flags(dev, ifr->ifr_flags);
4701 case SIOCSIFMETRIC: /* Set the metric on the interface
4702 (currently unused) */
4705 case SIOCSIFMTU: /* Set the MTU of a device */
4706 return dev_set_mtu(dev, ifr->ifr_mtu);
4709 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
4711 case SIOCSIFHWBROADCAST:
4712 if (ifr->ifr_hwaddr.sa_family != dev->type)
4714 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4715 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4716 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4720 if (ops->ndo_set_config) {
4721 if (!netif_device_present(dev))
4723 return ops->ndo_set_config(dev, &ifr->ifr_map);
4728 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4729 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4731 if (!netif_device_present(dev))
4733 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
4736 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4737 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4739 if (!netif_device_present(dev))
4741 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
4744 if (ifr->ifr_qlen < 0)
4746 dev->tx_queue_len = ifr->ifr_qlen;
4750 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4751 return dev_change_name(dev, ifr->ifr_newname);
4754 * Unknown or private ioctl
4757 if ((cmd >= SIOCDEVPRIVATE &&
4758 cmd <= SIOCDEVPRIVATE + 15) ||
4759 cmd == SIOCBONDENSLAVE ||
4760 cmd == SIOCBONDRELEASE ||
4761 cmd == SIOCBONDSETHWADDR ||
4762 cmd == SIOCBONDSLAVEINFOQUERY ||
4763 cmd == SIOCBONDINFOQUERY ||
4764 cmd == SIOCBONDCHANGEACTIVE ||
4765 cmd == SIOCGMIIPHY ||
4766 cmd == SIOCGMIIREG ||
4767 cmd == SIOCSMIIREG ||
4768 cmd == SIOCBRADDIF ||
4769 cmd == SIOCBRDELIF ||
4770 cmd == SIOCSHWTSTAMP ||
4771 cmd == SIOCWANDEV) {
4773 if (ops->ndo_do_ioctl) {
4774 if (netif_device_present(dev))
4775 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4787 * This function handles all "interface"-type I/O control requests. The actual
4788 * 'doing' part of this is dev_ifsioc above.
4792 * dev_ioctl - network device ioctl
4793 * @net: the applicable net namespace
4794 * @cmd: command to issue
4795 * @arg: pointer to a struct ifreq in user space
4797 * Issue ioctl functions to devices. This is normally called by the
4798 * user space syscall interfaces but can sometimes be useful for
4799 * other purposes. The return value is the return from the syscall if
4800 * positive or a negative errno code on error.
4803 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4809 /* One special case: SIOCGIFCONF takes ifconf argument
4810 and requires shared lock, because it sleeps writing
4814 if (cmd == SIOCGIFCONF) {
4816 ret = dev_ifconf(net, (char __user *) arg);
4820 if (cmd == SIOCGIFNAME)
4821 return dev_ifname(net, (struct ifreq __user *)arg);
4823 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4826 ifr.ifr_name[IFNAMSIZ-1] = 0;
4828 colon = strchr(ifr.ifr_name, ':');
4833 * See which interface the caller is talking about.
4838 * These ioctl calls:
4839 * - can be done by all.
4840 * - atomic and do not require locking.
4851 dev_load(net, ifr.ifr_name);
4853 ret = dev_ifsioc_locked(net, &ifr, cmd);
4858 if (copy_to_user(arg, &ifr,
4859 sizeof(struct ifreq)))
4865 dev_load(net, ifr.ifr_name);
4867 ret = dev_ethtool(net, &ifr);
4872 if (copy_to_user(arg, &ifr,
4873 sizeof(struct ifreq)))
4879 * These ioctl calls:
4880 * - require superuser power.
4881 * - require strict serialization.
4887 if (!capable(CAP_NET_ADMIN))
4889 dev_load(net, ifr.ifr_name);
4891 ret = dev_ifsioc(net, &ifr, cmd);
4896 if (copy_to_user(arg, &ifr,
4897 sizeof(struct ifreq)))
4903 * These ioctl calls:
4904 * - require superuser power.
4905 * - require strict serialization.
4906 * - do not return a value
4916 case SIOCSIFHWBROADCAST:
4919 case SIOCBONDENSLAVE:
4920 case SIOCBONDRELEASE:
4921 case SIOCBONDSETHWADDR:
4922 case SIOCBONDCHANGEACTIVE:
4926 if (!capable(CAP_NET_ADMIN))
4929 case SIOCBONDSLAVEINFOQUERY:
4930 case SIOCBONDINFOQUERY:
4931 dev_load(net, ifr.ifr_name);
4933 ret = dev_ifsioc(net, &ifr, cmd);
4938 /* Get the per device memory space. We can add this but
4939 * currently do not support it */
4941 /* Set the per device memory buffer space.
4942 * Not applicable in our case */
4947 * Unknown or private ioctl.
4950 if (cmd == SIOCWANDEV ||
4951 (cmd >= SIOCDEVPRIVATE &&
4952 cmd <= SIOCDEVPRIVATE + 15)) {
4953 dev_load(net, ifr.ifr_name);
4955 ret = dev_ifsioc(net, &ifr, cmd);
4957 if (!ret && copy_to_user(arg, &ifr,
4958 sizeof(struct ifreq)))
4962 /* Take care of Wireless Extensions */
4963 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
4964 return wext_handle_ioctl(net, &ifr, cmd, arg);
4971 * dev_new_index - allocate an ifindex
4972 * @net: the applicable net namespace
4974 * Returns a suitable unique value for a new device interface
4975 * number. The caller must hold the rtnl semaphore or the
4976 * dev_base_lock to be sure it remains unique.
4978 static int dev_new_index(struct net *net)
4984 if (!__dev_get_by_index(net, ifindex))
4989 /* Delayed registration/unregisteration */
4990 static LIST_HEAD(net_todo_list);
4992 static void net_set_todo(struct net_device *dev)
4994 list_add_tail(&dev->todo_list, &net_todo_list);
4997 static void rollback_registered_many(struct list_head *head)
4999 struct net_device *dev, *tmp;
5001 BUG_ON(dev_boot_phase);
5004 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
5005 /* Some devices call without registering
5006 * for initialization unwind. Remove those
5007 * devices and proceed with the remaining.
5009 if (dev->reg_state == NETREG_UNINITIALIZED) {
5010 pr_debug("unregister_netdevice: device %s/%p never "
5011 "was registered\n", dev->name, dev);
5014 list_del(&dev->unreg_list);
5018 BUG_ON(dev->reg_state != NETREG_REGISTERED);
5021 /* If device is running, close it first. */
5022 dev_close_many(head);
5024 list_for_each_entry(dev, head, unreg_list) {
5025 /* And unlink it from device chain. */
5026 unlist_netdevice(dev);
5028 dev->reg_state = NETREG_UNREGISTERING;
5033 list_for_each_entry(dev, head, unreg_list) {
5034 /* Shutdown queueing discipline. */
5038 /* Notify protocols, that we are about to destroy
5039 this device. They should clean all the things.
5041 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5043 if (!dev->rtnl_link_ops ||
5044 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5045 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5048 * Flush the unicast and multicast chains
5053 if (dev->netdev_ops->ndo_uninit)
5054 dev->netdev_ops->ndo_uninit(dev);
5056 /* Notifier chain MUST detach us from master device. */
5057 WARN_ON(dev->master);
5059 /* Remove entries from kobject tree */
5060 netdev_unregister_kobject(dev);
5063 /* Process any work delayed until the end of the batch */
5064 dev = list_first_entry(head, struct net_device, unreg_list);
5065 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5069 list_for_each_entry(dev, head, unreg_list)
5073 static void rollback_registered(struct net_device *dev)
5077 list_add(&dev->unreg_list, &single);
5078 rollback_registered_many(&single);
5082 unsigned long netdev_fix_features(unsigned long features, const char *name)
5084 /* Fix illegal SG+CSUM combinations. */
5085 if ((features & NETIF_F_SG) &&
5086 !(features & NETIF_F_ALL_CSUM)) {
5088 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
5089 "checksum feature.\n", name);
5090 features &= ~NETIF_F_SG;
5093 /* TSO requires that SG is present as well. */
5094 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
5096 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
5097 "SG feature.\n", name);
5098 features &= ~NETIF_F_TSO;
5101 if (features & NETIF_F_UFO) {
5102 /* maybe split UFO into V4 and V6? */
5103 if (!((features & NETIF_F_GEN_CSUM) ||
5104 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5105 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5107 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
5108 "since no checksum offload features.\n",
5110 features &= ~NETIF_F_UFO;
5113 if (!(features & NETIF_F_SG)) {
5115 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
5116 "since no NETIF_F_SG feature.\n", name);
5117 features &= ~NETIF_F_UFO;
5123 EXPORT_SYMBOL(netdev_fix_features);
5126 * netif_stacked_transfer_operstate - transfer operstate
5127 * @rootdev: the root or lower level device to transfer state from
5128 * @dev: the device to transfer operstate to
5130 * Transfer operational state from root to device. This is normally
5131 * called when a stacking relationship exists between the root
5132 * device and the device(a leaf device).
5134 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5135 struct net_device *dev)
5137 if (rootdev->operstate == IF_OPER_DORMANT)
5138 netif_dormant_on(dev);
5140 netif_dormant_off(dev);
5142 if (netif_carrier_ok(rootdev)) {
5143 if (!netif_carrier_ok(dev))
5144 netif_carrier_on(dev);
5146 if (netif_carrier_ok(dev))
5147 netif_carrier_off(dev);
5150 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5153 static int netif_alloc_rx_queues(struct net_device *dev)
5155 unsigned int i, count = dev->num_rx_queues;
5156 struct netdev_rx_queue *rx;
5160 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5162 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5167 for (i = 0; i < count; i++)
5173 static void netdev_init_one_queue(struct net_device *dev,
5174 struct netdev_queue *queue, void *_unused)
5176 /* Initialize queue lock */
5177 spin_lock_init(&queue->_xmit_lock);
5178 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5179 queue->xmit_lock_owner = -1;
5180 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5184 static int netif_alloc_netdev_queues(struct net_device *dev)
5186 unsigned int count = dev->num_tx_queues;
5187 struct netdev_queue *tx;
5191 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5193 pr_err("netdev: Unable to allocate %u tx queues.\n",
5199 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5200 spin_lock_init(&dev->tx_global_lock);
5206 * register_netdevice - register a network device
5207 * @dev: device to register
5209 * Take a completed network device structure and add it to the kernel
5210 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5211 * chain. 0 is returned on success. A negative errno code is returned
5212 * on a failure to set up the device, or if the name is a duplicate.
5214 * Callers must hold the rtnl semaphore. You may want
5215 * register_netdev() instead of this.
5218 * The locking appears insufficient to guarantee two parallel registers
5219 * will not get the same name.
5222 int register_netdevice(struct net_device *dev)
5225 struct net *net = dev_net(dev);
5227 BUG_ON(dev_boot_phase);
5232 /* When net_device's are persistent, this will be fatal. */
5233 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5236 spin_lock_init(&dev->addr_list_lock);
5237 netdev_set_addr_lockdep_class(dev);
5241 /* Init, if this function is available */
5242 if (dev->netdev_ops->ndo_init) {
5243 ret = dev->netdev_ops->ndo_init(dev);
5251 ret = dev_get_valid_name(dev, dev->name, 0);
5255 dev->ifindex = dev_new_index(net);
5256 if (dev->iflink == -1)
5257 dev->iflink = dev->ifindex;
5259 /* Fix illegal checksum combinations */
5260 if ((dev->features & NETIF_F_HW_CSUM) &&
5261 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5262 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
5264 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5267 if ((dev->features & NETIF_F_NO_CSUM) &&
5268 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5269 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
5271 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5274 dev->features = netdev_fix_features(dev->features, dev->name);
5276 /* Enable software GSO if SG is supported. */
5277 if (dev->features & NETIF_F_SG)
5278 dev->features |= NETIF_F_GSO;
5280 /* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
5281 * vlan_dev_init() will do the dev->features check, so these features
5282 * are enabled only if supported by underlying device.
5284 dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);
5286 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5287 ret = notifier_to_errno(ret);
5291 ret = netdev_register_kobject(dev);
5294 dev->reg_state = NETREG_REGISTERED;
5297 * Default initial state at registry is that the
5298 * device is present.
5301 set_bit(__LINK_STATE_PRESENT, &dev->state);
5303 dev_init_scheduler(dev);
5305 list_netdevice(dev);
5307 /* Notify protocols, that a new device appeared. */
5308 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5309 ret = notifier_to_errno(ret);
5311 rollback_registered(dev);
5312 dev->reg_state = NETREG_UNREGISTERED;
5315 * Prevent userspace races by waiting until the network
5316 * device is fully setup before sending notifications.
5318 if (!dev->rtnl_link_ops ||
5319 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5320 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5326 if (dev->netdev_ops->ndo_uninit)
5327 dev->netdev_ops->ndo_uninit(dev);
5330 EXPORT_SYMBOL(register_netdevice);
5333 * init_dummy_netdev - init a dummy network device for NAPI
5334 * @dev: device to init
5336 * This takes a network device structure and initialize the minimum
5337 * amount of fields so it can be used to schedule NAPI polls without
5338 * registering a full blown interface. This is to be used by drivers
5339 * that need to tie several hardware interfaces to a single NAPI
5340 * poll scheduler due to HW limitations.
5342 int init_dummy_netdev(struct net_device *dev)
5344 /* Clear everything. Note we don't initialize spinlocks
5345 * are they aren't supposed to be taken by any of the
5346 * NAPI code and this dummy netdev is supposed to be
5347 * only ever used for NAPI polls
5349 memset(dev, 0, sizeof(struct net_device));
5351 /* make sure we BUG if trying to hit standard
5352 * register/unregister code path
5354 dev->reg_state = NETREG_DUMMY;
5356 /* NAPI wants this */
5357 INIT_LIST_HEAD(&dev->napi_list);
5359 /* a dummy interface is started by default */
5360 set_bit(__LINK_STATE_PRESENT, &dev->state);
5361 set_bit(__LINK_STATE_START, &dev->state);
5363 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5364 * because users of this 'device' dont need to change
5370 EXPORT_SYMBOL_GPL(init_dummy_netdev);
5374 * register_netdev - register a network device
5375 * @dev: device to register
5377 * Take a completed network device structure and add it to the kernel
5378 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5379 * chain. 0 is returned on success. A negative errno code is returned
5380 * on a failure to set up the device, or if the name is a duplicate.
5382 * This is a wrapper around register_netdevice that takes the rtnl semaphore
5383 * and expands the device name if you passed a format string to
5386 int register_netdev(struct net_device *dev)
5393 * If the name is a format string the caller wants us to do a
5396 if (strchr(dev->name, '%')) {
5397 err = dev_alloc_name(dev, dev->name);
5402 err = register_netdevice(dev);
5407 EXPORT_SYMBOL(register_netdev);
5409 int netdev_refcnt_read(const struct net_device *dev)
5413 for_each_possible_cpu(i)
5414 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5417 EXPORT_SYMBOL(netdev_refcnt_read);
5420 * netdev_wait_allrefs - wait until all references are gone.
5422 * This is called when unregistering network devices.
5424 * Any protocol or device that holds a reference should register
5425 * for netdevice notification, and cleanup and put back the
5426 * reference if they receive an UNREGISTER event.
5427 * We can get stuck here if buggy protocols don't correctly
5430 static void netdev_wait_allrefs(struct net_device *dev)
5432 unsigned long rebroadcast_time, warning_time;
5435 linkwatch_forget_dev(dev);
5437 rebroadcast_time = warning_time = jiffies;
5438 refcnt = netdev_refcnt_read(dev);
5440 while (refcnt != 0) {
5441 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5444 /* Rebroadcast unregister notification */
5445 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5446 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
5447 * should have already handle it the first time */
5449 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5451 /* We must not have linkwatch events
5452 * pending on unregister. If this
5453 * happens, we simply run the queue
5454 * unscheduled, resulting in a noop
5457 linkwatch_run_queue();
5462 rebroadcast_time = jiffies;
5467 refcnt = netdev_refcnt_read(dev);
5469 if (time_after(jiffies, warning_time + 10 * HZ)) {
5470 printk(KERN_EMERG "unregister_netdevice: "
5471 "waiting for %s to become free. Usage "
5474 warning_time = jiffies;
5483 * register_netdevice(x1);
5484 * register_netdevice(x2);
5486 * unregister_netdevice(y1);
5487 * unregister_netdevice(y2);
5493 * We are invoked by rtnl_unlock().
5494 * This allows us to deal with problems:
5495 * 1) We can delete sysfs objects which invoke hotplug
5496 * without deadlocking with linkwatch via keventd.
5497 * 2) Since we run with the RTNL semaphore not held, we can sleep
5498 * safely in order to wait for the netdev refcnt to drop to zero.
5500 * We must not return until all unregister events added during
5501 * the interval the lock was held have been completed.
5503 void netdev_run_todo(void)
5505 struct list_head list;
5507 /* Snapshot list, allow later requests */
5508 list_replace_init(&net_todo_list, &list);
5512 while (!list_empty(&list)) {
5513 struct net_device *dev
5514 = list_first_entry(&list, struct net_device, todo_list);
5515 list_del(&dev->todo_list);
5517 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5518 printk(KERN_ERR "network todo '%s' but state %d\n",
5519 dev->name, dev->reg_state);
5524 dev->reg_state = NETREG_UNREGISTERED;
5526 on_each_cpu(flush_backlog, dev, 1);
5528 netdev_wait_allrefs(dev);
5531 BUG_ON(netdev_refcnt_read(dev));
5532 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
5533 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
5534 WARN_ON(dev->dn_ptr);
5536 if (dev->destructor)
5537 dev->destructor(dev);
5539 /* Free network device */
5540 kobject_put(&dev->dev.kobj);
5544 /* Convert net_device_stats to rtnl_link_stats64. They have the same
5545 * fields in the same order, with only the type differing.
5547 static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5548 const struct net_device_stats *netdev_stats)
5550 #if BITS_PER_LONG == 64
5551 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5552 memcpy(stats64, netdev_stats, sizeof(*stats64));
5554 size_t i, n = sizeof(*stats64) / sizeof(u64);
5555 const unsigned long *src = (const unsigned long *)netdev_stats;
5556 u64 *dst = (u64 *)stats64;
5558 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5559 sizeof(*stats64) / sizeof(u64));
5560 for (i = 0; i < n; i++)
5566 * dev_get_stats - get network device statistics
5567 * @dev: device to get statistics from
5568 * @storage: place to store stats
5570 * Get network statistics from device. Return @storage.
5571 * The device driver may provide its own method by setting
5572 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5573 * otherwise the internal statistics structure is used.
5575 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5576 struct rtnl_link_stats64 *storage)
5578 const struct net_device_ops *ops = dev->netdev_ops;
5580 if (ops->ndo_get_stats64) {
5581 memset(storage, 0, sizeof(*storage));
5582 ops->ndo_get_stats64(dev, storage);
5583 } else if (ops->ndo_get_stats) {
5584 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5586 netdev_stats_to_stats64(storage, &dev->stats);
5588 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5591 EXPORT_SYMBOL(dev_get_stats);
5593 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5595 struct netdev_queue *queue = dev_ingress_queue(dev);
5597 #ifdef CONFIG_NET_CLS_ACT
5600 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5603 netdev_init_one_queue(dev, queue, NULL);
5604 queue->qdisc = &noop_qdisc;
5605 queue->qdisc_sleeping = &noop_qdisc;
5606 rcu_assign_pointer(dev->ingress_queue, queue);
5612 * alloc_netdev_mqs - allocate network device
5613 * @sizeof_priv: size of private data to allocate space for
5614 * @name: device name format string
5615 * @setup: callback to initialize device
5616 * @txqs: the number of TX subqueues to allocate
5617 * @rxqs: the number of RX subqueues to allocate
5619 * Allocates a struct net_device with private data area for driver use
5620 * and performs basic initialization. Also allocates subquue structs
5621 * for each queue on the device.
5623 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5624 void (*setup)(struct net_device *),
5625 unsigned int txqs, unsigned int rxqs)
5627 struct net_device *dev;
5629 struct net_device *p;
5631 BUG_ON(strlen(name) >= sizeof(dev->name));
5634 pr_err("alloc_netdev: Unable to allocate device "
5635 "with zero queues.\n");
5641 pr_err("alloc_netdev: Unable to allocate device "
5642 "with zero RX queues.\n");
5647 alloc_size = sizeof(struct net_device);
5649 /* ensure 32-byte alignment of private area */
5650 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5651 alloc_size += sizeof_priv;
5653 /* ensure 32-byte alignment of whole construct */
5654 alloc_size += NETDEV_ALIGN - 1;
5656 p = kzalloc(alloc_size, GFP_KERNEL);
5658 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
5662 dev = PTR_ALIGN(p, NETDEV_ALIGN);
5663 dev->padded = (char *)dev - (char *)p;
5665 dev->pcpu_refcnt = alloc_percpu(int);
5666 if (!dev->pcpu_refcnt)
5669 if (dev_addr_init(dev))
5675 dev_net_set(dev, &init_net);
5677 dev->gso_max_size = GSO_MAX_SIZE;
5679 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5680 dev->ethtool_ntuple_list.count = 0;
5681 INIT_LIST_HEAD(&dev->napi_list);
5682 INIT_LIST_HEAD(&dev->unreg_list);
5683 INIT_LIST_HEAD(&dev->link_watch_list);
5684 dev->priv_flags = IFF_XMIT_DST_RELEASE;
5687 dev->num_tx_queues = txqs;
5688 dev->real_num_tx_queues = txqs;
5689 if (netif_alloc_netdev_queues(dev))
5693 dev->num_rx_queues = rxqs;
5694 dev->real_num_rx_queues = rxqs;
5695 if (netif_alloc_rx_queues(dev))
5699 strcpy(dev->name, name);
5707 free_percpu(dev->pcpu_refcnt);
5717 EXPORT_SYMBOL(alloc_netdev_mqs);
5720 * free_netdev - free network device
5723 * This function does the last stage of destroying an allocated device
5724 * interface. The reference to the device object is released.
5725 * If this is the last reference then it will be freed.
5727 void free_netdev(struct net_device *dev)
5729 struct napi_struct *p, *n;
5731 release_net(dev_net(dev));
5738 kfree(rcu_dereference_raw(dev->ingress_queue));
5740 /* Flush device addresses */
5741 dev_addr_flush(dev);
5743 /* Clear ethtool n-tuple list */
5744 ethtool_ntuple_flush(dev);
5746 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5749 free_percpu(dev->pcpu_refcnt);
5750 dev->pcpu_refcnt = NULL;
5752 /* Compatibility with error handling in drivers */
5753 if (dev->reg_state == NETREG_UNINITIALIZED) {
5754 kfree((char *)dev - dev->padded);
5758 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5759 dev->reg_state = NETREG_RELEASED;
5761 /* will free via device release */
5762 put_device(&dev->dev);
5764 EXPORT_SYMBOL(free_netdev);
5767 * synchronize_net - Synchronize with packet receive processing
5769 * Wait for packets currently being received to be done.
5770 * Does not block later packets from starting.
5772 void synchronize_net(void)
5777 EXPORT_SYMBOL(synchronize_net);
5780 * unregister_netdevice_queue - remove device from the kernel
5784 * This function shuts down a device interface and removes it
5785 * from the kernel tables.
5786 * If head not NULL, device is queued to be unregistered later.
5788 * Callers must hold the rtnl semaphore. You may want
5789 * unregister_netdev() instead of this.
5792 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5797 list_move_tail(&dev->unreg_list, head);
5799 rollback_registered(dev);
5800 /* Finish processing unregister after unlock */
5804 EXPORT_SYMBOL(unregister_netdevice_queue);
5807 * unregister_netdevice_many - unregister many devices
5808 * @head: list of devices
5810 void unregister_netdevice_many(struct list_head *head)
5812 struct net_device *dev;
5814 if (!list_empty(head)) {
5815 rollback_registered_many(head);
5816 list_for_each_entry(dev, head, unreg_list)
5820 EXPORT_SYMBOL(unregister_netdevice_many);
5823 * unregister_netdev - remove device from the kernel
5826 * This function shuts down a device interface and removes it
5827 * from the kernel tables.
5829 * This is just a wrapper for unregister_netdevice that takes
5830 * the rtnl semaphore. In general you want to use this and not
5831 * unregister_netdevice.
5833 void unregister_netdev(struct net_device *dev)
5836 unregister_netdevice(dev);
5839 EXPORT_SYMBOL(unregister_netdev);
5842 * dev_change_net_namespace - move device to different nethost namespace
5844 * @net: network namespace
5845 * @pat: If not NULL name pattern to try if the current device name
5846 * is already taken in the destination network namespace.
5848 * This function shuts down a device interface and moves it
5849 * to a new network namespace. On success 0 is returned, on
5850 * a failure a netagive errno code is returned.
5852 * Callers must hold the rtnl semaphore.
5855 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5861 /* Don't allow namespace local devices to be moved. */
5863 if (dev->features & NETIF_F_NETNS_LOCAL)
5866 /* Ensure the device has been registrered */
5868 if (dev->reg_state != NETREG_REGISTERED)
5871 /* Get out if there is nothing todo */
5873 if (net_eq(dev_net(dev), net))
5876 /* Pick the destination device name, and ensure
5877 * we can use it in the destination network namespace.
5880 if (__dev_get_by_name(net, dev->name)) {
5881 /* We get here if we can't use the current device name */
5884 if (dev_get_valid_name(dev, pat, 1))
5889 * And now a mini version of register_netdevice unregister_netdevice.
5892 /* If device is running close it first. */
5895 /* And unlink it from device chain */
5897 unlist_netdevice(dev);
5901 /* Shutdown queueing discipline. */
5904 /* Notify protocols, that we are about to destroy
5905 this device. They should clean all the things.
5907 Note that dev->reg_state stays at NETREG_REGISTERED.
5908 This is wanted because this way 8021q and macvlan know
5909 the device is just moving and can keep their slaves up.
5911 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5912 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5915 * Flush the unicast and multicast chains
5920 /* Actually switch the network namespace */
5921 dev_net_set(dev, net);
5923 /* If there is an ifindex conflict assign a new one */
5924 if (__dev_get_by_index(net, dev->ifindex)) {
5925 int iflink = (dev->iflink == dev->ifindex);
5926 dev->ifindex = dev_new_index(net);
5928 dev->iflink = dev->ifindex;
5931 /* Fixup kobjects */
5932 err = device_rename(&dev->dev, dev->name);
5935 /* Add the device back in the hashes */
5936 list_netdevice(dev);
5938 /* Notify protocols, that a new device appeared. */
5939 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5942 * Prevent userspace races by waiting until the network
5943 * device is fully setup before sending notifications.
5945 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5952 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5954 static int dev_cpu_callback(struct notifier_block *nfb,
5955 unsigned long action,
5958 struct sk_buff **list_skb;
5959 struct sk_buff *skb;
5960 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5961 struct softnet_data *sd, *oldsd;
5963 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
5966 local_irq_disable();
5967 cpu = smp_processor_id();
5968 sd = &per_cpu(softnet_data, cpu);
5969 oldsd = &per_cpu(softnet_data, oldcpu);
5971 /* Find end of our completion_queue. */
5972 list_skb = &sd->completion_queue;
5974 list_skb = &(*list_skb)->next;
5975 /* Append completion queue from offline CPU. */
5976 *list_skb = oldsd->completion_queue;
5977 oldsd->completion_queue = NULL;
5979 /* Append output queue from offline CPU. */
5980 if (oldsd->output_queue) {
5981 *sd->output_queue_tailp = oldsd->output_queue;
5982 sd->output_queue_tailp = oldsd->output_queue_tailp;
5983 oldsd->output_queue = NULL;
5984 oldsd->output_queue_tailp = &oldsd->output_queue;
5987 raise_softirq_irqoff(NET_TX_SOFTIRQ);
5990 /* Process offline CPU's input_pkt_queue */
5991 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
5993 input_queue_head_incr(oldsd);
5995 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
5997 input_queue_head_incr(oldsd);
6005 * netdev_increment_features - increment feature set by one
6006 * @all: current feature set
6007 * @one: new feature set
6008 * @mask: mask feature set
6010 * Computes a new feature set after adding a device with feature set
6011 * @one to the master device with current feature set @all. Will not
6012 * enable anything that is off in @mask. Returns the new feature set.
6014 unsigned long netdev_increment_features(unsigned long all, unsigned long one,
6017 /* If device needs checksumming, downgrade to it. */
6018 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
6019 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
6020 else if (mask & NETIF_F_ALL_CSUM) {
6021 /* If one device supports v4/v6 checksumming, set for all. */
6022 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
6023 !(all & NETIF_F_GEN_CSUM)) {
6024 all &= ~NETIF_F_ALL_CSUM;
6025 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
6028 /* If one device supports hw checksumming, set for all. */
6029 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
6030 all &= ~NETIF_F_ALL_CSUM;
6031 all |= NETIF_F_HW_CSUM;
6035 one |= NETIF_F_ALL_CSUM;
6037 one |= all & NETIF_F_ONE_FOR_ALL;
6038 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
6039 all |= one & mask & NETIF_F_ONE_FOR_ALL;
6043 EXPORT_SYMBOL(netdev_increment_features);
6045 static struct hlist_head *netdev_create_hash(void)
6048 struct hlist_head *hash;
6050 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6052 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6053 INIT_HLIST_HEAD(&hash[i]);
6058 /* Initialize per network namespace state */
6059 static int __net_init netdev_init(struct net *net)
6061 INIT_LIST_HEAD(&net->dev_base_head);
6063 net->dev_name_head = netdev_create_hash();
6064 if (net->dev_name_head == NULL)
6067 net->dev_index_head = netdev_create_hash();
6068 if (net->dev_index_head == NULL)
6074 kfree(net->dev_name_head);
6080 * netdev_drivername - network driver for the device
6081 * @dev: network device
6082 * @buffer: buffer for resulting name
6083 * @len: size of buffer
6085 * Determine network driver for device.
6087 char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
6089 const struct device_driver *driver;
6090 const struct device *parent;
6092 if (len <= 0 || !buffer)
6096 parent = dev->dev.parent;
6101 driver = parent->driver;
6102 if (driver && driver->name)
6103 strlcpy(buffer, driver->name, len);
6107 static int __netdev_printk(const char *level, const struct net_device *dev,
6108 struct va_format *vaf)
6112 if (dev && dev->dev.parent)
6113 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6114 netdev_name(dev), vaf);
6116 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6118 r = printk("%s(NULL net_device): %pV", level, vaf);
6123 int netdev_printk(const char *level, const struct net_device *dev,
6124 const char *format, ...)
6126 struct va_format vaf;
6130 va_start(args, format);
6135 r = __netdev_printk(level, dev, &vaf);
6140 EXPORT_SYMBOL(netdev_printk);
6142 #define define_netdev_printk_level(func, level) \
6143 int func(const struct net_device *dev, const char *fmt, ...) \
6146 struct va_format vaf; \
6149 va_start(args, fmt); \
6154 r = __netdev_printk(level, dev, &vaf); \
6159 EXPORT_SYMBOL(func);
6161 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6162 define_netdev_printk_level(netdev_alert, KERN_ALERT);
6163 define_netdev_printk_level(netdev_crit, KERN_CRIT);
6164 define_netdev_printk_level(netdev_err, KERN_ERR);
6165 define_netdev_printk_level(netdev_warn, KERN_WARNING);
6166 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6167 define_netdev_printk_level(netdev_info, KERN_INFO);
6169 static void __net_exit netdev_exit(struct net *net)
6171 kfree(net->dev_name_head);
6172 kfree(net->dev_index_head);
6175 static struct pernet_operations __net_initdata netdev_net_ops = {
6176 .init = netdev_init,
6177 .exit = netdev_exit,
6180 static void __net_exit default_device_exit(struct net *net)
6182 struct net_device *dev, *aux;
6184 * Push all migratable network devices back to the
6185 * initial network namespace
6188 for_each_netdev_safe(net, dev, aux) {
6190 char fb_name[IFNAMSIZ];
6192 /* Ignore unmoveable devices (i.e. loopback) */
6193 if (dev->features & NETIF_F_NETNS_LOCAL)
6196 /* Leave virtual devices for the generic cleanup */
6197 if (dev->rtnl_link_ops)
6200 /* Push remaing network devices to init_net */
6201 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6202 err = dev_change_net_namespace(dev, &init_net, fb_name);
6204 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
6205 __func__, dev->name, err);
6212 static void __net_exit default_device_exit_batch(struct list_head *net_list)
6214 /* At exit all network devices most be removed from a network
6215 * namespace. Do this in the reverse order of registration.
6216 * Do this across as many network namespaces as possible to
6217 * improve batching efficiency.
6219 struct net_device *dev;
6221 LIST_HEAD(dev_kill_list);
6224 list_for_each_entry(net, net_list, exit_list) {
6225 for_each_netdev_reverse(net, dev) {
6226 if (dev->rtnl_link_ops)
6227 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6229 unregister_netdevice_queue(dev, &dev_kill_list);
6232 unregister_netdevice_many(&dev_kill_list);
6233 list_del(&dev_kill_list);
6237 static struct pernet_operations __net_initdata default_device_ops = {
6238 .exit = default_device_exit,
6239 .exit_batch = default_device_exit_batch,
6243 * Initialize the DEV module. At boot time this walks the device list and
6244 * unhooks any devices that fail to initialise (normally hardware not
6245 * present) and leaves us with a valid list of present and active devices.
6250 * This is called single threaded during boot, so no need
6251 * to take the rtnl semaphore.
6253 static int __init net_dev_init(void)
6255 int i, rc = -ENOMEM;
6257 BUG_ON(!dev_boot_phase);
6259 if (dev_proc_init())
6262 if (netdev_kobject_init())
6265 INIT_LIST_HEAD(&ptype_all);
6266 for (i = 0; i < PTYPE_HASH_SIZE; i++)
6267 INIT_LIST_HEAD(&ptype_base[i]);
6269 if (register_pernet_subsys(&netdev_net_ops))
6273 * Initialise the packet receive queues.
6276 for_each_possible_cpu(i) {
6277 struct softnet_data *sd = &per_cpu(softnet_data, i);
6279 memset(sd, 0, sizeof(*sd));
6280 skb_queue_head_init(&sd->input_pkt_queue);
6281 skb_queue_head_init(&sd->process_queue);
6282 sd->completion_queue = NULL;
6283 INIT_LIST_HEAD(&sd->poll_list);
6284 sd->output_queue = NULL;
6285 sd->output_queue_tailp = &sd->output_queue;
6287 sd->csd.func = rps_trigger_softirq;
6293 sd->backlog.poll = process_backlog;
6294 sd->backlog.weight = weight_p;
6295 sd->backlog.gro_list = NULL;
6296 sd->backlog.gro_count = 0;
6301 /* The loopback device is special if any other network devices
6302 * is present in a network namespace the loopback device must
6303 * be present. Since we now dynamically allocate and free the
6304 * loopback device ensure this invariant is maintained by
6305 * keeping the loopback device as the first device on the
6306 * list of network devices. Ensuring the loopback devices
6307 * is the first device that appears and the last network device
6310 if (register_pernet_device(&loopback_net_ops))
6313 if (register_pernet_device(&default_device_ops))
6316 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6317 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
6319 hotcpu_notifier(dev_cpu_callback, 0);
6327 subsys_initcall(net_dev_init);
6329 static int __init initialize_hashrnd(void)
6331 get_random_bytes(&hashrnd, sizeof(hashrnd));
6335 late_initcall_sync(initialize_hashrnd);