blob: 89e33a5d4d932c54af74a08cb080809affc7dd67 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
100#include <linux/proc_fs.h>
101#include <linux/seq_file.h>
102#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/dst.h>
104#include <net/pkt_sched.h>
105#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000106#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/highmem.h>
108#include <linux/init.h>
109#include <linux/kmod.h>
110#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700114#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500117#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700118#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700119#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700120#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700121#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700122#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700123#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700124#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ipv6.h>
126#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700127#include <linux/jhash.h>
128#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700129#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900130#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900131#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000132#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700133#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000134#include <linux/cpu_rmap.h>
Richard Cochran4dc360c2011-10-19 17:00:35 -0400135#include <linux/net_tstamp.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100136#include <linux/static_key.h>
Eric Dumazet4504b862011-11-28 05:23:23 +0000137#include <net/flow_keys.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700139#include "net-sysfs.h"
140
Herbert Xud565b0a2008-12-15 23:38:52 -0800141/* Instead of increasing this, you should create a hash table. */
142#define MAX_GRO_SKBS 8
143
Herbert Xu5d38a072009-01-04 16:13:40 -0800144/* This should be increased if a protocol with a bigger head is added. */
145#define GRO_MAX_HEAD (MAX_HEADER + 128)
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147/*
148 * The list of packet types we will receive (as opposed to discard)
149 * and the routines to invoke.
150 *
151 * Why 16. Because with 16 the only overlap we get on a hash of the
152 * low nibble of the protocol value is RARP/SNAP/X.25.
153 *
154 * NOTE: That is no longer true with the addition of VLAN tags. Not
155 * sure which should go first, but I bet it won't make much
156 * difference if we are running VLANs. The good news is that
157 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700158 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 * --BLG
160 *
161 * 0800 IP
162 * 8100 802.1Q VLAN
163 * 0001 802.3
164 * 0002 AX.25
165 * 0004 802.2
166 * 8035 RARP
167 * 0005 SNAP
168 * 0805 X.25
169 * 0806 ARP
170 * 8137 IPX
171 * 0009 Localtalk
172 * 86DD IPv6
173 */
174
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800175#define PTYPE_HASH_SIZE (16)
176#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
177
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800179static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700180static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700183 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 * semaphore.
185 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800186 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 *
188 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700189 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 * actual updates. This allows pure readers to access the list even
191 * while a writer is preparing to update it.
192 *
193 * To put it another way, dev_base_lock is held for writing only to
194 * protect against pure readers; the rtnl semaphore provides the
195 * protection against other writers.
196 *
197 * See, for example usages, register_netdevice() and
198 * unregister_netdevice(), which must be called with the rtnl
199 * semaphore held.
200 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202EXPORT_SYMBOL(dev_base_lock);
203
Thomas Graf4e985ad2011-06-21 03:11:20 +0000204static inline void dev_base_seq_inc(struct net *net)
205{
206 while (++net->dev_base_seq == 0);
207}
208
Eric W. Biederman881d9662007-09-17 11:56:21 -0700209static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
Eric Dumazet95c96172012-04-15 05:58:06 +0000211 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
212
stephen hemminger08e98972009-11-10 07:20:34 +0000213 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214}
215
Eric W. Biederman881d9662007-09-17 11:56:21 -0700216static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700218 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219}
220
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000221static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000222{
223#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000224 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000225#endif
226}
227
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000229{
230#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000231 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000232#endif
233}
234
Eric W. Biedermance286d32007-09-12 13:53:49 +0200235/* Device list insertion */
236static int list_netdevice(struct net_device *dev)
237{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900238 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239
240 ASSERT_RTNL();
241
242 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800243 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000244 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000245 hlist_add_head_rcu(&dev->index_hlist,
246 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200247 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000248
249 dev_base_seq_inc(net);
250
Eric W. Biedermance286d32007-09-12 13:53:49 +0200251 return 0;
252}
253
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000254/* Device list removal
255 * caller must respect a RCU grace period before freeing/reusing dev
256 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200257static void unlist_netdevice(struct net_device *dev)
258{
259 ASSERT_RTNL();
260
261 /* Unlink dev from the device chain */
262 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800263 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000264 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000265 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200266 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000267
268 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200269}
270
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271/*
272 * Our notifier list
273 */
274
Alan Sternf07d5b92006-05-09 15:23:03 -0700275static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276
277/*
278 * Device drivers call our routines to queue packets here. We empty the
279 * queue in the local softnet handler.
280 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700281
Eric Dumazet9958da02010-04-17 04:17:02 +0000282DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700283EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
David S. Millercf508b12008-07-22 14:16:42 -0700285#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700286/*
David S. Millerc773e842008-07-08 23:13:53 -0700287 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700288 * according to dev->type
289 */
290static const unsigned short netdev_lock_type[] =
291 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
292 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
293 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
294 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
295 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
296 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
297 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
298 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
299 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
300 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
301 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
302 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400303 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
304 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
305 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700306
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700307static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700308 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
309 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
310 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
311 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
312 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
313 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
314 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
315 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
316 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
317 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
318 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
319 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400320 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
321 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
322 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700323
324static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700325static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700326
327static inline unsigned short netdev_lock_pos(unsigned short dev_type)
328{
329 int i;
330
331 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
332 if (netdev_lock_type[i] == dev_type)
333 return i;
334 /* the last key is used by default */
335 return ARRAY_SIZE(netdev_lock_type) - 1;
336}
337
David S. Millercf508b12008-07-22 14:16:42 -0700338static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
339 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700340{
341 int i;
342
343 i = netdev_lock_pos(dev_type);
344 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
345 netdev_lock_name[i]);
346}
David S. Millercf508b12008-07-22 14:16:42 -0700347
348static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
349{
350 int i;
351
352 i = netdev_lock_pos(dev->type);
353 lockdep_set_class_and_name(&dev->addr_list_lock,
354 &netdev_addr_lock_key[i],
355 netdev_lock_name[i]);
356}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700357#else
David S. Millercf508b12008-07-22 14:16:42 -0700358static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
359 unsigned short dev_type)
360{
361}
362static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700363{
364}
365#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366
367/*******************************************************************************
368
369 Protocol management and registration routines
370
371*******************************************************************************/
372
373/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 * Add a protocol ID to the list. Now that the input handler is
375 * smarter we can dispense with all the messy stuff that used to be
376 * here.
377 *
378 * BEWARE!!! Protocol handlers, mangling input packets,
379 * MUST BE last in hash buckets and checking protocol handlers
380 * MUST start from promiscuous ptype_all chain in net_bh.
381 * It is true now, do not change it.
382 * Explanation follows: if protocol handler, mangling packet, will
383 * be the first on list, it is not able to sense, that packet
384 * is cloned and should be copied-on-write, so that it will
385 * change it and subsequent readers will get broken packet.
386 * --ANK (980803)
387 */
388
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000389static inline struct list_head *ptype_head(const struct packet_type *pt)
390{
391 if (pt->type == htons(ETH_P_ALL))
392 return &ptype_all;
393 else
394 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
395}
396
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397/**
398 * dev_add_pack - add packet handler
399 * @pt: packet type declaration
400 *
401 * Add a protocol handler to the networking stack. The passed &packet_type
402 * is linked into kernel lists and may not be freed until it has been
403 * removed from the kernel lists.
404 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900405 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 * guarantee all CPU's that are in middle of receiving packets
407 * will see the new packet type (until the next received packet).
408 */
409
410void dev_add_pack(struct packet_type *pt)
411{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000412 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000414 spin_lock(&ptype_lock);
415 list_add_rcu(&pt->list, head);
416 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700418EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420/**
421 * __dev_remove_pack - remove packet handler
422 * @pt: packet type declaration
423 *
424 * Remove a protocol handler that was previously added to the kernel
425 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
426 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900427 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 *
429 * The packet type might still be in use by receivers
430 * and must not be freed until after all the CPU's have gone
431 * through a quiescent state.
432 */
433void __dev_remove_pack(struct packet_type *pt)
434{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000435 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 struct packet_type *pt1;
437
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000438 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
440 list_for_each_entry(pt1, head, list) {
441 if (pt == pt1) {
442 list_del_rcu(&pt->list);
443 goto out;
444 }
445 }
446
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000447 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000449 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700451EXPORT_SYMBOL(__dev_remove_pack);
452
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453/**
454 * dev_remove_pack - remove packet handler
455 * @pt: packet type declaration
456 *
457 * Remove a protocol handler that was previously added to the kernel
458 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
459 * from the kernel lists and can be freed or reused once this function
460 * returns.
461 *
462 * This call sleeps to guarantee that no CPU is looking at the packet
463 * type after return.
464 */
465void dev_remove_pack(struct packet_type *pt)
466{
467 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900468
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 synchronize_net();
470}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700471EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472
473/******************************************************************************
474
475 Device Boot-time Settings Routines
476
477*******************************************************************************/
478
479/* Boot time configuration table */
480static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
481
482/**
483 * netdev_boot_setup_add - add new setup entry
484 * @name: name of the device
485 * @map: configured settings for the device
486 *
487 * Adds new setup entry to the dev_boot_setup list. The function
488 * returns 0 on error and 1 on success. This is a generic routine to
489 * all netdevices.
490 */
491static int netdev_boot_setup_add(char *name, struct ifmap *map)
492{
493 struct netdev_boot_setup *s;
494 int i;
495
496 s = dev_boot_setup;
497 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
498 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
499 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700500 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501 memcpy(&s[i].map, map, sizeof(s[i].map));
502 break;
503 }
504 }
505
506 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
507}
508
509/**
510 * netdev_boot_setup_check - check boot time settings
511 * @dev: the netdevice
512 *
513 * Check boot time settings for the device.
514 * The found settings are set for the device to be used
515 * later in the device probing.
516 * Returns 0 if no settings found, 1 if they are.
517 */
518int netdev_boot_setup_check(struct net_device *dev)
519{
520 struct netdev_boot_setup *s = dev_boot_setup;
521 int i;
522
523 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
524 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700525 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 dev->irq = s[i].map.irq;
527 dev->base_addr = s[i].map.base_addr;
528 dev->mem_start = s[i].map.mem_start;
529 dev->mem_end = s[i].map.mem_end;
530 return 1;
531 }
532 }
533 return 0;
534}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700535EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536
537
538/**
539 * netdev_boot_base - get address from boot time settings
540 * @prefix: prefix for network device
541 * @unit: id for network device
542 *
543 * Check boot time settings for the base address of device.
544 * The found settings are set for the device to be used
545 * later in the device probing.
546 * Returns 0 if no settings found.
547 */
548unsigned long netdev_boot_base(const char *prefix, int unit)
549{
550 const struct netdev_boot_setup *s = dev_boot_setup;
551 char name[IFNAMSIZ];
552 int i;
553
554 sprintf(name, "%s%d", prefix, unit);
555
556 /*
557 * If device already registered then return base of 1
558 * to indicate not to probe for this interface
559 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700560 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 return 1;
562
563 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
564 if (!strcmp(name, s[i].name))
565 return s[i].map.base_addr;
566 return 0;
567}
568
569/*
570 * Saves at boot time configured settings for any netdevice.
571 */
572int __init netdev_boot_setup(char *str)
573{
574 int ints[5];
575 struct ifmap map;
576
577 str = get_options(str, ARRAY_SIZE(ints), ints);
578 if (!str || !*str)
579 return 0;
580
581 /* Save settings */
582 memset(&map, 0, sizeof(map));
583 if (ints[0] > 0)
584 map.irq = ints[1];
585 if (ints[0] > 1)
586 map.base_addr = ints[2];
587 if (ints[0] > 2)
588 map.mem_start = ints[3];
589 if (ints[0] > 3)
590 map.mem_end = ints[4];
591
592 /* Add new entry to the list */
593 return netdev_boot_setup_add(str, &map);
594}
595
596__setup("netdev=", netdev_boot_setup);
597
598/*******************************************************************************
599
600 Device Interface Subroutines
601
602*******************************************************************************/
603
604/**
605 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700606 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 * @name: name to find
608 *
609 * Find an interface by name. Must be called under RTNL semaphore
610 * or @dev_base_lock. If the name is found a pointer to the device
611 * is returned. If the name is not found then %NULL is returned. The
612 * reference counters are not incremented so the caller must be
613 * careful with locks.
614 */
615
Eric W. Biederman881d9662007-09-17 11:56:21 -0700616struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617{
618 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700619 struct net_device *dev;
620 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700622 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 if (!strncmp(dev->name, name, IFNAMSIZ))
624 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700625
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 return NULL;
627}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700628EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629
630/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000631 * dev_get_by_name_rcu - find a device by its name
632 * @net: the applicable net namespace
633 * @name: name to find
634 *
635 * Find an interface by name.
636 * If the name is found a pointer to the device is returned.
637 * If the name is not found then %NULL is returned.
638 * The reference counters are not incremented so the caller must be
639 * careful with locks. The caller must hold RCU lock.
640 */
641
642struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
643{
644 struct hlist_node *p;
645 struct net_device *dev;
646 struct hlist_head *head = dev_name_hash(net, name);
647
648 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
649 if (!strncmp(dev->name, name, IFNAMSIZ))
650 return dev;
651
652 return NULL;
653}
654EXPORT_SYMBOL(dev_get_by_name_rcu);
655
656/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700658 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 * @name: name to find
660 *
661 * Find an interface by name. This can be called from any
662 * context and does its own locking. The returned handle has
663 * the usage count incremented and the caller must use dev_put() to
664 * release it when it is no longer needed. %NULL is returned if no
665 * matching device is found.
666 */
667
Eric W. Biederman881d9662007-09-17 11:56:21 -0700668struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669{
670 struct net_device *dev;
671
Eric Dumazet72c95282009-10-30 07:11:27 +0000672 rcu_read_lock();
673 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 if (dev)
675 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000676 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 return dev;
678}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700679EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680
681/**
682 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700683 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 * @ifindex: index of device
685 *
686 * Search for an interface by index. Returns %NULL if the device
687 * is not found or a pointer to the device. The device has not
688 * had its reference counter increased so the caller must be careful
689 * about locking. The caller must hold either the RTNL semaphore
690 * or @dev_base_lock.
691 */
692
Eric W. Biederman881d9662007-09-17 11:56:21 -0700693struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694{
695 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700696 struct net_device *dev;
697 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700699 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700 if (dev->ifindex == ifindex)
701 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700702
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 return NULL;
704}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700705EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000707/**
708 * dev_get_by_index_rcu - find a device by its ifindex
709 * @net: the applicable net namespace
710 * @ifindex: index of device
711 *
712 * Search for an interface by index. Returns %NULL if the device
713 * is not found or a pointer to the device. The device has not
714 * had its reference counter increased so the caller must be careful
715 * about locking. The caller must hold RCU lock.
716 */
717
718struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
719{
720 struct hlist_node *p;
721 struct net_device *dev;
722 struct hlist_head *head = dev_index_hash(net, ifindex);
723
724 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
725 if (dev->ifindex == ifindex)
726 return dev;
727
728 return NULL;
729}
730EXPORT_SYMBOL(dev_get_by_index_rcu);
731
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
733/**
734 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700735 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 * @ifindex: index of device
737 *
738 * Search for an interface by index. Returns NULL if the device
739 * is not found or a pointer to the device. The device returned has
740 * had a reference added and the pointer is safe until the user calls
741 * dev_put to indicate they have finished with it.
742 */
743
Eric W. Biederman881d9662007-09-17 11:56:21 -0700744struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745{
746 struct net_device *dev;
747
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000748 rcu_read_lock();
749 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 if (dev)
751 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000752 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 return dev;
754}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700755EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756
757/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000758 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700759 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 * @type: media type of device
761 * @ha: hardware address
762 *
763 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800764 * is not found or a pointer to the device.
765 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000766 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 * and the caller must therefore be careful about locking
768 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 */
770
Eric Dumazet941666c2010-12-05 01:23:53 +0000771struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
772 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773{
774 struct net_device *dev;
775
Eric Dumazet941666c2010-12-05 01:23:53 +0000776 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 if (dev->type == type &&
778 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700779 return dev;
780
781 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782}
Eric Dumazet941666c2010-12-05 01:23:53 +0000783EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300784
Eric W. Biederman881d9662007-09-17 11:56:21 -0700785struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700786{
787 struct net_device *dev;
788
789 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700790 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700791 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700792 return dev;
793
794 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700795}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700796EXPORT_SYMBOL(__dev_getfirstbyhwtype);
797
Eric W. Biederman881d9662007-09-17 11:56:21 -0700798struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000800 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000802 rcu_read_lock();
803 for_each_netdev_rcu(net, dev)
804 if (dev->type == type) {
805 dev_hold(dev);
806 ret = dev;
807 break;
808 }
809 rcu_read_unlock();
810 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812EXPORT_SYMBOL(dev_getfirstbyhwtype);
813
814/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000815 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700816 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 * @if_flags: IFF_* values
818 * @mask: bitmask of bits in if_flags to check
819 *
820 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000821 * is not found or a pointer to the device. Must be called inside
822 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 */
824
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000825struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700826 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700828 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
Pavel Emelianov7562f872007-05-03 15:13:45 -0700830 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800831 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700833 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 break;
835 }
836 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700837 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000839EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840
841/**
842 * dev_valid_name - check if name is okay for network device
843 * @name: name string
844 *
845 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700846 * to allow sysfs to work. We also disallow any kind of
847 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 */
David S. Miller95f050b2012-03-06 16:12:15 -0500849bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700851 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500852 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700853 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500854 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700855 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500856 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700857
858 while (*name) {
859 if (*name == '/' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500860 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700861 name++;
862 }
David S. Miller95f050b2012-03-06 16:12:15 -0500863 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700865EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866
867/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200868 * __dev_alloc_name - allocate a name for a device
869 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200871 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872 *
873 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700874 * id. It scans list of devices to build up a free map, then chooses
875 * the first empty slot. The caller must hold the dev_base or rtnl lock
876 * while allocating the name and adding the device in order to avoid
877 * duplicates.
878 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
879 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 */
881
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200882static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883{
884 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 const char *p;
886 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700887 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 struct net_device *d;
889
890 p = strnchr(name, IFNAMSIZ-1, '%');
891 if (p) {
892 /*
893 * Verify the string as this thing may have come from
894 * the user. There must be either one "%d" and no other "%"
895 * characters.
896 */
897 if (p[1] != 'd' || strchr(p + 2, '%'))
898 return -EINVAL;
899
900 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700901 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 if (!inuse)
903 return -ENOMEM;
904
Eric W. Biederman881d9662007-09-17 11:56:21 -0700905 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 if (!sscanf(d->name, name, &i))
907 continue;
908 if (i < 0 || i >= max_netdevices)
909 continue;
910
911 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200912 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 if (!strncmp(buf, d->name, IFNAMSIZ))
914 set_bit(i, inuse);
915 }
916
917 i = find_first_zero_bit(inuse, max_netdevices);
918 free_page((unsigned long) inuse);
919 }
920
Octavian Purdilad9031022009-11-18 02:36:59 +0000921 if (buf != name)
922 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200923 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925
926 /* It is possible to run out of possible slots
927 * when the name is long and there isn't enough space left
928 * for the digits, or if all bits are used.
929 */
930 return -ENFILE;
931}
932
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200933/**
934 * dev_alloc_name - allocate a name for a device
935 * @dev: device
936 * @name: name format string
937 *
938 * Passed a format string - eg "lt%d" it will try and find a suitable
939 * id. It scans list of devices to build up a free map, then chooses
940 * the first empty slot. The caller must hold the dev_base or rtnl lock
941 * while allocating the name and adding the device in order to avoid
942 * duplicates.
943 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
944 * Returns the number of the unit assigned or a negative errno code.
945 */
946
947int dev_alloc_name(struct net_device *dev, const char *name)
948{
949 char buf[IFNAMSIZ];
950 struct net *net;
951 int ret;
952
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900953 BUG_ON(!dev_net(dev));
954 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200955 ret = __dev_alloc_name(net, name, buf);
956 if (ret >= 0)
957 strlcpy(dev->name, buf, IFNAMSIZ);
958 return ret;
959}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700960EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200961
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000962static int dev_get_valid_name(struct net_device *dev, const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +0000963{
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000964 struct net *net;
965
966 BUG_ON(!dev_net(dev));
967 net = dev_net(dev);
968
Octavian Purdilad9031022009-11-18 02:36:59 +0000969 if (!dev_valid_name(name))
970 return -EINVAL;
971
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000972 if (strchr(name, '%'))
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000973 return dev_alloc_name(dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +0000974 else if (__dev_get_by_name(net, name))
975 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000976 else if (dev->name != name)
977 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +0000978
979 return 0;
980}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981
982/**
983 * dev_change_name - change name of a device
984 * @dev: device
985 * @newname: name (or format string) must be at least IFNAMSIZ
986 *
987 * Change name of a device, can pass format strings "eth%d".
988 * for wildcarding.
989 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700990int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991{
Herbert Xufcc5a032007-07-30 17:03:38 -0700992 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700994 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700995 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996
997 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900998 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001000 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 if (dev->flags & IFF_UP)
1002 return -EBUSY;
1003
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001004 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
1005 return 0;
1006
Herbert Xufcc5a032007-07-30 17:03:38 -07001007 memcpy(oldname, dev->name, IFNAMSIZ);
1008
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001009 err = dev_get_valid_name(dev, newname);
Octavian Purdilad9031022009-11-18 02:36:59 +00001010 if (err < 0)
1011 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
Herbert Xufcc5a032007-07-30 17:03:38 -07001013rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001014 ret = device_rename(&dev->dev, dev->name);
1015 if (ret) {
1016 memcpy(dev->name, oldname, IFNAMSIZ);
1017 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001018 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001019
1020 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001021 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001022 write_unlock_bh(&dev_base_lock);
1023
1024 synchronize_rcu();
1025
1026 write_lock_bh(&dev_base_lock);
1027 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001028 write_unlock_bh(&dev_base_lock);
1029
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001030 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001031 ret = notifier_to_errno(ret);
1032
1033 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001034 /* err >= 0 after dev_alloc_name() or stores the first errno */
1035 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001036 err = ret;
1037 memcpy(dev->name, oldname, IFNAMSIZ);
1038 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001039 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001040 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001041 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001042 }
1043 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044
1045 return err;
1046}
1047
1048/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001049 * dev_set_alias - change ifalias of a device
1050 * @dev: device
1051 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001052 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001053 *
1054 * Set ifalias for a device,
1055 */
1056int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1057{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001058 char *new_ifalias;
1059
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001060 ASSERT_RTNL();
1061
1062 if (len >= IFALIASZ)
1063 return -EINVAL;
1064
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001065 if (!len) {
1066 if (dev->ifalias) {
1067 kfree(dev->ifalias);
1068 dev->ifalias = NULL;
1069 }
1070 return 0;
1071 }
1072
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001073 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1074 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001075 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001076 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001077
1078 strlcpy(dev->ifalias, alias, len+1);
1079 return len;
1080}
1081
1082
1083/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001084 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001085 * @dev: device to cause notification
1086 *
1087 * Called to indicate a device has changed features.
1088 */
1089void netdev_features_change(struct net_device *dev)
1090{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001091 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001092}
1093EXPORT_SYMBOL(netdev_features_change);
1094
1095/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096 * netdev_state_change - device changes state
1097 * @dev: device to cause notification
1098 *
1099 * Called to indicate a device has changed state. This function calls
1100 * the notifier chains for netdev_chain and sends a NEWLINK message
1101 * to the routing socket.
1102 */
1103void netdev_state_change(struct net_device *dev)
1104{
1105 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001106 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1108 }
1109}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001110EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001112int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001113{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001114 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001115}
1116EXPORT_SYMBOL(netdev_bonding_change);
1117
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118/**
1119 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001120 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 * @name: name of interface
1122 *
1123 * If a network interface is not present and the process has suitable
1124 * privileges this function loads the module. If module loading is not
1125 * available in this kernel then it becomes a nop.
1126 */
1127
Eric W. Biederman881d9662007-09-17 11:56:21 -07001128void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001130 struct net_device *dev;
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001131 int no_module;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132
Eric Dumazet72c95282009-10-30 07:11:27 +00001133 rcu_read_lock();
1134 dev = dev_get_by_name_rcu(net, name);
1135 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001137 no_module = !dev;
1138 if (no_module && capable(CAP_NET_ADMIN))
1139 no_module = request_module("netdev-%s", name);
1140 if (no_module && capable(CAP_SYS_MODULE)) {
1141 if (!request_module("%s", name))
Vinson Lee7cecb522012-06-27 14:32:07 +00001142 pr_warn("Loading kernel module for a network device with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s instead.\n",
1143 name);
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001144 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001146EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147
Patrick McHardybd380812010-02-26 06:34:53 +00001148static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001150 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001151 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001153 ASSERT_RTNL();
1154
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 if (!netif_device_present(dev))
1156 return -ENODEV;
1157
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001158 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1159 ret = notifier_to_errno(ret);
1160 if (ret)
1161 return ret;
1162
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001164
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001165 if (ops->ndo_validate_addr)
1166 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001167
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001168 if (!ret && ops->ndo_open)
1169 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170
Jeff Garzikbada3392007-10-23 20:19:37 -07001171 if (ret)
1172 clear_bit(__LINK_STATE_START, &dev->state);
1173 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001175 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001176 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001178 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001180
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 return ret;
1182}
Patrick McHardybd380812010-02-26 06:34:53 +00001183
1184/**
1185 * dev_open - prepare an interface for use.
1186 * @dev: device to open
1187 *
1188 * Takes a device from down to up state. The device's private open
1189 * function is invoked and then the multicast lists are loaded. Finally
1190 * the device is moved into the up state and a %NETDEV_UP message is
1191 * sent to the netdev notifier chain.
1192 *
1193 * Calling this function on an active interface is a nop. On a failure
1194 * a negative errno code is returned.
1195 */
1196int dev_open(struct net_device *dev)
1197{
1198 int ret;
1199
Patrick McHardybd380812010-02-26 06:34:53 +00001200 if (dev->flags & IFF_UP)
1201 return 0;
1202
Patrick McHardybd380812010-02-26 06:34:53 +00001203 ret = __dev_open(dev);
1204 if (ret < 0)
1205 return ret;
1206
Patrick McHardybd380812010-02-26 06:34:53 +00001207 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1208 call_netdevice_notifiers(NETDEV_UP, dev);
1209
1210 return ret;
1211}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001212EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213
Octavian Purdila44345722010-12-13 12:44:07 +00001214static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215{
Octavian Purdila44345722010-12-13 12:44:07 +00001216 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001217
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001218 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001219 might_sleep();
1220
Octavian Purdila44345722010-12-13 12:44:07 +00001221 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001222 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223
Octavian Purdila44345722010-12-13 12:44:07 +00001224 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225
Octavian Purdila44345722010-12-13 12:44:07 +00001226 /* Synchronize to scheduled poll. We cannot touch poll list, it
1227 * can be even on different cpu. So just clear netif_running().
1228 *
1229 * dev->stop() will invoke napi_disable() on all of it's
1230 * napi_struct instances on this device.
1231 */
1232 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1233 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234
Octavian Purdila44345722010-12-13 12:44:07 +00001235 dev_deactivate_many(head);
1236
1237 list_for_each_entry(dev, head, unreg_list) {
1238 const struct net_device_ops *ops = dev->netdev_ops;
1239
1240 /*
1241 * Call the device specific close. This cannot fail.
1242 * Only if device is UP
1243 *
1244 * We allow it to be called even after a DETACH hot-plug
1245 * event.
1246 */
1247 if (ops->ndo_stop)
1248 ops->ndo_stop(dev);
1249
Octavian Purdila44345722010-12-13 12:44:07 +00001250 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001251 net_dmaengine_put();
1252 }
1253
1254 return 0;
1255}
1256
1257static int __dev_close(struct net_device *dev)
1258{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001259 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001260 LIST_HEAD(single);
1261
1262 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001263 retval = __dev_close_many(&single);
1264 list_del(&single);
1265 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001266}
1267
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001268static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001269{
1270 struct net_device *dev, *tmp;
1271 LIST_HEAD(tmp_list);
1272
1273 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1274 if (!(dev->flags & IFF_UP))
1275 list_move(&dev->unreg_list, &tmp_list);
1276
1277 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001278
Octavian Purdila44345722010-12-13 12:44:07 +00001279 list_for_each_entry(dev, head, unreg_list) {
1280 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1281 call_netdevice_notifiers(NETDEV_DOWN, dev);
1282 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
Octavian Purdila44345722010-12-13 12:44:07 +00001284 /* rollback_registered_many needs the complete original list */
1285 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 return 0;
1287}
Patrick McHardybd380812010-02-26 06:34:53 +00001288
1289/**
1290 * dev_close - shutdown an interface.
1291 * @dev: device to shutdown
1292 *
1293 * This function moves an active device into down state. A
1294 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1295 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1296 * chain.
1297 */
1298int dev_close(struct net_device *dev)
1299{
Eric Dumazete14a5992011-05-10 12:26:06 -07001300 if (dev->flags & IFF_UP) {
1301 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001302
Eric Dumazete14a5992011-05-10 12:26:06 -07001303 list_add(&dev->unreg_list, &single);
1304 dev_close_many(&single);
1305 list_del(&single);
1306 }
Patrick McHardybd380812010-02-26 06:34:53 +00001307 return 0;
1308}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001309EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310
1311
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001312/**
1313 * dev_disable_lro - disable Large Receive Offload on a device
1314 * @dev: device
1315 *
1316 * Disable Large Receive Offload (LRO) on a net device. Must be
1317 * called under RTNL. This is needed if received packets may be
1318 * forwarded to another interface.
1319 */
1320void dev_disable_lro(struct net_device *dev)
1321{
Neil Hormanf11970e2011-05-24 08:31:09 +00001322 /*
1323 * If we're trying to disable lro on a vlan device
1324 * use the underlying physical device instead
1325 */
1326 if (is_vlan_dev(dev))
1327 dev = vlan_dev_real_dev(dev);
1328
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001329 dev->wanted_features &= ~NETIF_F_LRO;
1330 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001331
Michał Mirosław22d59692011-04-21 12:42:15 +00001332 if (unlikely(dev->features & NETIF_F_LRO))
1333 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001334}
1335EXPORT_SYMBOL(dev_disable_lro);
1336
1337
Eric W. Biederman881d9662007-09-17 11:56:21 -07001338static int dev_boot_phase = 1;
1339
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340/**
1341 * register_netdevice_notifier - register a network notifier block
1342 * @nb: notifier
1343 *
1344 * Register a notifier to be called when network device events occur.
1345 * The notifier passed is linked into the kernel structures and must
1346 * not be reused until it has been unregistered. A negative errno code
1347 * is returned on a failure.
1348 *
1349 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001350 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351 * view of the network device list.
1352 */
1353
1354int register_netdevice_notifier(struct notifier_block *nb)
1355{
1356 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001357 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001358 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 int err;
1360
1361 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001362 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001363 if (err)
1364 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001365 if (dev_boot_phase)
1366 goto unlock;
1367 for_each_net(net) {
1368 for_each_netdev(net, dev) {
1369 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1370 err = notifier_to_errno(err);
1371 if (err)
1372 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001373
Eric W. Biederman881d9662007-09-17 11:56:21 -07001374 if (!(dev->flags & IFF_UP))
1375 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001376
Eric W. Biederman881d9662007-09-17 11:56:21 -07001377 nb->notifier_call(nb, NETDEV_UP, dev);
1378 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001380
1381unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 rtnl_unlock();
1383 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001384
1385rollback:
1386 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001387 for_each_net(net) {
1388 for_each_netdev(net, dev) {
1389 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001390 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001391
Eric W. Biederman881d9662007-09-17 11:56:21 -07001392 if (dev->flags & IFF_UP) {
1393 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1394 nb->notifier_call(nb, NETDEV_DOWN, dev);
1395 }
1396 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001397 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001398 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001399 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001400
RongQing.Li8f891482011-11-30 23:43:07 -05001401outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001402 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001403 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001405EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406
1407/**
1408 * unregister_netdevice_notifier - unregister a network notifier block
1409 * @nb: notifier
1410 *
1411 * Unregister a notifier previously registered by
1412 * register_netdevice_notifier(). The notifier is unlinked into the
1413 * kernel structures and may then be reused. A negative errno code
1414 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001415 *
1416 * After unregistering unregister and down device events are synthesized
1417 * for all devices on the device list to the removed notifier to remove
1418 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 */
1420
1421int unregister_netdevice_notifier(struct notifier_block *nb)
1422{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001423 struct net_device *dev;
1424 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001425 int err;
1426
1427 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001428 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001429 if (err)
1430 goto unlock;
1431
1432 for_each_net(net) {
1433 for_each_netdev(net, dev) {
1434 if (dev->flags & IFF_UP) {
1435 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1436 nb->notifier_call(nb, NETDEV_DOWN, dev);
1437 }
1438 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1439 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1440 }
1441 }
1442unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001443 rtnl_unlock();
1444 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001446EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447
1448/**
1449 * call_netdevice_notifiers - call all network notifier blocks
1450 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001451 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452 *
1453 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001454 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455 */
1456
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001457int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458{
Jiri Pirkoab930472010-04-20 01:45:37 -07001459 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001460 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001462EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463
Ingo Molnarc5905af2012-02-24 08:31:31 +01001464static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001465#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001466/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001467 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001468 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001469 */
1470static atomic_t netstamp_needed_deferred;
1471#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472
1473void net_enable_timestamp(void)
1474{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001475#ifdef HAVE_JUMP_LABEL
1476 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1477
1478 if (deferred) {
1479 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001480 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001481 return;
1482 }
1483#endif
1484 WARN_ON(in_interrupt());
Ingo Molnarc5905af2012-02-24 08:31:31 +01001485 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001487EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488
1489void net_disable_timestamp(void)
1490{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001491#ifdef HAVE_JUMP_LABEL
1492 if (in_interrupt()) {
1493 atomic_inc(&netstamp_needed_deferred);
1494 return;
1495 }
1496#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001497 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001499EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500
Eric Dumazet3b098e22010-05-15 23:57:10 -07001501static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502{
Eric Dumazet588f0332011-11-15 04:12:55 +00001503 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001504 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001505 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506}
1507
Eric Dumazet588f0332011-11-15 04:12:55 +00001508#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001509 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001510 if ((COND) && !(SKB)->tstamp.tv64) \
1511 __net_timestamp(SKB); \
1512 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001513
Richard Cochran4dc360c2011-10-19 17:00:35 -04001514static int net_hwtstamp_validate(struct ifreq *ifr)
1515{
1516 struct hwtstamp_config cfg;
1517 enum hwtstamp_tx_types tx_type;
1518 enum hwtstamp_rx_filters rx_filter;
1519 int tx_type_valid = 0;
1520 int rx_filter_valid = 0;
1521
1522 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1523 return -EFAULT;
1524
1525 if (cfg.flags) /* reserved for future extensions */
1526 return -EINVAL;
1527
1528 tx_type = cfg.tx_type;
1529 rx_filter = cfg.rx_filter;
1530
1531 switch (tx_type) {
1532 case HWTSTAMP_TX_OFF:
1533 case HWTSTAMP_TX_ON:
1534 case HWTSTAMP_TX_ONESTEP_SYNC:
1535 tx_type_valid = 1;
1536 break;
1537 }
1538
1539 switch (rx_filter) {
1540 case HWTSTAMP_FILTER_NONE:
1541 case HWTSTAMP_FILTER_ALL:
1542 case HWTSTAMP_FILTER_SOME:
1543 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1544 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1545 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1546 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1547 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1548 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1549 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1550 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1551 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1552 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1553 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1554 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1555 rx_filter_valid = 1;
1556 break;
1557 }
1558
1559 if (!tx_type_valid || !rx_filter_valid)
1560 return -ERANGE;
1561
1562 return 0;
1563}
1564
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001565static inline bool is_skb_forwardable(struct net_device *dev,
1566 struct sk_buff *skb)
1567{
1568 unsigned int len;
1569
1570 if (!(dev->flags & IFF_UP))
1571 return false;
1572
1573 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1574 if (skb->len <= len)
1575 return true;
1576
1577 /* if TSO is enabled, we don't care about the length as the packet
1578 * could be forwarded without being segmented before
1579 */
1580 if (skb_is_gso(skb))
1581 return true;
1582
1583 return false;
1584}
1585
Arnd Bergmann44540962009-11-26 06:07:08 +00001586/**
1587 * dev_forward_skb - loopback an skb to another netif
1588 *
1589 * @dev: destination network device
1590 * @skb: buffer to forward
1591 *
1592 * return values:
1593 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001594 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001595 *
1596 * dev_forward_skb can be used for injecting an skb from the
1597 * start_xmit function of one device into the receive queue
1598 * of another device.
1599 *
1600 * The receiving device may be in another namespace, so
1601 * we have to clear all information in the skb that could
1602 * impact namespace isolation.
1603 */
1604int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1605{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001606 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1607 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1608 atomic_long_inc(&dev->rx_dropped);
1609 kfree_skb(skb);
1610 return NET_RX_DROP;
1611 }
1612 }
1613
Arnd Bergmann44540962009-11-26 06:07:08 +00001614 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001615 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001616
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001617 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001618 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001619 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001620 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001621 }
Benjamin LaHaise3b9785c2012-03-27 15:55:44 +00001622 skb->skb_iif = 0;
David S. Miller59b99972012-05-10 23:03:34 -04001623 skb->dev = dev;
1624 skb_dst_drop(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001625 skb->tstamp.tv64 = 0;
1626 skb->pkt_type = PACKET_HOST;
1627 skb->protocol = eth_type_trans(skb, dev);
David S. Miller59b99972012-05-10 23:03:34 -04001628 skb->mark = 0;
1629 secpath_reset(skb);
1630 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001631 return netif_rx(skb);
1632}
1633EXPORT_SYMBOL_GPL(dev_forward_skb);
1634
Changli Gao71d9dec2010-12-15 19:57:25 +00001635static inline int deliver_skb(struct sk_buff *skb,
1636 struct packet_type *pt_prev,
1637 struct net_device *orig_dev)
1638{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001639 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1640 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001641 atomic_inc(&skb->users);
1642 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1643}
1644
Eric Leblondc0de08d2012-08-16 22:02:58 +00001645static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1646{
1647 if (ptype->af_packet_priv == NULL)
1648 return false;
1649
1650 if (ptype->id_match)
1651 return ptype->id_match(ptype, skb->sk);
1652 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1653 return true;
1654
1655 return false;
1656}
1657
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658/*
1659 * Support routine. Sends outgoing frames to any network
1660 * taps currently in use.
1661 */
1662
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001663static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664{
1665 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001666 struct sk_buff *skb2 = NULL;
1667 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001668
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 rcu_read_lock();
1670 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1671 /* Never send packets back to the socket
1672 * they originated from - MvS (miquels@drinkel.ow.org)
1673 */
1674 if ((ptype->dev == dev || !ptype->dev) &&
Eric Leblondc0de08d2012-08-16 22:02:58 +00001675 (!skb_loop_sk(ptype, skb))) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001676 if (pt_prev) {
1677 deliver_skb(skb2, pt_prev, skb->dev);
1678 pt_prev = ptype;
1679 continue;
1680 }
1681
1682 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 if (!skb2)
1684 break;
1685
Eric Dumazet70978182010-12-20 21:22:51 +00001686 net_timestamp_set(skb2);
1687
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688 /* skb->nh should be correctly
1689 set by sender, so that the second statement is
1690 just protection against buggy protocols.
1691 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001692 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001694 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001695 skb2->network_header > skb2->tail) {
Joe Perchese87cc472012-05-13 21:56:26 +00001696 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1697 ntohs(skb2->protocol),
1698 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001699 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 }
1701
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001702 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001703 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001704 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705 }
1706 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001707 if (pt_prev)
1708 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 rcu_read_unlock();
1710}
1711
Ben Hutchings2c530402012-07-10 10:55:09 +00001712/**
1713 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001714 * @dev: Network device
1715 * @txq: number of queues available
1716 *
1717 * If real_num_tx_queues is changed the tc mappings may no longer be
1718 * valid. To resolve this verify the tc mapping remains valid and if
1719 * not NULL the mapping. With no priorities mapping to this
1720 * offset/count pair it will no longer be used. In the worst case TC0
1721 * is invalid nothing can be done so disable priority mappings. If is
1722 * expected that drivers will fix this mapping if they can before
1723 * calling netif_set_real_num_tx_queues.
1724 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001725static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001726{
1727 int i;
1728 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1729
1730 /* If TC0 is invalidated disable TC mapping */
1731 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001732 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001733 dev->num_tc = 0;
1734 return;
1735 }
1736
1737 /* Invalidated prio to tc mappings set to TC0 */
1738 for (i = 1; i < TC_BITMASK + 1; i++) {
1739 int q = netdev_get_prio_tc_map(dev, i);
1740
1741 tc = &dev->tc_to_txq[q];
1742 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001743 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1744 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001745 netdev_set_prio_tc_map(dev, i, 0);
1746 }
1747 }
1748}
1749
John Fastabendf0796d52010-07-01 13:21:57 +00001750/*
1751 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1752 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1753 */
Tom Herberte6484932010-10-18 18:04:39 +00001754int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001755{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001756 int rc;
1757
Tom Herberte6484932010-10-18 18:04:39 +00001758 if (txq < 1 || txq > dev->num_tx_queues)
1759 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001760
Ben Hutchings5c565802011-02-15 19:39:21 +00001761 if (dev->reg_state == NETREG_REGISTERED ||
1762 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001763 ASSERT_RTNL();
1764
Tom Herbert1d24eb42010-11-21 13:17:27 +00001765 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1766 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001767 if (rc)
1768 return rc;
1769
John Fastabend4f57c082011-01-17 08:06:04 +00001770 if (dev->num_tc)
1771 netif_setup_tc(dev, txq);
1772
Tom Herberte6484932010-10-18 18:04:39 +00001773 if (txq < dev->real_num_tx_queues)
1774 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001775 }
Tom Herberte6484932010-10-18 18:04:39 +00001776
1777 dev->real_num_tx_queues = txq;
1778 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001779}
1780EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001781
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001782#ifdef CONFIG_RPS
1783/**
1784 * netif_set_real_num_rx_queues - set actual number of RX queues used
1785 * @dev: Network device
1786 * @rxq: Actual number of RX queues
1787 *
1788 * This must be called either with the rtnl_lock held or before
1789 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001790 * negative error code. If called before registration, it always
1791 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001792 */
1793int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1794{
1795 int rc;
1796
Tom Herbertbd25fa72010-10-18 18:00:16 +00001797 if (rxq < 1 || rxq > dev->num_rx_queues)
1798 return -EINVAL;
1799
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001800 if (dev->reg_state == NETREG_REGISTERED) {
1801 ASSERT_RTNL();
1802
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001803 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1804 rxq);
1805 if (rc)
1806 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001807 }
1808
1809 dev->real_num_rx_queues = rxq;
1810 return 0;
1811}
1812EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1813#endif
1814
Ben Hutchings2c530402012-07-10 10:55:09 +00001815/**
1816 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00001817 *
1818 * This routine should set an upper limit on the number of RSS queues
1819 * used by default by multiqueue devices.
1820 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00001821int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00001822{
1823 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
1824}
1825EXPORT_SYMBOL(netif_get_num_default_rss_queues);
1826
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001827static inline void __netif_reschedule(struct Qdisc *q)
1828{
1829 struct softnet_data *sd;
1830 unsigned long flags;
1831
1832 local_irq_save(flags);
1833 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001834 q->next_sched = NULL;
1835 *sd->output_queue_tailp = q;
1836 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001837 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1838 local_irq_restore(flags);
1839}
1840
David S. Miller37437bb2008-07-16 02:15:04 -07001841void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001842{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001843 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1844 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001845}
1846EXPORT_SYMBOL(__netif_schedule);
1847
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001848void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001849{
David S. Miller3578b0c2010-08-03 00:24:04 -07001850 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001851 struct softnet_data *sd;
1852 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001853
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001854 local_irq_save(flags);
1855 sd = &__get_cpu_var(softnet_data);
1856 skb->next = sd->completion_queue;
1857 sd->completion_queue = skb;
1858 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1859 local_irq_restore(flags);
1860 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001861}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001862EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001863
1864void dev_kfree_skb_any(struct sk_buff *skb)
1865{
1866 if (in_irq() || irqs_disabled())
1867 dev_kfree_skb_irq(skb);
1868 else
1869 dev_kfree_skb(skb);
1870}
1871EXPORT_SYMBOL(dev_kfree_skb_any);
1872
1873
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001874/**
1875 * netif_device_detach - mark device as removed
1876 * @dev: network device
1877 *
1878 * Mark device as removed from system and therefore no longer available.
1879 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001880void netif_device_detach(struct net_device *dev)
1881{
1882 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1883 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001884 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001885 }
1886}
1887EXPORT_SYMBOL(netif_device_detach);
1888
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001889/**
1890 * netif_device_attach - mark device as attached
1891 * @dev: network device
1892 *
1893 * Mark device as attached from system and restart if needed.
1894 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001895void netif_device_attach(struct net_device *dev)
1896{
1897 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1898 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001899 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001900 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001901 }
1902}
1903EXPORT_SYMBOL(netif_device_attach);
1904
Ben Hutchings36c92472012-01-17 07:57:56 +00001905static void skb_warn_bad_offload(const struct sk_buff *skb)
1906{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00001907 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00001908 struct net_device *dev = skb->dev;
1909 const char *driver = "";
1910
1911 if (dev && dev->dev.parent)
1912 driver = dev_driver_string(dev->dev.parent);
1913
1914 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
1915 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00001916 driver, dev ? &dev->features : &null_features,
1917 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00001918 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
1919 skb_shinfo(skb)->gso_type, skb->ip_summed);
1920}
1921
Linus Torvalds1da177e2005-04-16 15:20:36 -07001922/*
1923 * Invalidate hardware checksum when packet is to be mangled, and
1924 * complete checksum manually on outgoing path.
1925 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001926int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001927{
Al Virod3bc23e2006-11-14 21:24:49 -08001928 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001929 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930
Patrick McHardy84fa7932006-08-29 16:44:56 -07001931 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001932 goto out_set_summed;
1933
1934 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00001935 skb_warn_bad_offload(skb);
1936 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937 }
1938
Michał Mirosław55508d62010-12-14 15:24:08 +00001939 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07001940 BUG_ON(offset >= skb_headlen(skb));
1941 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1942
1943 offset += skb->csum_offset;
1944 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1945
1946 if (skb_cloned(skb) &&
1947 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001948 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1949 if (ret)
1950 goto out;
1951 }
1952
Herbert Xua0308472007-10-15 01:47:15 -07001953 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001954out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001956out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 return ret;
1958}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001959EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001961/**
1962 * skb_gso_segment - Perform segmentation on skb.
1963 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001964 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001965 *
1966 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001967 *
1968 * It may return NULL if the skb requires no segmentation. This is
1969 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001970 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00001971struct sk_buff *skb_gso_segment(struct sk_buff *skb,
1972 netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001973{
1974 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1975 struct packet_type *ptype;
Al Viro252e3342006-11-14 20:48:11 -08001976 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001977 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07001978 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001979
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001980 while (type == htons(ETH_P_8021Q)) {
1981 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001982
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001983 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00001984 return ERR_PTR(-EINVAL);
1985
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001986 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1987 type = vh->h_vlan_encapsulated_proto;
1988 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001989 }
1990
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001991 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001992 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001993 __skb_pull(skb, skb->mac_len);
1994
Herbert Xu67fd1a72009-01-19 16:26:44 -08001995 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00001996 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08001997
Herbert Xua430a432006-07-08 13:34:56 -07001998 if (skb_header_cloned(skb) &&
1999 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2000 return ERR_PTR(err);
2001 }
2002
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002003 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08002004 list_for_each_entry_rcu(ptype,
2005 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002006 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07002007 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07002008 err = ptype->gso_send_check(skb);
2009 segs = ERR_PTR(err);
2010 if (err || skb_gso_ok(skb, features))
2011 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07002012 __skb_push(skb, (skb->data -
2013 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07002014 }
Herbert Xu576a30e2006-06-27 13:22:38 -07002015 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002016 break;
2017 }
2018 }
2019 rcu_read_unlock();
2020
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002021 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07002022
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002023 return segs;
2024}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002025EXPORT_SYMBOL(skb_gso_segment);
2026
Herbert Xufb286bb2005-11-10 13:01:24 -08002027/* Take action when hardware reception checksum errors are detected. */
2028#ifdef CONFIG_BUG
2029void netdev_rx_csum_fault(struct net_device *dev)
2030{
2031 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002032 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002033 dump_stack();
2034 }
2035}
2036EXPORT_SYMBOL(netdev_rx_csum_fault);
2037#endif
2038
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039/* Actually, we should eliminate this check as soon as we know, that:
2040 * 1. IOMMU is present and allows to map all the memory.
2041 * 2. No high memory really exists on this machine.
2042 */
2043
Eric Dumazet9092c652010-04-02 13:34:49 -07002044static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002046#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002047 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002048 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002049 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2050 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2051 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002052 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002053 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002054 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002055
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002056 if (PCI_DMA_BUS_IS_PHYS) {
2057 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058
Eric Dumazet9092c652010-04-02 13:34:49 -07002059 if (!pdev)
2060 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002061 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002062 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2063 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002064 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2065 return 1;
2066 }
2067 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002068#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069 return 0;
2070}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002072struct dev_gso_cb {
2073 void (*destructor)(struct sk_buff *skb);
2074};
2075
2076#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2077
2078static void dev_gso_skb_destructor(struct sk_buff *skb)
2079{
2080 struct dev_gso_cb *cb;
2081
2082 do {
2083 struct sk_buff *nskb = skb->next;
2084
2085 skb->next = nskb->next;
2086 nskb->next = NULL;
2087 kfree_skb(nskb);
2088 } while (skb->next);
2089
2090 cb = DEV_GSO_CB(skb);
2091 if (cb->destructor)
2092 cb->destructor(skb);
2093}
2094
2095/**
2096 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2097 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002098 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002099 *
2100 * This function segments the given skb and stores the list of segments
2101 * in skb->next.
2102 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002103static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002104{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002105 struct sk_buff *segs;
2106
Herbert Xu576a30e2006-06-27 13:22:38 -07002107 segs = skb_gso_segment(skb, features);
2108
2109 /* Verifying header integrity only. */
2110 if (!segs)
2111 return 0;
2112
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002113 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002114 return PTR_ERR(segs);
2115
2116 skb->next = segs;
2117 DEV_GSO_CB(skb)->destructor = skb->destructor;
2118 skb->destructor = dev_gso_skb_destructor;
2119
2120 return 0;
2121}
2122
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002123static bool can_checksum_protocol(netdev_features_t features, __be16 protocol)
Jesse Gross03634662011-01-09 06:23:35 +00002124{
2125 return ((features & NETIF_F_GEN_CSUM) ||
2126 ((features & NETIF_F_V4_CSUM) &&
2127 protocol == htons(ETH_P_IP)) ||
2128 ((features & NETIF_F_V6_CSUM) &&
2129 protocol == htons(ETH_P_IPV6)) ||
2130 ((features & NETIF_F_FCOE_CRC) &&
2131 protocol == htons(ETH_P_FCOE)));
2132}
2133
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002134static netdev_features_t harmonize_features(struct sk_buff *skb,
2135 __be16 protocol, netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002136{
Ed Cashinc0d680e2012-09-19 15:49:00 +00002137 if (skb->ip_summed != CHECKSUM_NONE &&
2138 !can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002139 features &= ~NETIF_F_ALL_CSUM;
2140 features &= ~NETIF_F_SG;
2141 } else if (illegal_highdma(skb->dev, skb)) {
2142 features &= ~NETIF_F_SG;
2143 }
2144
2145 return features;
2146}
2147
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002148netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002149{
2150 __be16 protocol = skb->protocol;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002151 netdev_features_t features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002152
Ben Hutchings30b678d2012-07-30 15:57:00 +00002153 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2154 features &= ~NETIF_F_GSO_MASK;
2155
Jesse Gross58e998c2010-10-29 12:14:55 +00002156 if (protocol == htons(ETH_P_8021Q)) {
2157 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2158 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002159 } else if (!vlan_tx_tag_present(skb)) {
2160 return harmonize_features(skb, protocol, features);
2161 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002162
Jesse Gross6ee400a2011-01-17 20:46:00 +00002163 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002164
2165 if (protocol != htons(ETH_P_8021Q)) {
2166 return harmonize_features(skb, protocol, features);
2167 } else {
2168 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002169 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002170 return harmonize_features(skb, protocol, features);
2171 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002172}
Jesse Grossf01a5232011-01-09 06:23:31 +00002173EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002174
John Fastabend6afff0c2010-06-16 14:18:12 +00002175/*
2176 * Returns true if either:
2177 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2178 * 2. skb is fragmented and the device does not support SG, or if
2179 * at least one of fragments is in highmem and device does not
2180 * support DMA from it.
2181 */
2182static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002183 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002184{
Jesse Gross02932ce2011-01-09 06:23:34 +00002185 return skb_is_nonlinear(skb) &&
2186 ((skb_has_frag_list(skb) &&
2187 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002188 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002189 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002190}
2191
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002192int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2193 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002194{
Stephen Hemminger00829822008-11-20 20:14:53 -08002195 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002196 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002197 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002198
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002199 if (likely(!skb->next)) {
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002200 netdev_features_t features;
Jesse Grossfc741212011-01-09 06:23:32 +00002201
Eric Dumazet93f154b2009-05-18 22:19:19 -07002202 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002203 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002204 * its hot in this cpu cache
2205 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002206 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2207 skb_dst_drop(skb);
2208
Eric Dumazet15c2d75f2010-12-07 00:30:37 +00002209 if (!list_empty(&ptype_all))
2210 dev_queue_xmit_nit(skb, dev);
2211
Jesse Grossfc741212011-01-09 06:23:32 +00002212 features = netif_skb_features(skb);
2213
Jesse Gross7b9c6092010-10-20 13:56:04 +00002214 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002215 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002216 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2217 if (unlikely(!skb))
2218 goto out;
2219
2220 skb->vlan_tci = 0;
2221 }
2222
Jesse Grossfc741212011-01-09 06:23:32 +00002223 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002224 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002225 goto out_kfree_skb;
2226 if (skb->next)
2227 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002228 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002229 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002230 __skb_linearize(skb))
2231 goto out_kfree_skb;
2232
2233 /* If packet is not checksummed and device does not
2234 * support checksumming for this protocol, complete
2235 * checksumming here.
2236 */
2237 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Michał Mirosław55508d62010-12-14 15:24:08 +00002238 skb_set_transport_header(skb,
2239 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002240 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002241 skb_checksum_help(skb))
2242 goto out_kfree_skb;
2243 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002244 }
2245
Koki Sanagiec764bf2011-05-30 21:48:34 +00002246 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002247 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002248 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002249 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002250 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002251 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002252 }
2253
Herbert Xu576a30e2006-06-27 13:22:38 -07002254gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002255 do {
2256 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002257
2258 skb->next = nskb->next;
2259 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002260
2261 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002262 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002263 * its hot in this cpu cache
2264 */
2265 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2266 skb_dst_drop(nskb);
2267
Koki Sanagiec764bf2011-05-30 21:48:34 +00002268 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002269 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002270 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002271 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002272 if (rc & ~NETDEV_TX_MASK)
2273 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002274 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002275 skb->next = nskb;
2276 return rc;
2277 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002278 txq_trans_update(txq);
Tom Herbert734664982011-11-28 16:32:44 +00002279 if (unlikely(netif_xmit_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002280 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002281 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002282
Patrick McHardy572a9d72009-11-10 06:14:14 +00002283out_kfree_gso_skb:
2284 if (likely(skb->next == NULL))
2285 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002286out_kfree_skb:
2287 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002288out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002289 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002290}
2291
Tom Herbert0a9627f2010-03-16 08:03:29 +00002292static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002293
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002294/*
2295 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2296 * to be used as a distribution range.
2297 */
2298u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2299 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002300{
David S. Miller70192982009-01-27 16:34:47 -08002301 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002302 u16 qoffset = 0;
2303 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002304
David S. Miller513de112009-05-03 14:43:10 -07002305 if (skb_rx_queue_recorded(skb)) {
2306 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002307 while (unlikely(hash >= num_tx_queues))
2308 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002309 return hash;
2310 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002311
John Fastabend4f57c082011-01-17 08:06:04 +00002312 if (dev->num_tc) {
2313 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2314 qoffset = dev->tc_to_txq[tc].offset;
2315 qcount = dev->tc_to_txq[tc].count;
2316 }
2317
Eric Dumazetec581f62009-05-01 09:05:06 -07002318 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002319 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002320 else
Eric Dumazet62b1a8a2012-06-14 06:42:44 +00002321 hash = (__force u16) skb->protocol;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002322 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002323
John Fastabend4f57c082011-01-17 08:06:04 +00002324 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002325}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002326EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002327
Eric Dumazeted046422009-11-13 21:54:04 +00002328static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2329{
2330 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
Joe Perchese87cc472012-05-13 21:56:26 +00002331 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2332 dev->name, queue_index,
2333 dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002334 return 0;
2335 }
2336 return queue_index;
2337}
2338
Tom Herbert1d24eb42010-11-21 13:17:27 +00002339static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2340{
Tom Herbertbf264142010-11-26 08:36:09 +00002341#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002342 struct xps_dev_maps *dev_maps;
2343 struct xps_map *map;
2344 int queue_index = -1;
2345
2346 rcu_read_lock();
2347 dev_maps = rcu_dereference(dev->xps_maps);
2348 if (dev_maps) {
2349 map = rcu_dereference(
2350 dev_maps->cpu_map[raw_smp_processor_id()]);
2351 if (map) {
2352 if (map->len == 1)
2353 queue_index = map->queues[0];
2354 else {
2355 u32 hash;
2356 if (skb->sk && skb->sk->sk_hash)
2357 hash = skb->sk->sk_hash;
2358 else
2359 hash = (__force u16) skb->protocol ^
2360 skb->rxhash;
2361 hash = jhash_1word(hash, hashrnd);
2362 queue_index = map->queues[
2363 ((u64)hash * map->len) >> 32];
2364 }
2365 if (unlikely(queue_index >= dev->real_num_tx_queues))
2366 queue_index = -1;
2367 }
2368 }
2369 rcu_read_unlock();
2370
2371 return queue_index;
2372#else
2373 return -1;
2374#endif
2375}
2376
David S. Millere8a04642008-07-17 00:34:19 -07002377static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2378 struct sk_buff *skb)
2379{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002380 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002381 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002382
Tom Herbert3853b582010-11-21 13:17:29 +00002383 if (dev->real_num_tx_queues == 1)
2384 queue_index = 0;
2385 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002386 queue_index = ops->ndo_select_queue(dev, skb);
2387 queue_index = dev_cap_txqueue(dev, queue_index);
2388 } else {
2389 struct sock *sk = skb->sk;
2390 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002391
Tom Herbert3853b582010-11-21 13:17:29 +00002392 if (queue_index < 0 || skb->ooo_okay ||
2393 queue_index >= dev->real_num_tx_queues) {
2394 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002395
Tom Herbert1d24eb42010-11-21 13:17:27 +00002396 queue_index = get_xps_queue(dev, skb);
2397 if (queue_index < 0)
2398 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002399
2400 if (queue_index != old_index && sk) {
2401 struct dst_entry *dst =
2402 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002403
2404 if (dst && skb_dst(skb) == dst)
2405 sk_tx_queue_set(sk, queue_index);
2406 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002407 }
2408 }
David S. Millereae792b2008-07-15 03:03:33 -07002409
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002410 skb_set_queue_mapping(skb, queue_index);
2411 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002412}
2413
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002414static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2415 struct net_device *dev,
2416 struct netdev_queue *txq)
2417{
2418 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002419 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002420 int rc;
2421
Eric Dumazeta2da5702011-01-20 03:48:19 +00002422 qdisc_skb_cb(skb)->pkt_len = skb->len;
2423 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002424 /*
2425 * Heuristic to force contended enqueues to serialize on a
2426 * separate lock before trying to get qdisc main lock.
2427 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2428 * and dequeue packets faster.
2429 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002430 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002431 if (unlikely(contended))
2432 spin_lock(&q->busylock);
2433
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002434 spin_lock(root_lock);
2435 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2436 kfree_skb(skb);
2437 rc = NET_XMIT_DROP;
2438 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002439 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002440 /*
2441 * This is a work-conserving queue; there are no old skbs
2442 * waiting to be sent out; and the qdisc is not running -
2443 * xmit the skb directly.
2444 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002445 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2446 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002447
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002448 qdisc_bstats_update(q, skb);
2449
Eric Dumazet79640a42010-06-02 05:09:29 -07002450 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2451 if (unlikely(contended)) {
2452 spin_unlock(&q->busylock);
2453 contended = false;
2454 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002455 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002456 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002457 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002458
2459 rc = NET_XMIT_SUCCESS;
2460 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002461 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002462 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002463 if (qdisc_run_begin(q)) {
2464 if (unlikely(contended)) {
2465 spin_unlock(&q->busylock);
2466 contended = false;
2467 }
2468 __qdisc_run(q);
2469 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002470 }
2471 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002472 if (unlikely(contended))
2473 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002474 return rc;
2475}
2476
Neil Horman5bc14212011-11-22 05:10:51 +00002477#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2478static void skb_update_prio(struct sk_buff *skb)
2479{
Igor Maravic6977a792011-11-25 07:44:54 +00002480 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002481
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002482 if (!skb->priority && skb->sk && map) {
2483 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2484
2485 if (prioidx < map->priomap_len)
2486 skb->priority = map->priomap[prioidx];
2487 }
Neil Horman5bc14212011-11-22 05:10:51 +00002488}
2489#else
2490#define skb_update_prio(skb)
2491#endif
2492
Eric Dumazet745e20f2010-09-29 13:23:09 -07002493static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002494#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002495
Dave Jonesd29f7492008-07-22 14:09:06 -07002496/**
Michel Machado95603e22012-06-12 10:16:35 +00002497 * dev_loopback_xmit - loop back @skb
2498 * @skb: buffer to transmit
2499 */
2500int dev_loopback_xmit(struct sk_buff *skb)
2501{
2502 skb_reset_mac_header(skb);
2503 __skb_pull(skb, skb_network_offset(skb));
2504 skb->pkt_type = PACKET_LOOPBACK;
2505 skb->ip_summed = CHECKSUM_UNNECESSARY;
2506 WARN_ON(!skb_dst(skb));
2507 skb_dst_force(skb);
2508 netif_rx_ni(skb);
2509 return 0;
2510}
2511EXPORT_SYMBOL(dev_loopback_xmit);
2512
2513/**
Dave Jonesd29f7492008-07-22 14:09:06 -07002514 * dev_queue_xmit - transmit a buffer
2515 * @skb: buffer to transmit
2516 *
2517 * Queue a buffer for transmission to a network device. The caller must
2518 * have set the device and priority and built the buffer before calling
2519 * this function. The function can be called from an interrupt.
2520 *
2521 * A negative errno code is returned on a failure. A success does not
2522 * guarantee the frame will be transmitted as it may be dropped due
2523 * to congestion or traffic shaping.
2524 *
2525 * -----------------------------------------------------------------------------------
2526 * I notice this method can also return errors from the queue disciplines,
2527 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2528 * be positive.
2529 *
2530 * Regardless of the return value, the skb is consumed, so it is currently
2531 * difficult to retry a send to this method. (You can bump the ref count
2532 * before sending to hold a reference for retry if you are careful.)
2533 *
2534 * When calling this method, interrupts MUST be enabled. This is because
2535 * the BH enable code must have IRQs enabled so that it will not deadlock.
2536 * --BLG
2537 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002538int dev_queue_xmit(struct sk_buff *skb)
2539{
2540 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002541 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002542 struct Qdisc *q;
2543 int rc = -ENOMEM;
2544
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002545 /* Disable soft irqs for various locks below. Also
2546 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002547 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002548 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002549
Neil Horman5bc14212011-11-22 05:10:51 +00002550 skb_update_prio(skb);
2551
David S. Millereae792b2008-07-15 03:03:33 -07002552 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002553 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002554
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002556 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002558 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002560 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002561 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562 }
2563
2564 /* The device has no queue. Common case for software devices:
2565 loopback, all the sorts of tunnels...
2566
Herbert Xu932ff272006-06-09 12:20:56 -07002567 Really, it is unlikely that netif_tx_lock protection is necessary
2568 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569 counters.)
2570 However, it is possible, that they rely on protection
2571 made by us here.
2572
2573 Check this and shot the lock. It is not prone from deadlocks.
2574 Either shot noqueue qdisc, it is even simpler 8)
2575 */
2576 if (dev->flags & IFF_UP) {
2577 int cpu = smp_processor_id(); /* ok because BHs are off */
2578
David S. Millerc773e842008-07-08 23:13:53 -07002579 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002580
Eric Dumazet745e20f2010-09-29 13:23:09 -07002581 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2582 goto recursion_alert;
2583
David S. Millerc773e842008-07-08 23:13:53 -07002584 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002585
Tom Herbert734664982011-11-28 16:32:44 +00002586 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002587 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002588 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002589 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002590 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002591 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002592 goto out;
2593 }
2594 }
David S. Millerc773e842008-07-08 23:13:53 -07002595 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002596 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2597 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002598 } else {
2599 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002600 * unfortunately
2601 */
2602recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00002603 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2604 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002605 }
2606 }
2607
2608 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002609 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002610
Linus Torvalds1da177e2005-04-16 15:20:36 -07002611 kfree_skb(skb);
2612 return rc;
2613out:
Herbert Xud4828d82006-06-22 02:28:18 -07002614 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002615 return rc;
2616}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002617EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002618
2619
2620/*=======================================================================
2621 Receiver routines
2622 =======================================================================*/
2623
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002624int netdev_max_backlog __read_mostly = 1000;
Eric Dumazet3b098e22010-05-15 23:57:10 -07002625int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002626int netdev_budget __read_mostly = 300;
2627int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002628
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002629/* Called with irq disabled */
2630static inline void ____napi_schedule(struct softnet_data *sd,
2631 struct napi_struct *napi)
2632{
2633 list_add_tail(&napi->poll_list, &sd->poll_list);
2634 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2635}
2636
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002637/*
2638 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
Tom Herbertbdeab992011-08-14 19:45:55 +00002639 * and src/dst port numbers. Sets rxhash in skb to non-zero hash value
2640 * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
2641 * if hash is a canonical 4-tuple hash over transport ports.
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002642 */
Tom Herbertbdeab992011-08-14 19:45:55 +00002643void __skb_get_rxhash(struct sk_buff *skb)
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002644{
Eric Dumazet4504b862011-11-28 05:23:23 +00002645 struct flow_keys keys;
2646 u32 hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002647
Eric Dumazet4504b862011-11-28 05:23:23 +00002648 if (!skb_flow_dissect(skb, &keys))
2649 return;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002650
Chema Gonzalez68622342012-09-07 13:40:50 +00002651 if (keys.ports)
Eric Dumazet4504b862011-11-28 05:23:23 +00002652 skb->l4_rxhash = 1;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002653
2654 /* get a consistent hash (same value on both flow directions) */
Chema Gonzalez68622342012-09-07 13:40:50 +00002655 if (((__force u32)keys.dst < (__force u32)keys.src) ||
2656 (((__force u32)keys.dst == (__force u32)keys.src) &&
2657 ((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) {
Eric Dumazet4504b862011-11-28 05:23:23 +00002658 swap(keys.dst, keys.src);
Chema Gonzalez68622342012-09-07 13:40:50 +00002659 swap(keys.port16[0], keys.port16[1]);
2660 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002661
Eric Dumazet4504b862011-11-28 05:23:23 +00002662 hash = jhash_3words((__force u32)keys.dst,
2663 (__force u32)keys.src,
2664 (__force u32)keys.ports, hashrnd);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002665 if (!hash)
2666 hash = 1;
2667
Tom Herbertbdeab992011-08-14 19:45:55 +00002668 skb->rxhash = hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002669}
2670EXPORT_SYMBOL(__skb_get_rxhash);
2671
Eric Dumazetdf334542010-03-24 19:13:54 +00002672#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002673
2674/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002675struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002676EXPORT_SYMBOL(rps_sock_flow_table);
2677
Ingo Molnarc5905af2012-02-24 08:31:31 +01002678struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00002679
Ben Hutchingsc4454772011-01-19 11:03:53 +00002680static struct rps_dev_flow *
2681set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2682 struct rps_dev_flow *rflow, u16 next_cpu)
2683{
Ben Hutchings09994d12011-10-03 04:42:46 +00002684 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002685#ifdef CONFIG_RFS_ACCEL
2686 struct netdev_rx_queue *rxqueue;
2687 struct rps_dev_flow_table *flow_table;
2688 struct rps_dev_flow *old_rflow;
2689 u32 flow_id;
2690 u16 rxq_index;
2691 int rc;
2692
2693 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002694 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2695 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002696 goto out;
2697 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2698 if (rxq_index == skb_get_rx_queue(skb))
2699 goto out;
2700
2701 rxqueue = dev->_rx + rxq_index;
2702 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2703 if (!flow_table)
2704 goto out;
2705 flow_id = skb->rxhash & flow_table->mask;
2706 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2707 rxq_index, flow_id);
2708 if (rc < 0)
2709 goto out;
2710 old_rflow = rflow;
2711 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002712 rflow->filter = rc;
2713 if (old_rflow->filter == rflow->filter)
2714 old_rflow->filter = RPS_NO_FILTER;
2715 out:
2716#endif
2717 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002718 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002719 }
2720
Ben Hutchings09994d12011-10-03 04:42:46 +00002721 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002722 return rflow;
2723}
2724
Tom Herbert0a9627f2010-03-16 08:03:29 +00002725/*
2726 * get_rps_cpu is called from netif_receive_skb and returns the target
2727 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002728 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002729 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002730static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2731 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002732{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002733 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002734 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002735 struct rps_dev_flow_table *flow_table;
2736 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002737 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002738 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002739
Tom Herbert0a9627f2010-03-16 08:03:29 +00002740 if (skb_rx_queue_recorded(skb)) {
2741 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002742 if (unlikely(index >= dev->real_num_rx_queues)) {
2743 WARN_ONCE(dev->real_num_rx_queues > 1,
2744 "%s received packet on queue %u, but number "
2745 "of RX queues is %u\n",
2746 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002747 goto done;
2748 }
2749 rxqueue = dev->_rx + index;
2750 } else
2751 rxqueue = dev->_rx;
2752
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002753 map = rcu_dereference(rxqueue->rps_map);
2754 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002755 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002756 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002757 tcpu = map->cpus[0];
2758 if (cpu_online(tcpu))
2759 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002760 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002761 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002762 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002763 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002764 }
2765
Changli Gao2d47b452010-08-17 19:00:56 +00002766 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002767 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002768 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002769
Tom Herbertfec5e652010-04-16 16:01:27 -07002770 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2771 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2772 if (flow_table && sock_flow_table) {
2773 u16 next_cpu;
2774 struct rps_dev_flow *rflow;
2775
2776 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2777 tcpu = rflow->cpu;
2778
2779 next_cpu = sock_flow_table->ents[skb->rxhash &
2780 sock_flow_table->mask];
2781
2782 /*
2783 * If the desired CPU (where last recvmsg was done) is
2784 * different from current CPU (one in the rx-queue flow
2785 * table entry), switch if one of the following holds:
2786 * - Current CPU is unset (equal to RPS_NO_CPU).
2787 * - Current CPU is offline.
2788 * - The current CPU's queue tail has advanced beyond the
2789 * last packet that was enqueued using this table entry.
2790 * This guarantees that all previous packets for the flow
2791 * have been dequeued, thus preserving in order delivery.
2792 */
2793 if (unlikely(tcpu != next_cpu) &&
2794 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2795 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Ben Hutchingsc4454772011-01-19 11:03:53 +00002796 rflow->last_qtail)) >= 0))
2797 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2798
Tom Herbertfec5e652010-04-16 16:01:27 -07002799 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2800 *rflowp = rflow;
2801 cpu = tcpu;
2802 goto done;
2803 }
2804 }
2805
Tom Herbert0a9627f2010-03-16 08:03:29 +00002806 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002807 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002808
2809 if (cpu_online(tcpu)) {
2810 cpu = tcpu;
2811 goto done;
2812 }
2813 }
2814
2815done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002816 return cpu;
2817}
2818
Ben Hutchingsc4454772011-01-19 11:03:53 +00002819#ifdef CONFIG_RFS_ACCEL
2820
2821/**
2822 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2823 * @dev: Device on which the filter was set
2824 * @rxq_index: RX queue index
2825 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2826 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2827 *
2828 * Drivers that implement ndo_rx_flow_steer() should periodically call
2829 * this function for each installed filter and remove the filters for
2830 * which it returns %true.
2831 */
2832bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2833 u32 flow_id, u16 filter_id)
2834{
2835 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2836 struct rps_dev_flow_table *flow_table;
2837 struct rps_dev_flow *rflow;
2838 bool expire = true;
2839 int cpu;
2840
2841 rcu_read_lock();
2842 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2843 if (flow_table && flow_id <= flow_table->mask) {
2844 rflow = &flow_table->flows[flow_id];
2845 cpu = ACCESS_ONCE(rflow->cpu);
2846 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2847 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2848 rflow->last_qtail) <
2849 (int)(10 * flow_table->mask)))
2850 expire = false;
2851 }
2852 rcu_read_unlock();
2853 return expire;
2854}
2855EXPORT_SYMBOL(rps_may_expire_flow);
2856
2857#endif /* CONFIG_RFS_ACCEL */
2858
Tom Herbert0a9627f2010-03-16 08:03:29 +00002859/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002860static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002861{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002862 struct softnet_data *sd = data;
2863
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002864 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002865 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002866}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002867
Tom Herbertfec5e652010-04-16 16:01:27 -07002868#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002869
2870/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002871 * Check if this softnet_data structure is another cpu one
2872 * If yes, queue it to our IPI list and return 1
2873 * If no, return 0
2874 */
2875static int rps_ipi_queued(struct softnet_data *sd)
2876{
2877#ifdef CONFIG_RPS
2878 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2879
2880 if (sd != mysd) {
2881 sd->rps_ipi_next = mysd->rps_ipi_list;
2882 mysd->rps_ipi_list = sd;
2883
2884 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2885 return 1;
2886 }
2887#endif /* CONFIG_RPS */
2888 return 0;
2889}
2890
2891/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00002892 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2893 * queue (may be a remote CPU queue).
2894 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002895static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2896 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002897{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002898 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002899 unsigned long flags;
2900
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002901 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002902
2903 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002904
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002905 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07002906 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2907 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002908enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002909 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00002910 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002911 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00002912 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002913 return NET_RX_SUCCESS;
2914 }
2915
Eric Dumazetebda37c22010-05-06 23:51:21 +00002916 /* Schedule NAPI for backlog device
2917 * We can use non atomic operation since we own the queue lock
2918 */
2919 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002920 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002921 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002922 }
2923 goto enqueue;
2924 }
2925
Changli Gaodee42872010-05-02 05:42:16 +00002926 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002927 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002928
Tom Herbert0a9627f2010-03-16 08:03:29 +00002929 local_irq_restore(flags);
2930
Eric Dumazetcaf586e2010-09-30 21:06:55 +00002931 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002932 kfree_skb(skb);
2933 return NET_RX_DROP;
2934}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002935
Linus Torvalds1da177e2005-04-16 15:20:36 -07002936/**
2937 * netif_rx - post buffer to the network code
2938 * @skb: buffer to post
2939 *
2940 * This function receives a packet from a device driver and queues it for
2941 * the upper (protocol) levels to process. It always succeeds. The buffer
2942 * may be dropped during processing for congestion control or by the
2943 * protocol layers.
2944 *
2945 * return values:
2946 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002947 * NET_RX_DROP (packet was dropped)
2948 *
2949 */
2950
2951int netif_rx(struct sk_buff *skb)
2952{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002953 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002954
2955 /* if netpoll wants it, pretend we never saw it */
2956 if (netpoll_rx(skb))
2957 return NET_RX_DROP;
2958
Eric Dumazet588f0332011-11-15 04:12:55 +00002959 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002960
Koki Sanagicf66ba52010-08-23 18:45:02 +09002961 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00002962#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01002963 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002964 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002965 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002966
Changli Gaocece1942010-08-07 20:35:43 -07002967 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002968 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07002969
2970 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002971 if (cpu < 0)
2972 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07002973
2974 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2975
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002976 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07002977 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00002978 } else
2979#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07002980 {
2981 unsigned int qtail;
2982 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2983 put_cpu();
2984 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002985 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002986}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002987EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002988
2989int netif_rx_ni(struct sk_buff *skb)
2990{
2991 int err;
2992
2993 preempt_disable();
2994 err = netif_rx(skb);
2995 if (local_softirq_pending())
2996 do_softirq();
2997 preempt_enable();
2998
2999 return err;
3000}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001EXPORT_SYMBOL(netif_rx_ni);
3002
Linus Torvalds1da177e2005-04-16 15:20:36 -07003003static void net_tx_action(struct softirq_action *h)
3004{
3005 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3006
3007 if (sd->completion_queue) {
3008 struct sk_buff *clist;
3009
3010 local_irq_disable();
3011 clist = sd->completion_queue;
3012 sd->completion_queue = NULL;
3013 local_irq_enable();
3014
3015 while (clist) {
3016 struct sk_buff *skb = clist;
3017 clist = clist->next;
3018
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003019 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09003020 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003021 __kfree_skb(skb);
3022 }
3023 }
3024
3025 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003026 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003027
3028 local_irq_disable();
3029 head = sd->output_queue;
3030 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003031 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003032 local_irq_enable();
3033
3034 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003035 struct Qdisc *q = head;
3036 spinlock_t *root_lock;
3037
Linus Torvalds1da177e2005-04-16 15:20:36 -07003038 head = head->next_sched;
3039
David S. Miller5fb66222008-08-02 20:02:43 -07003040 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003041 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003042 smp_mb__before_clear_bit();
3043 clear_bit(__QDISC_STATE_SCHED,
3044 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003045 qdisc_run(q);
3046 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003047 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003048 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003049 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003050 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003051 } else {
3052 smp_mb__before_clear_bit();
3053 clear_bit(__QDISC_STATE_SCHED,
3054 &q->state);
3055 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003056 }
3057 }
3058 }
3059}
3060
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003061#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3062 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003063/* This hook is defined here for ATM LANE */
3064int (*br_fdb_test_addr_hook)(struct net_device *dev,
3065 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003066EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003067#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003068
Linus Torvalds1da177e2005-04-16 15:20:36 -07003069#ifdef CONFIG_NET_CLS_ACT
3070/* TODO: Maybe we should just force sch_ingress to be compiled in
3071 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3072 * a compare and 2 stores extra right now if we dont have it on
3073 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003074 * NOTE: This doesn't stop any functionality; if you dont have
3075 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003076 *
3077 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003078static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003079{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003081 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003082 int result = TC_ACT_OK;
3083 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003084
Stephen Hemmingerde384832010-08-01 00:33:23 -07003085 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003086 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3087 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003088 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003089 }
3090
Herbert Xuf697c3e2007-10-14 00:38:47 -07003091 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3092 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3093
David S. Miller83874002008-07-17 00:53:03 -07003094 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003095 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003096 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003097 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3098 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003099 spin_unlock(qdisc_lock(q));
3100 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003101
Linus Torvalds1da177e2005-04-16 15:20:36 -07003102 return result;
3103}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003104
3105static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3106 struct packet_type **pt_prev,
3107 int *ret, struct net_device *orig_dev)
3108{
Eric Dumazet24824a02010-10-02 06:11:55 +00003109 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3110
3111 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003112 goto out;
3113
3114 if (*pt_prev) {
3115 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3116 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003117 }
3118
Eric Dumazet24824a02010-10-02 06:11:55 +00003119 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003120 case TC_ACT_SHOT:
3121 case TC_ACT_STOLEN:
3122 kfree_skb(skb);
3123 return NULL;
3124 }
3125
3126out:
3127 skb->tc_verd = 0;
3128 return skb;
3129}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130#endif
3131
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003132/**
3133 * netdev_rx_handler_register - register receive handler
3134 * @dev: device to register a handler for
3135 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003136 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003137 *
3138 * Register a receive hander for a device. This handler will then be
3139 * called from __netif_receive_skb. A negative errno code is returned
3140 * on a failure.
3141 *
3142 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003143 *
3144 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003145 */
3146int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003147 rx_handler_func_t *rx_handler,
3148 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003149{
3150 ASSERT_RTNL();
3151
3152 if (dev->rx_handler)
3153 return -EBUSY;
3154
Jiri Pirko93e2c322010-06-10 03:34:59 +00003155 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003156 rcu_assign_pointer(dev->rx_handler, rx_handler);
3157
3158 return 0;
3159}
3160EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3161
3162/**
3163 * netdev_rx_handler_unregister - unregister receive handler
3164 * @dev: device to unregister a handler from
3165 *
3166 * Unregister a receive hander from a device.
3167 *
3168 * The caller must hold the rtnl_mutex.
3169 */
3170void netdev_rx_handler_unregister(struct net_device *dev)
3171{
3172
3173 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003174 RCU_INIT_POINTER(dev->rx_handler, NULL);
3175 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003176}
3177EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3178
Mel Gormanb4b9e352012-07-31 16:44:26 -07003179/*
3180 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3181 * the special handling of PFMEMALLOC skbs.
3182 */
3183static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3184{
3185 switch (skb->protocol) {
3186 case __constant_htons(ETH_P_ARP):
3187 case __constant_htons(ETH_P_IP):
3188 case __constant_htons(ETH_P_IPV6):
3189 case __constant_htons(ETH_P_8021Q):
3190 return true;
3191 default:
3192 return false;
3193 }
3194}
3195
Eric Dumazet10f744d2010-03-28 23:07:20 -07003196static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003197{
3198 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003199 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003200 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003201 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003202 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003203 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003204 __be16 type;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003205 unsigned long pflags = current->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206
Eric Dumazet588f0332011-11-15 04:12:55 +00003207 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003208
Koki Sanagicf66ba52010-08-23 18:45:02 +09003209 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003210
Mel Gormanb4b9e352012-07-31 16:44:26 -07003211 /*
3212 * PFMEMALLOC skbs are special, they should
3213 * - be delivered to SOCK_MEMALLOC sockets only
3214 * - stay away from userspace
3215 * - have bounded memory usage
3216 *
3217 * Use PF_MEMALLOC as this saves us from propagating the allocation
3218 * context down to all allocation sites.
3219 */
3220 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
3221 current->flags |= PF_MEMALLOC;
3222
Linus Torvalds1da177e2005-04-16 15:20:36 -07003223 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003224 if (netpoll_receive_skb(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003225 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003226
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003227 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003228
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003229 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003230 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003231 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003232
3233 pt_prev = NULL;
3234
3235 rcu_read_lock();
3236
David S. Miller63d8ea72011-02-28 10:48:59 -08003237another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003238 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003239
3240 __this_cpu_inc(softnet_data.processed);
3241
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003242 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3243 skb = vlan_untag(skb);
3244 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003245 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003246 }
3247
Linus Torvalds1da177e2005-04-16 15:20:36 -07003248#ifdef CONFIG_NET_CLS_ACT
3249 if (skb->tc_verd & TC_NCLS) {
3250 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3251 goto ncls;
3252 }
3253#endif
3254
Mel Gormanb4b9e352012-07-31 16:44:26 -07003255 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
3256 goto skip_taps;
3257
Linus Torvalds1da177e2005-04-16 15:20:36 -07003258 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003259 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003260 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003261 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262 pt_prev = ptype;
3263 }
3264 }
3265
Mel Gormanb4b9e352012-07-31 16:44:26 -07003266skip_taps:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003267#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003268 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3269 if (!skb)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003270 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003271ncls:
3272#endif
3273
Mel Gormanb4b9e352012-07-31 16:44:26 -07003274 if (sk_memalloc_socks() && skb_pfmemalloc(skb)
3275 && !skb_pfmemalloc_protocol(skb))
3276 goto drop;
3277
Eric Dumazet6a32e4f2011-10-29 06:13:39 +00003278 rx_handler = rcu_dereference(skb->dev->rx_handler);
John Fastabend24257172011-10-10 09:16:41 +00003279 if (vlan_tx_tag_present(skb)) {
3280 if (pt_prev) {
3281 ret = deliver_skb(skb, pt_prev, orig_dev);
3282 pt_prev = NULL;
3283 }
Eric Dumazet6a32e4f2011-10-29 06:13:39 +00003284 if (vlan_do_receive(&skb, !rx_handler))
John Fastabend24257172011-10-10 09:16:41 +00003285 goto another_round;
3286 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003287 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003288 }
3289
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003290 if (rx_handler) {
3291 if (pt_prev) {
3292 ret = deliver_skb(skb, pt_prev, orig_dev);
3293 pt_prev = NULL;
3294 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003295 switch (rx_handler(&skb)) {
3296 case RX_HANDLER_CONSUMED:
Mel Gormanb4b9e352012-07-31 16:44:26 -07003297 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003298 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003299 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003300 case RX_HANDLER_EXACT:
3301 deliver_exact = true;
3302 case RX_HANDLER_PASS:
3303 break;
3304 default:
3305 BUG();
3306 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003307 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003308
David S. Miller63d8ea72011-02-28 10:48:59 -08003309 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003310 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003311
Linus Torvalds1da177e2005-04-16 15:20:36 -07003312 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003313 list_for_each_entry_rcu(ptype,
3314 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003315 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003316 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3317 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003318 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003319 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003320 pt_prev = ptype;
3321 }
3322 }
3323
3324 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003325 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003326 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003327 else
3328 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003329 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003330drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003331 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003332 kfree_skb(skb);
3333 /* Jamal, now you will not able to escape explaining
3334 * me how you were going to use this. :-)
3335 */
3336 ret = NET_RX_DROP;
3337 }
3338
Mel Gormanb4b9e352012-07-31 16:44:26 -07003339unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003340 rcu_read_unlock();
Mel Gormanb4b9e352012-07-31 16:44:26 -07003341out:
3342 tsk_restore_flags(current, pflags, PF_MEMALLOC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003343 return ret;
3344}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003345
3346/**
3347 * netif_receive_skb - process receive buffer from network
3348 * @skb: buffer to process
3349 *
3350 * netif_receive_skb() is the main receive data processing function.
3351 * It always succeeds. The buffer may be dropped during processing
3352 * for congestion control or by the protocol layers.
3353 *
3354 * This function may only be called from softirq context and interrupts
3355 * should be enabled.
3356 *
3357 * Return values (usually ignored):
3358 * NET_RX_SUCCESS: no congestion
3359 * NET_RX_DROP: packet was dropped
3360 */
3361int netif_receive_skb(struct sk_buff *skb)
3362{
Eric Dumazet588f0332011-11-15 04:12:55 +00003363 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003364
Richard Cochranc1f19b52010-07-17 08:49:36 +00003365 if (skb_defer_rx_timestamp(skb))
3366 return NET_RX_SUCCESS;
3367
Eric Dumazetdf334542010-03-24 19:13:54 +00003368#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003369 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003370 struct rps_dev_flow voidflow, *rflow = &voidflow;
3371 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003372
Eric Dumazet3b098e22010-05-15 23:57:10 -07003373 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003374
Eric Dumazet3b098e22010-05-15 23:57:10 -07003375 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003376
Eric Dumazet3b098e22010-05-15 23:57:10 -07003377 if (cpu >= 0) {
3378 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3379 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003380 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003381 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003382 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003383 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003384#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003385 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003386}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003387EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003388
Eric Dumazet88751272010-04-19 05:07:33 +00003389/* Network device is going away, flush any packets still pending
3390 * Called with irqs disabled.
3391 */
Changli Gao152102c2010-03-30 20:16:22 +00003392static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003393{
Changli Gao152102c2010-03-30 20:16:22 +00003394 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003395 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003396 struct sk_buff *skb, *tmp;
3397
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003398 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003399 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003400 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003401 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003402 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003403 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003404 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003405 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003406 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003407
3408 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3409 if (skb->dev == dev) {
3410 __skb_unlink(skb, &sd->process_queue);
3411 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003412 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003413 }
3414 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003415}
3416
Herbert Xud565b0a2008-12-15 23:38:52 -08003417static int napi_gro_complete(struct sk_buff *skb)
3418{
3419 struct packet_type *ptype;
3420 __be16 type = skb->protocol;
3421 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3422 int err = -ENOENT;
3423
Herbert Xufc59f9a2009-04-14 15:11:06 -07003424 if (NAPI_GRO_CB(skb)->count == 1) {
3425 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003426 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003427 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003428
3429 rcu_read_lock();
3430 list_for_each_entry_rcu(ptype, head, list) {
3431 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3432 continue;
3433
3434 err = ptype->gro_complete(skb);
3435 break;
3436 }
3437 rcu_read_unlock();
3438
3439 if (err) {
3440 WARN_ON(&ptype->list == head);
3441 kfree_skb(skb);
3442 return NET_RX_SUCCESS;
3443 }
3444
3445out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003446 return netif_receive_skb(skb);
3447}
3448
Eric Dumazet86cac582010-08-31 18:25:32 +00003449inline void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08003450{
3451 struct sk_buff *skb, *next;
3452
3453 for (skb = napi->gro_list; skb; skb = next) {
3454 next = skb->next;
3455 skb->next = NULL;
3456 napi_gro_complete(skb);
3457 }
3458
Herbert Xu4ae55442009-02-08 18:00:36 +00003459 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003460 napi->gro_list = NULL;
3461}
Eric Dumazet86cac582010-08-31 18:25:32 +00003462EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003463
Ben Hutchings5b252f02009-10-29 07:17:09 +00003464enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003465{
3466 struct sk_buff **pp = NULL;
3467 struct packet_type *ptype;
3468 __be16 type = skb->protocol;
3469 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08003470 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003471 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003472 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003473
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003474 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003475 goto normal;
3476
David S. Miller21dc3302010-08-23 00:13:46 -07003477 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003478 goto normal;
3479
Herbert Xud565b0a2008-12-15 23:38:52 -08003480 rcu_read_lock();
3481 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08003482 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3483 continue;
3484
Herbert Xu86911732009-01-29 14:19:50 +00003485 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003486 mac_len = skb->network_header - skb->mac_header;
3487 skb->mac_len = mac_len;
3488 NAPI_GRO_CB(skb)->same_flow = 0;
3489 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003490 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003491
Herbert Xud565b0a2008-12-15 23:38:52 -08003492 pp = ptype->gro_receive(&napi->gro_list, skb);
3493 break;
3494 }
3495 rcu_read_unlock();
3496
3497 if (&ptype->list == head)
3498 goto normal;
3499
Herbert Xu0da2afd52008-12-26 14:57:42 -08003500 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003501 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003502
Herbert Xud565b0a2008-12-15 23:38:52 -08003503 if (pp) {
3504 struct sk_buff *nskb = *pp;
3505
3506 *pp = nskb->next;
3507 nskb->next = NULL;
3508 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003509 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003510 }
3511
Herbert Xu0da2afd52008-12-26 14:57:42 -08003512 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003513 goto ok;
3514
Herbert Xu4ae55442009-02-08 18:00:36 +00003515 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003516 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003517
Herbert Xu4ae55442009-02-08 18:00:36 +00003518 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003519 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00003520 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003521 skb->next = napi->gro_list;
3522 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003523 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003524
Herbert Xuad0f9902009-02-01 01:24:55 -08003525pull:
Herbert Xucb189782009-05-26 18:50:31 +00003526 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3527 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3528
3529 BUG_ON(skb->end - skb->tail < grow);
3530
3531 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3532
3533 skb->tail += grow;
3534 skb->data_len -= grow;
3535
3536 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003537 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003538
Eric Dumazet9e903e02011-10-18 21:00:24 +00003539 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003540 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003541 memmove(skb_shinfo(skb)->frags,
3542 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003543 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003544 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003545 }
3546
Herbert Xud565b0a2008-12-15 23:38:52 -08003547ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003548 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003549
3550normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003551 ret = GRO_NORMAL;
3552 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003553}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003554EXPORT_SYMBOL(dev_gro_receive);
3555
Eric Dumazet40d08022010-08-26 22:03:08 -07003556static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003557__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003558{
3559 struct sk_buff *p;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003560 unsigned int maclen = skb->dev->hard_header_len;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003561
3562 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003563 unsigned long diffs;
3564
3565 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003566 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003567 if (maclen == ETH_HLEN)
3568 diffs |= compare_ether_header(skb_mac_header(p),
3569 skb_gro_mac_header(skb));
3570 else if (!diffs)
3571 diffs = memcmp(skb_mac_header(p),
3572 skb_gro_mac_header(skb),
3573 maclen);
Eric Dumazet40d08022010-08-26 22:03:08 -07003574 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003575 NAPI_GRO_CB(p)->flush = 0;
3576 }
3577
3578 return dev_gro_receive(napi, skb);
3579}
Herbert Xu5d38a072009-01-04 16:13:40 -08003580
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003581gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003582{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003583 switch (ret) {
3584 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003585 if (netif_receive_skb(skb))
3586 ret = GRO_DROP;
3587 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003588
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003589 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08003590 kfree_skb(skb);
3591 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003592
Eric Dumazetdaa86542012-04-19 07:07:40 +00003593 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00003594 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3595 kmem_cache_free(skbuff_head_cache, skb);
3596 else
3597 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00003598 break;
3599
Ben Hutchings5b252f02009-10-29 07:17:09 +00003600 case GRO_HELD:
3601 case GRO_MERGED:
3602 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003603 }
3604
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003605 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003606}
3607EXPORT_SYMBOL(napi_skb_finish);
3608
Herbert Xu78a478d2009-05-26 18:50:21 +00003609void skb_gro_reset_offset(struct sk_buff *skb)
3610{
3611 NAPI_GRO_CB(skb)->data_offset = 0;
3612 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003613 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003614
Herbert Xu78d3fd02009-05-26 18:50:23 +00003615 if (skb->mac_header == skb->tail &&
Ian Campbellea2ab692011-08-22 23:44:58 +00003616 !PageHighMem(skb_frag_page(&skb_shinfo(skb)->frags[0]))) {
Herbert Xu78a478d2009-05-26 18:50:21 +00003617 NAPI_GRO_CB(skb)->frag0 =
Ian Campbellea2ab692011-08-22 23:44:58 +00003618 skb_frag_address(&skb_shinfo(skb)->frags[0]);
Eric Dumazet9e903e02011-10-18 21:00:24 +00003619 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(&skb_shinfo(skb)->frags[0]);
Herbert Xu74895942009-05-26 18:50:27 +00003620 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003621}
3622EXPORT_SYMBOL(skb_gro_reset_offset);
3623
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003624gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003625{
Herbert Xu86911732009-01-29 14:19:50 +00003626 skb_gro_reset_offset(skb);
3627
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003628 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003629}
3630EXPORT_SYMBOL(napi_gro_receive);
3631
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003632static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003633{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003634 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00003635 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3636 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003637 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003638 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003639 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003640
3641 napi->skb = skb;
3642}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003643
Herbert Xu76620aa2009-04-16 02:02:07 -07003644struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003645{
Herbert Xu5d38a072009-01-04 16:13:40 -08003646 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003647
3648 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003649 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3650 if (skb)
3651 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003652 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003653 return skb;
3654}
Herbert Xu76620aa2009-04-16 02:02:07 -07003655EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003656
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003657gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3658 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003659{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003660 switch (ret) {
3661 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003662 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003663 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003664
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003665 if (ret == GRO_HELD)
3666 skb_gro_pull(skb, -ETH_HLEN);
3667 else if (netif_receive_skb(skb))
3668 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003669 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003670
3671 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003672 case GRO_MERGED_FREE:
3673 napi_reuse_skb(napi, skb);
3674 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003675
3676 case GRO_MERGED:
3677 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003678 }
3679
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003680 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003681}
3682EXPORT_SYMBOL(napi_frags_finish);
3683
Eric Dumazet4adb9c42012-05-18 20:49:06 +00003684static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003685{
Herbert Xu76620aa2009-04-16 02:02:07 -07003686 struct sk_buff *skb = napi->skb;
3687 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003688 unsigned int hlen;
3689 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003690
3691 napi->skb = NULL;
3692
3693 skb_reset_mac_header(skb);
3694 skb_gro_reset_offset(skb);
3695
Herbert Xua5b1cf22009-05-26 18:50:28 +00003696 off = skb_gro_offset(skb);
3697 hlen = off + sizeof(*eth);
3698 eth = skb_gro_header_fast(skb, off);
3699 if (skb_gro_header_hard(skb, hlen)) {
3700 eth = skb_gro_header_slow(skb, hlen, off);
3701 if (unlikely(!eth)) {
3702 napi_reuse_skb(napi, skb);
3703 skb = NULL;
3704 goto out;
3705 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003706 }
3707
3708 skb_gro_pull(skb, sizeof(*eth));
3709
3710 /*
3711 * This works because the only protocols we care about don't require
3712 * special handling. We'll fix it up properly at the end.
3713 */
3714 skb->protocol = eth->h_proto;
3715
3716out:
3717 return skb;
3718}
Herbert Xu76620aa2009-04-16 02:02:07 -07003719
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003720gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003721{
3722 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003723
3724 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003725 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003726
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003727 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003728}
3729EXPORT_SYMBOL(napi_gro_frags);
3730
Eric Dumazete326bed2010-04-22 00:22:45 -07003731/*
3732 * net_rps_action sends any pending IPI's for rps.
3733 * Note: called with local irq disabled, but exits with local irq enabled.
3734 */
3735static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3736{
3737#ifdef CONFIG_RPS
3738 struct softnet_data *remsd = sd->rps_ipi_list;
3739
3740 if (remsd) {
3741 sd->rps_ipi_list = NULL;
3742
3743 local_irq_enable();
3744
3745 /* Send pending IPI's to kick RPS processing on remote cpus. */
3746 while (remsd) {
3747 struct softnet_data *next = remsd->rps_ipi_next;
3748
3749 if (cpu_online(remsd->cpu))
3750 __smp_call_function_single(remsd->cpu,
3751 &remsd->csd, 0);
3752 remsd = next;
3753 }
3754 } else
3755#endif
3756 local_irq_enable();
3757}
3758
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003759static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003760{
3761 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003762 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003763
Eric Dumazete326bed2010-04-22 00:22:45 -07003764#ifdef CONFIG_RPS
3765 /* Check if we have pending ipi, its better to send them now,
3766 * not waiting net_rx_action() end.
3767 */
3768 if (sd->rps_ipi_list) {
3769 local_irq_disable();
3770 net_rps_action_and_irq_enable(sd);
3771 }
3772#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003773 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003774 local_irq_disable();
3775 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003776 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003777 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003778
Changli Gao6e7676c2010-04-27 15:07:33 -07003779 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003780 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003781 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003782 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003783 input_queue_head_incr(sd);
3784 if (++work >= quota) {
3785 local_irq_enable();
3786 return work;
3787 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003788 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003789
Changli Gao6e7676c2010-04-27 15:07:33 -07003790 rps_lock(sd);
3791 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003792 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003793 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3794 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003795
Changli Gao6e7676c2010-04-27 15:07:33 -07003796 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003797 /*
3798 * Inline a custom version of __napi_complete().
3799 * only current cpu owns and manipulates this napi,
3800 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3801 * we can use a plain write instead of clear_bit(),
3802 * and we dont need an smp_mb() memory barrier.
3803 */
3804 list_del(&napi->poll_list);
3805 napi->state = 0;
3806
Changli Gao6e7676c2010-04-27 15:07:33 -07003807 quota = work + qlen;
3808 }
3809 rps_unlock(sd);
3810 }
3811 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003812
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003813 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003814}
3815
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003816/**
3817 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003818 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003819 *
3820 * The entry's receive function will be scheduled to run
3821 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003822void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003823{
3824 unsigned long flags;
3825
3826 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003827 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003828 local_irq_restore(flags);
3829}
3830EXPORT_SYMBOL(__napi_schedule);
3831
Herbert Xud565b0a2008-12-15 23:38:52 -08003832void __napi_complete(struct napi_struct *n)
3833{
3834 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3835 BUG_ON(n->gro_list);
3836
3837 list_del(&n->poll_list);
3838 smp_mb__before_clear_bit();
3839 clear_bit(NAPI_STATE_SCHED, &n->state);
3840}
3841EXPORT_SYMBOL(__napi_complete);
3842
3843void napi_complete(struct napi_struct *n)
3844{
3845 unsigned long flags;
3846
3847 /*
3848 * don't let napi dequeue from the cpu poll list
3849 * just in case its running on a different cpu
3850 */
3851 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3852 return;
3853
3854 napi_gro_flush(n);
3855 local_irq_save(flags);
3856 __napi_complete(n);
3857 local_irq_restore(flags);
3858}
3859EXPORT_SYMBOL(napi_complete);
3860
3861void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3862 int (*poll)(struct napi_struct *, int), int weight)
3863{
3864 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003865 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003866 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003867 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003868 napi->poll = poll;
3869 napi->weight = weight;
3870 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003871 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003872#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003873 spin_lock_init(&napi->poll_lock);
3874 napi->poll_owner = -1;
3875#endif
3876 set_bit(NAPI_STATE_SCHED, &napi->state);
3877}
3878EXPORT_SYMBOL(netif_napi_add);
3879
3880void netif_napi_del(struct napi_struct *napi)
3881{
3882 struct sk_buff *skb, *next;
3883
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003884 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003885 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003886
3887 for (skb = napi->gro_list; skb; skb = next) {
3888 next = skb->next;
3889 skb->next = NULL;
3890 kfree_skb(skb);
3891 }
3892
3893 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003894 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003895}
3896EXPORT_SYMBOL(netif_napi_del);
3897
Linus Torvalds1da177e2005-04-16 15:20:36 -07003898static void net_rx_action(struct softirq_action *h)
3899{
Eric Dumazete326bed2010-04-22 00:22:45 -07003900 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003901 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003902 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003903 void *have;
3904
Linus Torvalds1da177e2005-04-16 15:20:36 -07003905 local_irq_disable();
3906
Eric Dumazete326bed2010-04-22 00:22:45 -07003907 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003908 struct napi_struct *n;
3909 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003910
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003911 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003912 * Allow this to run for 2 jiffies since which will allow
3913 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003914 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003915 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003916 goto softnet_break;
3917
3918 local_irq_enable();
3919
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003920 /* Even though interrupts have been re-enabled, this
3921 * access is safe because interrupts can only add new
3922 * entries to the tail of this list, and only ->poll()
3923 * calls can remove this head entry from the list.
3924 */
Eric Dumazete326bed2010-04-22 00:22:45 -07003925 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003926
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003927 have = netpoll_poll_lock(n);
3928
3929 weight = n->weight;
3930
David S. Miller0a7606c2007-10-29 21:28:47 -07003931 /* This NAPI_STATE_SCHED test is for avoiding a race
3932 * with netpoll's poll_napi(). Only the entity which
3933 * obtains the lock and sees NAPI_STATE_SCHED set will
3934 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003935 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07003936 */
3937 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003938 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003939 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003940 trace_napi_poll(n);
3941 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003942
3943 WARN_ON_ONCE(work > weight);
3944
3945 budget -= work;
3946
3947 local_irq_disable();
3948
3949 /* Drivers must not modify the NAPI state if they
3950 * consume the entire weight. In such cases this code
3951 * still "owns" the NAPI instance and therefore can
3952 * move the instance around on the list at-will.
3953 */
David S. Millerfed17f32008-01-07 21:00:40 -08003954 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003955 if (unlikely(napi_disable_pending(n))) {
3956 local_irq_enable();
3957 napi_complete(n);
3958 local_irq_disable();
3959 } else
Eric Dumazete326bed2010-04-22 00:22:45 -07003960 list_move_tail(&n->poll_list, &sd->poll_list);
David S. Millerfed17f32008-01-07 21:00:40 -08003961 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003962
3963 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003964 }
3965out:
Eric Dumazete326bed2010-04-22 00:22:45 -07003966 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003967
Chris Leechdb217332006-06-17 21:24:58 -07003968#ifdef CONFIG_NET_DMA
3969 /*
3970 * There may not be any more sk_buffs coming right now, so push
3971 * any pending DMA copies to hardware
3972 */
Dan Williams2ba05622009-01-06 11:38:14 -07003973 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003974#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003975
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976 return;
3977
3978softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00003979 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003980 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3981 goto out;
3982}
3983
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003984static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003985
3986/**
3987 * register_gifconf - register a SIOCGIF handler
3988 * @family: Address family
3989 * @gifconf: Function handler
3990 *
3991 * Register protocol dependent address dumping routines. The handler
3992 * that is passed must not be freed or reused until it has been replaced
3993 * by another handler.
3994 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003995int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003996{
3997 if (family >= NPROTO)
3998 return -EINVAL;
3999 gifconf_list[family] = gifconf;
4000 return 0;
4001}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004002EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004003
4004
4005/*
4006 * Map an interface index to its name (SIOCGIFNAME)
4007 */
4008
4009/*
4010 * We need this ioctl for efficient implementation of the
4011 * if_indextoname() function required by the IPv6 API. Without
4012 * it, we would have to search all the interfaces to find a
4013 * match. --pb
4014 */
4015
Eric W. Biederman881d9662007-09-17 11:56:21 -07004016static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004017{
4018 struct net_device *dev;
4019 struct ifreq ifr;
4020
4021 /*
4022 * Fetch the caller's info block.
4023 */
4024
4025 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4026 return -EFAULT;
4027
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004028 rcu_read_lock();
4029 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004030 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004031 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004032 return -ENODEV;
4033 }
4034
4035 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004036 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004037
4038 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
4039 return -EFAULT;
4040 return 0;
4041}
4042
4043/*
4044 * Perform a SIOCGIFCONF call. This structure will change
4045 * size eventually, and there is nothing I can do about it.
4046 * Thus we will need a 'compatibility mode'.
4047 */
4048
Eric W. Biederman881d9662007-09-17 11:56:21 -07004049static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004050{
4051 struct ifconf ifc;
4052 struct net_device *dev;
4053 char __user *pos;
4054 int len;
4055 int total;
4056 int i;
4057
4058 /*
4059 * Fetch the caller's info block.
4060 */
4061
4062 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
4063 return -EFAULT;
4064
4065 pos = ifc.ifc_buf;
4066 len = ifc.ifc_len;
4067
4068 /*
4069 * Loop over the interfaces, and write an info block for each.
4070 */
4071
4072 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004073 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004074 for (i = 0; i < NPROTO; i++) {
4075 if (gifconf_list[i]) {
4076 int done;
4077 if (!pos)
4078 done = gifconf_list[i](dev, NULL, 0);
4079 else
4080 done = gifconf_list[i](dev, pos + total,
4081 len - total);
4082 if (done < 0)
4083 return -EFAULT;
4084 total += done;
4085 }
4086 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004087 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004088
4089 /*
4090 * All done. Write the updated control block back to the caller.
4091 */
4092 ifc.ifc_len = total;
4093
4094 /*
4095 * Both BSD and Solaris return 0 here, so we do too.
4096 */
4097 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
4098}
4099
4100#ifdef CONFIG_PROC_FS
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004101
Eric Dumazet2def16a2012-04-02 22:33:02 +00004102#define BUCKET_SPACE (32 - NETDEV_HASHBITS - 1)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004103
4104#define get_bucket(x) ((x) >> BUCKET_SPACE)
4105#define get_offset(x) ((x) & ((1 << BUCKET_SPACE) - 1))
4106#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))
4107
Eric Dumazet2def16a2012-04-02 22:33:02 +00004108static inline struct net_device *dev_from_same_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004109{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004110 struct net *net = seq_file_net(seq);
4111 struct net_device *dev;
4112 struct hlist_node *p;
4113 struct hlist_head *h;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004114 unsigned int count = 0, offset = get_offset(*pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004115
Eric Dumazet2def16a2012-04-02 22:33:02 +00004116 h = &net->dev_name_head[get_bucket(*pos)];
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004117 hlist_for_each_entry_rcu(dev, p, h, name_hlist) {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004118 if (++count == offset)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004119 return dev;
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004120 }
4121
4122 return NULL;
4123}
4124
Eric Dumazet2def16a2012-04-02 22:33:02 +00004125static inline struct net_device *dev_from_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004126{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004127 struct net_device *dev;
4128 unsigned int bucket;
4129
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004130 do {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004131 dev = dev_from_same_bucket(seq, pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004132 if (dev)
4133 return dev;
4134
Eric Dumazet2def16a2012-04-02 22:33:02 +00004135 bucket = get_bucket(*pos) + 1;
4136 *pos = set_bucket_offset(bucket, 1);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004137 } while (bucket < NETDEV_HASHENTRIES);
4138
4139 return NULL;
4140}
4141
Linus Torvalds1da177e2005-04-16 15:20:36 -07004142/*
4143 * This is invoked by the /proc filesystem handler to display a device
4144 * in detail.
4145 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004146void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004147 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004148{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004149 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07004150 if (!*pos)
4151 return SEQ_START_TOKEN;
4152
Eric Dumazet2def16a2012-04-02 22:33:02 +00004153 if (get_bucket(*pos) >= NETDEV_HASHENTRIES)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004154 return NULL;
Pavel Emelianov7562f872007-05-03 15:13:45 -07004155
Eric Dumazet2def16a2012-04-02 22:33:02 +00004156 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004157}
4158
4159void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4160{
4161 ++*pos;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004162 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004163}
4164
4165void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004166 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004167{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004168 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004169}
4170
4171static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4172{
Eric Dumazet28172732010-07-07 14:58:56 -07004173 struct rtnl_link_stats64 temp;
4174 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004175
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004176 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4177 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004178 dev->name, stats->rx_bytes, stats->rx_packets,
4179 stats->rx_errors,
4180 stats->rx_dropped + stats->rx_missed_errors,
4181 stats->rx_fifo_errors,
4182 stats->rx_length_errors + stats->rx_over_errors +
4183 stats->rx_crc_errors + stats->rx_frame_errors,
4184 stats->rx_compressed, stats->multicast,
4185 stats->tx_bytes, stats->tx_packets,
4186 stats->tx_errors, stats->tx_dropped,
4187 stats->tx_fifo_errors, stats->collisions,
4188 stats->tx_carrier_errors +
4189 stats->tx_aborted_errors +
4190 stats->tx_window_errors +
4191 stats->tx_heartbeat_errors,
4192 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004193}
4194
4195/*
4196 * Called from the PROCfs module. This now uses the new arbitrary sized
4197 * /proc/net interface to create /proc/net/dev
4198 */
4199static int dev_seq_show(struct seq_file *seq, void *v)
4200{
4201 if (v == SEQ_START_TOKEN)
4202 seq_puts(seq, "Inter-| Receive "
4203 " | Transmit\n"
4204 " face |bytes packets errs drop fifo frame "
4205 "compressed multicast|bytes packets errs "
4206 "drop fifo colls carrier compressed\n");
4207 else
4208 dev_seq_printf_stats(seq, v);
4209 return 0;
4210}
4211
Changli Gaodee42872010-05-02 05:42:16 +00004212static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004213{
Changli Gaodee42872010-05-02 05:42:16 +00004214 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004215
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004216 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004217 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004218 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004219 break;
4220 } else
4221 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004222 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004223}
4224
4225static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4226{
4227 return softnet_get_online(pos);
4228}
4229
4230static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4231{
4232 ++*pos;
4233 return softnet_get_online(pos);
4234}
4235
4236static void softnet_seq_stop(struct seq_file *seq, void *v)
4237{
4238}
4239
4240static int softnet_seq_show(struct seq_file *seq, void *v)
4241{
Changli Gaodee42872010-05-02 05:42:16 +00004242 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004243
Tom Herbert0a9627f2010-03-16 08:03:29 +00004244 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004245 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004246 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004247 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004248 return 0;
4249}
4250
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004251static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004252 .start = dev_seq_start,
4253 .next = dev_seq_next,
4254 .stop = dev_seq_stop,
4255 .show = dev_seq_show,
4256};
4257
4258static int dev_seq_open(struct inode *inode, struct file *file)
4259{
Denis V. Luneve372c412007-11-19 22:31:54 -08004260 return seq_open_net(inode, file, &dev_seq_ops,
Eric Dumazet2def16a2012-04-02 22:33:02 +00004261 sizeof(struct seq_net_private));
Anton Blanchard5cac98d2011-11-27 21:14:46 +00004262}
4263
Arjan van de Ven9a321442007-02-12 00:55:35 -08004264static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004265 .owner = THIS_MODULE,
4266 .open = dev_seq_open,
4267 .read = seq_read,
4268 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004269 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004270};
4271
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004272static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004273 .start = softnet_seq_start,
4274 .next = softnet_seq_next,
4275 .stop = softnet_seq_stop,
4276 .show = softnet_seq_show,
4277};
4278
4279static int softnet_seq_open(struct inode *inode, struct file *file)
4280{
4281 return seq_open(file, &softnet_seq_ops);
4282}
4283
Arjan van de Ven9a321442007-02-12 00:55:35 -08004284static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004285 .owner = THIS_MODULE,
4286 .open = softnet_seq_open,
4287 .read = seq_read,
4288 .llseek = seq_lseek,
4289 .release = seq_release,
4290};
4291
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004292static void *ptype_get_idx(loff_t pos)
4293{
4294 struct packet_type *pt = NULL;
4295 loff_t i = 0;
4296 int t;
4297
4298 list_for_each_entry_rcu(pt, &ptype_all, list) {
4299 if (i == pos)
4300 return pt;
4301 ++i;
4302 }
4303
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004304 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004305 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4306 if (i == pos)
4307 return pt;
4308 ++i;
4309 }
4310 }
4311 return NULL;
4312}
4313
4314static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004315 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004316{
4317 rcu_read_lock();
4318 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4319}
4320
4321static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4322{
4323 struct packet_type *pt;
4324 struct list_head *nxt;
4325 int hash;
4326
4327 ++*pos;
4328 if (v == SEQ_START_TOKEN)
4329 return ptype_get_idx(0);
4330
4331 pt = v;
4332 nxt = pt->list.next;
4333 if (pt->type == htons(ETH_P_ALL)) {
4334 if (nxt != &ptype_all)
4335 goto found;
4336 hash = 0;
4337 nxt = ptype_base[0].next;
4338 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004339 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004340
4341 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004342 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004343 return NULL;
4344 nxt = ptype_base[hash].next;
4345 }
4346found:
4347 return list_entry(nxt, struct packet_type, list);
4348}
4349
4350static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004351 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004352{
4353 rcu_read_unlock();
4354}
4355
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004356static int ptype_seq_show(struct seq_file *seq, void *v)
4357{
4358 struct packet_type *pt = v;
4359
4360 if (v == SEQ_START_TOKEN)
4361 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004362 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004363 if (pt->type == htons(ETH_P_ALL))
4364 seq_puts(seq, "ALL ");
4365 else
4366 seq_printf(seq, "%04x", ntohs(pt->type));
4367
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004368 seq_printf(seq, " %-8s %pF\n",
4369 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004370 }
4371
4372 return 0;
4373}
4374
4375static const struct seq_operations ptype_seq_ops = {
4376 .start = ptype_seq_start,
4377 .next = ptype_seq_next,
4378 .stop = ptype_seq_stop,
4379 .show = ptype_seq_show,
4380};
4381
4382static int ptype_seq_open(struct inode *inode, struct file *file)
4383{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004384 return seq_open_net(inode, file, &ptype_seq_ops,
4385 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004386}
4387
4388static const struct file_operations ptype_seq_fops = {
4389 .owner = THIS_MODULE,
4390 .open = ptype_seq_open,
4391 .read = seq_read,
4392 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004393 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004394};
4395
4396
Pavel Emelyanov46650792007-10-08 20:38:39 -07004397static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004398{
4399 int rc = -ENOMEM;
4400
Eric W. Biederman881d9662007-09-17 11:56:21 -07004401 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004402 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004403 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004404 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004405 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004406 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004407
Eric W. Biederman881d9662007-09-17 11:56:21 -07004408 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004409 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004410 rc = 0;
4411out:
4412 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004413out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004414 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004415out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004416 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004417out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004418 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004419 goto out;
4420}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004421
Pavel Emelyanov46650792007-10-08 20:38:39 -07004422static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004423{
4424 wext_proc_exit(net);
4425
4426 proc_net_remove(net, "ptype");
4427 proc_net_remove(net, "softnet_stat");
4428 proc_net_remove(net, "dev");
4429}
4430
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004431static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004432 .init = dev_proc_net_init,
4433 .exit = dev_proc_net_exit,
4434};
4435
4436static int __init dev_proc_init(void)
4437{
4438 return register_pernet_subsys(&dev_proc_ops);
4439}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004440#else
4441#define dev_proc_init() 0
4442#endif /* CONFIG_PROC_FS */
4443
4444
4445/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004446 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004447 * @slave: slave device
4448 * @master: new master device
4449 *
4450 * Changes the master device of the slave. Pass %NULL to break the
4451 * bonding. The caller must hold the RTNL semaphore. On a failure
4452 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004453 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004454 */
4455int netdev_set_master(struct net_device *slave, struct net_device *master)
4456{
4457 struct net_device *old = slave->master;
4458
4459 ASSERT_RTNL();
4460
4461 if (master) {
4462 if (old)
4463 return -EBUSY;
4464 dev_hold(master);
4465 }
4466
4467 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004468
Eric Dumazet6df427f2011-05-19 19:37:40 +00004469 if (old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004470 dev_put(old);
Jiri Pirko1765a572011-02-12 06:48:36 +00004471 return 0;
4472}
4473EXPORT_SYMBOL(netdev_set_master);
4474
4475/**
4476 * netdev_set_bond_master - set up bonding master/slave pair
4477 * @slave: slave device
4478 * @master: new master device
4479 *
4480 * Changes the master device of the slave. Pass %NULL to break the
4481 * bonding. The caller must hold the RTNL semaphore. On a failure
4482 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4483 * to the routing socket and the function returns zero.
4484 */
4485int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4486{
4487 int err;
4488
4489 ASSERT_RTNL();
4490
4491 err = netdev_set_master(slave, master);
4492 if (err)
4493 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004494 if (master)
4495 slave->flags |= IFF_SLAVE;
4496 else
4497 slave->flags &= ~IFF_SLAVE;
4498
4499 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4500 return 0;
4501}
Jiri Pirko1765a572011-02-12 06:48:36 +00004502EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004503
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004504static void dev_change_rx_flags(struct net_device *dev, int flags)
4505{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004506 const struct net_device_ops *ops = dev->netdev_ops;
4507
4508 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4509 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004510}
4511
Wang Chendad9b332008-06-18 01:48:28 -07004512static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004513{
Eric Dumazetb536db92011-11-30 21:42:26 +00004514 unsigned int old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11004515 uid_t uid;
4516 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004517
Patrick McHardy24023452007-07-14 18:51:31 -07004518 ASSERT_RTNL();
4519
Wang Chendad9b332008-06-18 01:48:28 -07004520 dev->flags |= IFF_PROMISC;
4521 dev->promiscuity += inc;
4522 if (dev->promiscuity == 0) {
4523 /*
4524 * Avoid overflow.
4525 * If inc causes overflow, untouch promisc and return error.
4526 */
4527 if (inc < 0)
4528 dev->flags &= ~IFF_PROMISC;
4529 else {
4530 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004531 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4532 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004533 return -EOVERFLOW;
4534 }
4535 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004536 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004537 pr_info("device %s %s promiscuous mode\n",
4538 dev->name,
4539 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11004540 if (audit_enabled) {
4541 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004542 audit_log(current->audit_context, GFP_ATOMIC,
4543 AUDIT_ANOM_PROMISCUOUS,
4544 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4545 dev->name, (dev->flags & IFF_PROMISC),
4546 (old_flags & IFF_PROMISC),
4547 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11004548 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004549 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004550 }
Patrick McHardy24023452007-07-14 18:51:31 -07004551
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004552 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004553 }
Wang Chendad9b332008-06-18 01:48:28 -07004554 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004555}
4556
Linus Torvalds1da177e2005-04-16 15:20:36 -07004557/**
4558 * dev_set_promiscuity - update promiscuity count on a device
4559 * @dev: device
4560 * @inc: modifier
4561 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004562 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004563 * remains above zero the interface remains promiscuous. Once it hits zero
4564 * the device reverts back to normal filtering operation. A negative inc
4565 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004566 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004567 */
Wang Chendad9b332008-06-18 01:48:28 -07004568int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004569{
Eric Dumazetb536db92011-11-30 21:42:26 +00004570 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004571 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004572
Wang Chendad9b332008-06-18 01:48:28 -07004573 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004574 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004575 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004576 if (dev->flags != old_flags)
4577 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004578 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004579}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004580EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004581
4582/**
4583 * dev_set_allmulti - update allmulti count on a device
4584 * @dev: device
4585 * @inc: modifier
4586 *
4587 * Add or remove reception of all multicast frames to a device. While the
4588 * count in the device remains above zero the interface remains listening
4589 * to all interfaces. Once it hits zero the device reverts back to normal
4590 * filtering operation. A negative @inc value is used to drop the counter
4591 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004592 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004593 */
4594
Wang Chendad9b332008-06-18 01:48:28 -07004595int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004596{
Eric Dumazetb536db92011-11-30 21:42:26 +00004597 unsigned int old_flags = dev->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004598
Patrick McHardy24023452007-07-14 18:51:31 -07004599 ASSERT_RTNL();
4600
Linus Torvalds1da177e2005-04-16 15:20:36 -07004601 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004602 dev->allmulti += inc;
4603 if (dev->allmulti == 0) {
4604 /*
4605 * Avoid overflow.
4606 * If inc causes overflow, untouch allmulti and return error.
4607 */
4608 if (inc < 0)
4609 dev->flags &= ~IFF_ALLMULTI;
4610 else {
4611 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004612 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4613 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004614 return -EOVERFLOW;
4615 }
4616 }
Patrick McHardy24023452007-07-14 18:51:31 -07004617 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004618 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004619 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004620 }
Wang Chendad9b332008-06-18 01:48:28 -07004621 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004622}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004623EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004624
4625/*
4626 * Upload unicast and multicast address lists to device and
4627 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004628 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004629 * are present.
4630 */
4631void __dev_set_rx_mode(struct net_device *dev)
4632{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004633 const struct net_device_ops *ops = dev->netdev_ops;
4634
Patrick McHardy4417da62007-06-27 01:28:10 -07004635 /* dev_open will call this function so the list will stay sane. */
4636 if (!(dev->flags&IFF_UP))
4637 return;
4638
4639 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004640 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004641
Jiri Pirko01789342011-08-16 06:29:00 +00004642 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004643 /* Unicast addresses changes may only happen under the rtnl,
4644 * therefore calling __dev_set_promiscuity here is safe.
4645 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004646 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004647 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004648 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004649 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004650 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004651 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004652 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004653 }
Jiri Pirko01789342011-08-16 06:29:00 +00004654
4655 if (ops->ndo_set_rx_mode)
4656 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004657}
4658
4659void dev_set_rx_mode(struct net_device *dev)
4660{
David S. Millerb9e40852008-07-15 00:15:08 -07004661 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004662 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004663 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004664}
4665
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004666/**
4667 * dev_get_flags - get flags reported to userspace
4668 * @dev: device
4669 *
4670 * Get the combination of flag bits exported through APIs to userspace.
4671 */
Eric Dumazet95c96172012-04-15 05:58:06 +00004672unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004673{
Eric Dumazet95c96172012-04-15 05:58:06 +00004674 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004675
4676 flags = (dev->flags & ~(IFF_PROMISC |
4677 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004678 IFF_RUNNING |
4679 IFF_LOWER_UP |
4680 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004681 (dev->gflags & (IFF_PROMISC |
4682 IFF_ALLMULTI));
4683
Stefan Rompfb00055a2006-03-20 17:09:11 -08004684 if (netif_running(dev)) {
4685 if (netif_oper_up(dev))
4686 flags |= IFF_RUNNING;
4687 if (netif_carrier_ok(dev))
4688 flags |= IFF_LOWER_UP;
4689 if (netif_dormant(dev))
4690 flags |= IFF_DORMANT;
4691 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004692
4693 return flags;
4694}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004695EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004696
Patrick McHardybd380812010-02-26 06:34:53 +00004697int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004698{
Eric Dumazetb536db92011-11-30 21:42:26 +00004699 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004700 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004701
Patrick McHardy24023452007-07-14 18:51:31 -07004702 ASSERT_RTNL();
4703
Linus Torvalds1da177e2005-04-16 15:20:36 -07004704 /*
4705 * Set the flags on our device.
4706 */
4707
4708 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4709 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4710 IFF_AUTOMEDIA)) |
4711 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4712 IFF_ALLMULTI));
4713
4714 /*
4715 * Load in the correct multicast list now the flags have changed.
4716 */
4717
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004718 if ((old_flags ^ flags) & IFF_MULTICAST)
4719 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004720
Patrick McHardy4417da62007-06-27 01:28:10 -07004721 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004722
4723 /*
4724 * Have we downed the interface. We handle IFF_UP ourselves
4725 * according to user attempts to set it, rather than blindly
4726 * setting it.
4727 */
4728
4729 ret = 0;
4730 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004731 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004732
4733 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004734 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004735 }
4736
Linus Torvalds1da177e2005-04-16 15:20:36 -07004737 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004738 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4739
Linus Torvalds1da177e2005-04-16 15:20:36 -07004740 dev->gflags ^= IFF_PROMISC;
4741 dev_set_promiscuity(dev, inc);
4742 }
4743
4744 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4745 is important. Some (broken) drivers set IFF_PROMISC, when
4746 IFF_ALLMULTI is requested not asking us and not reporting.
4747 */
4748 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004749 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4750
Linus Torvalds1da177e2005-04-16 15:20:36 -07004751 dev->gflags ^= IFF_ALLMULTI;
4752 dev_set_allmulti(dev, inc);
4753 }
4754
Patrick McHardybd380812010-02-26 06:34:53 +00004755 return ret;
4756}
4757
4758void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4759{
4760 unsigned int changes = dev->flags ^ old_flags;
4761
4762 if (changes & IFF_UP) {
4763 if (dev->flags & IFF_UP)
4764 call_netdevice_notifiers(NETDEV_UP, dev);
4765 else
4766 call_netdevice_notifiers(NETDEV_DOWN, dev);
4767 }
4768
4769 if (dev->flags & IFF_UP &&
4770 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4771 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4772}
4773
4774/**
4775 * dev_change_flags - change device settings
4776 * @dev: device
4777 * @flags: device state flags
4778 *
4779 * Change settings on device based state flags. The flags are
4780 * in the userspace exported format.
4781 */
Eric Dumazetb536db92011-11-30 21:42:26 +00004782int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00004783{
Eric Dumazetb536db92011-11-30 21:42:26 +00004784 int ret;
4785 unsigned int changes, old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004786
4787 ret = __dev_change_flags(dev, flags);
4788 if (ret < 0)
4789 return ret;
4790
4791 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004792 if (changes)
4793 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004794
Patrick McHardybd380812010-02-26 06:34:53 +00004795 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004796 return ret;
4797}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004798EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004799
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004800/**
4801 * dev_set_mtu - Change maximum transfer unit
4802 * @dev: device
4803 * @new_mtu: new transfer unit
4804 *
4805 * Change the maximum transfer size of the network device.
4806 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004807int dev_set_mtu(struct net_device *dev, int new_mtu)
4808{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004809 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004810 int err;
4811
4812 if (new_mtu == dev->mtu)
4813 return 0;
4814
4815 /* MTU must be positive. */
4816 if (new_mtu < 0)
4817 return -EINVAL;
4818
4819 if (!netif_device_present(dev))
4820 return -ENODEV;
4821
4822 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004823 if (ops->ndo_change_mtu)
4824 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004825 else
4826 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004827
Linus Torvalds1da177e2005-04-16 15:20:36 -07004828 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004829 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004830 return err;
4831}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004832EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004833
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004834/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004835 * dev_set_group - Change group this device belongs to
4836 * @dev: device
4837 * @new_group: group this device should belong to
4838 */
4839void dev_set_group(struct net_device *dev, int new_group)
4840{
4841 dev->group = new_group;
4842}
4843EXPORT_SYMBOL(dev_set_group);
4844
4845/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004846 * dev_set_mac_address - Change Media Access Control Address
4847 * @dev: device
4848 * @sa: new address
4849 *
4850 * Change the hardware (MAC) address of the device
4851 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004852int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4853{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004854 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004855 int err;
4856
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004857 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004858 return -EOPNOTSUPP;
4859 if (sa->sa_family != dev->type)
4860 return -EINVAL;
4861 if (!netif_device_present(dev))
4862 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004863 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004864 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004865 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04004866 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004867 return err;
4868}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004869EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004870
4871/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004872 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004873 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004874static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004875{
4876 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004877 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004878
4879 if (!dev)
4880 return -ENODEV;
4881
4882 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004883 case SIOCGIFFLAGS: /* Get interface flags */
4884 ifr->ifr_flags = (short) dev_get_flags(dev);
4885 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004886
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004887 case SIOCGIFMETRIC: /* Get the metric on the interface
4888 (currently unused) */
4889 ifr->ifr_metric = 0;
4890 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004891
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004892 case SIOCGIFMTU: /* Get the MTU of a device */
4893 ifr->ifr_mtu = dev->mtu;
4894 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004895
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004896 case SIOCGIFHWADDR:
4897 if (!dev->addr_len)
4898 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4899 else
4900 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4901 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4902 ifr->ifr_hwaddr.sa_family = dev->type;
4903 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004904
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004905 case SIOCGIFSLAVE:
4906 err = -EINVAL;
4907 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004908
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004909 case SIOCGIFMAP:
4910 ifr->ifr_map.mem_start = dev->mem_start;
4911 ifr->ifr_map.mem_end = dev->mem_end;
4912 ifr->ifr_map.base_addr = dev->base_addr;
4913 ifr->ifr_map.irq = dev->irq;
4914 ifr->ifr_map.dma = dev->dma;
4915 ifr->ifr_map.port = dev->if_port;
4916 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004917
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004918 case SIOCGIFINDEX:
4919 ifr->ifr_ifindex = dev->ifindex;
4920 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004921
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004922 case SIOCGIFTXQLEN:
4923 ifr->ifr_qlen = dev->tx_queue_len;
4924 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004925
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004926 default:
4927 /* dev_ioctl() should ensure this case
4928 * is never reached
4929 */
4930 WARN_ON(1);
Lifeng Sun41c31f32011-04-27 22:04:51 +00004931 err = -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004932 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004933
4934 }
4935 return err;
4936}
4937
4938/*
4939 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4940 */
4941static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4942{
4943 int err;
4944 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004945 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004946
4947 if (!dev)
4948 return -ENODEV;
4949
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004950 ops = dev->netdev_ops;
4951
Jeff Garzik14e3e072007-10-08 00:06:32 -07004952 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004953 case SIOCSIFFLAGS: /* Set interface flags */
4954 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004955
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004956 case SIOCSIFMETRIC: /* Set the metric on the interface
4957 (currently unused) */
4958 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004959
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004960 case SIOCSIFMTU: /* Set the MTU of a device */
4961 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004962
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004963 case SIOCSIFHWADDR:
4964 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004965
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004966 case SIOCSIFHWBROADCAST:
4967 if (ifr->ifr_hwaddr.sa_family != dev->type)
4968 return -EINVAL;
4969 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4970 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4971 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4972 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004973
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004974 case SIOCSIFMAP:
4975 if (ops->ndo_set_config) {
4976 if (!netif_device_present(dev))
4977 return -ENODEV;
4978 return ops->ndo_set_config(dev, &ifr->ifr_map);
4979 }
4980 return -EOPNOTSUPP;
4981
4982 case SIOCADDMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00004983 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004984 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4985 return -EINVAL;
4986 if (!netif_device_present(dev))
4987 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00004988 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004989
4990 case SIOCDELMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00004991 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004992 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4993 return -EINVAL;
4994 if (!netif_device_present(dev))
4995 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00004996 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004997
4998 case SIOCSIFTXQLEN:
4999 if (ifr->ifr_qlen < 0)
5000 return -EINVAL;
5001 dev->tx_queue_len = ifr->ifr_qlen;
5002 return 0;
5003
5004 case SIOCSIFNAME:
5005 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
5006 return dev_change_name(dev, ifr->ifr_newname);
5007
Richard Cochran4dc360c2011-10-19 17:00:35 -04005008 case SIOCSHWTSTAMP:
5009 err = net_hwtstamp_validate(ifr);
5010 if (err)
5011 return err;
5012 /* fall through */
5013
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005014 /*
5015 * Unknown or private ioctl
5016 */
5017 default:
5018 if ((cmd >= SIOCDEVPRIVATE &&
5019 cmd <= SIOCDEVPRIVATE + 15) ||
5020 cmd == SIOCBONDENSLAVE ||
5021 cmd == SIOCBONDRELEASE ||
5022 cmd == SIOCBONDSETHWADDR ||
5023 cmd == SIOCBONDSLAVEINFOQUERY ||
5024 cmd == SIOCBONDINFOQUERY ||
5025 cmd == SIOCBONDCHANGEACTIVE ||
5026 cmd == SIOCGMIIPHY ||
5027 cmd == SIOCGMIIREG ||
5028 cmd == SIOCSMIIREG ||
5029 cmd == SIOCBRADDIF ||
5030 cmd == SIOCBRDELIF ||
5031 cmd == SIOCSHWTSTAMP ||
5032 cmd == SIOCWANDEV) {
5033 err = -EOPNOTSUPP;
5034 if (ops->ndo_do_ioctl) {
5035 if (netif_device_present(dev))
5036 err = ops->ndo_do_ioctl(dev, ifr, cmd);
5037 else
5038 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005039 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005040 } else
5041 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005042
5043 }
5044 return err;
5045}
5046
5047/*
5048 * This function handles all "interface"-type I/O control requests. The actual
5049 * 'doing' part of this is dev_ifsioc above.
5050 */
5051
5052/**
5053 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005054 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005055 * @cmd: command to issue
5056 * @arg: pointer to a struct ifreq in user space
5057 *
5058 * Issue ioctl functions to devices. This is normally called by the
5059 * user space syscall interfaces but can sometimes be useful for
5060 * other purposes. The return value is the return from the syscall if
5061 * positive or a negative errno code on error.
5062 */
5063
Eric W. Biederman881d9662007-09-17 11:56:21 -07005064int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005065{
5066 struct ifreq ifr;
5067 int ret;
5068 char *colon;
5069
5070 /* One special case: SIOCGIFCONF takes ifconf argument
5071 and requires shared lock, because it sleeps writing
5072 to user space.
5073 */
5074
5075 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005076 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005077 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005078 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005079 return ret;
5080 }
5081 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005082 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005083
5084 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
5085 return -EFAULT;
5086
5087 ifr.ifr_name[IFNAMSIZ-1] = 0;
5088
5089 colon = strchr(ifr.ifr_name, ':');
5090 if (colon)
5091 *colon = 0;
5092
5093 /*
5094 * See which interface the caller is talking about.
5095 */
5096
5097 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005098 /*
5099 * These ioctl calls:
5100 * - can be done by all.
5101 * - atomic and do not require locking.
5102 * - return a value
5103 */
5104 case SIOCGIFFLAGS:
5105 case SIOCGIFMETRIC:
5106 case SIOCGIFMTU:
5107 case SIOCGIFHWADDR:
5108 case SIOCGIFSLAVE:
5109 case SIOCGIFMAP:
5110 case SIOCGIFINDEX:
5111 case SIOCGIFTXQLEN:
5112 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005113 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005114 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005115 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005116 if (!ret) {
5117 if (colon)
5118 *colon = ':';
5119 if (copy_to_user(arg, &ifr,
5120 sizeof(struct ifreq)))
5121 ret = -EFAULT;
5122 }
5123 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005124
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005125 case SIOCETHTOOL:
5126 dev_load(net, ifr.ifr_name);
5127 rtnl_lock();
5128 ret = dev_ethtool(net, &ifr);
5129 rtnl_unlock();
5130 if (!ret) {
5131 if (colon)
5132 *colon = ':';
5133 if (copy_to_user(arg, &ifr,
5134 sizeof(struct ifreq)))
5135 ret = -EFAULT;
5136 }
5137 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005138
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005139 /*
5140 * These ioctl calls:
5141 * - require superuser power.
5142 * - require strict serialization.
5143 * - return a value
5144 */
5145 case SIOCGMIIPHY:
5146 case SIOCGMIIREG:
5147 case SIOCSIFNAME:
5148 if (!capable(CAP_NET_ADMIN))
5149 return -EPERM;
5150 dev_load(net, ifr.ifr_name);
5151 rtnl_lock();
5152 ret = dev_ifsioc(net, &ifr, cmd);
5153 rtnl_unlock();
5154 if (!ret) {
5155 if (colon)
5156 *colon = ':';
5157 if (copy_to_user(arg, &ifr,
5158 sizeof(struct ifreq)))
5159 ret = -EFAULT;
5160 }
5161 return ret;
5162
5163 /*
5164 * These ioctl calls:
5165 * - require superuser power.
5166 * - require strict serialization.
5167 * - do not return a value
5168 */
5169 case SIOCSIFFLAGS:
5170 case SIOCSIFMETRIC:
5171 case SIOCSIFMTU:
5172 case SIOCSIFMAP:
5173 case SIOCSIFHWADDR:
5174 case SIOCSIFSLAVE:
5175 case SIOCADDMULTI:
5176 case SIOCDELMULTI:
5177 case SIOCSIFHWBROADCAST:
5178 case SIOCSIFTXQLEN:
5179 case SIOCSMIIREG:
5180 case SIOCBONDENSLAVE:
5181 case SIOCBONDRELEASE:
5182 case SIOCBONDSETHWADDR:
5183 case SIOCBONDCHANGEACTIVE:
5184 case SIOCBRADDIF:
5185 case SIOCBRDELIF:
5186 case SIOCSHWTSTAMP:
5187 if (!capable(CAP_NET_ADMIN))
5188 return -EPERM;
5189 /* fall through */
5190 case SIOCBONDSLAVEINFOQUERY:
5191 case SIOCBONDINFOQUERY:
5192 dev_load(net, ifr.ifr_name);
5193 rtnl_lock();
5194 ret = dev_ifsioc(net, &ifr, cmd);
5195 rtnl_unlock();
5196 return ret;
5197
5198 case SIOCGIFMEM:
5199 /* Get the per device memory space. We can add this but
5200 * currently do not support it */
5201 case SIOCSIFMEM:
5202 /* Set the per device memory buffer space.
5203 * Not applicable in our case */
5204 case SIOCSIFLINK:
Lifeng Sun41c31f32011-04-27 22:04:51 +00005205 return -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005206
5207 /*
5208 * Unknown or private ioctl.
5209 */
5210 default:
5211 if (cmd == SIOCWANDEV ||
5212 (cmd >= SIOCDEVPRIVATE &&
5213 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005214 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005215 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005216 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005217 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005218 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005219 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005220 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005221 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005222 }
5223 /* Take care of Wireless Extensions */
5224 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5225 return wext_handle_ioctl(net, &ifr, cmd, arg);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005226 return -ENOTTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005227 }
5228}
5229
5230
5231/**
5232 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005233 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005234 *
5235 * Returns a suitable unique value for a new device interface
5236 * number. The caller must hold the rtnl semaphore or the
5237 * dev_base_lock to be sure it remains unique.
5238 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005239static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005240{
5241 static int ifindex;
5242 for (;;) {
5243 if (++ifindex <= 0)
5244 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005245 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005246 return ifindex;
5247 }
5248}
5249
Linus Torvalds1da177e2005-04-16 15:20:36 -07005250/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005251static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005252
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005253static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005254{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005255 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005256}
5257
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005258static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005259{
Krishna Kumare93737b2009-12-08 22:26:02 +00005260 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005261
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005262 BUG_ON(dev_boot_phase);
5263 ASSERT_RTNL();
5264
Krishna Kumare93737b2009-12-08 22:26:02 +00005265 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005266 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005267 * for initialization unwind. Remove those
5268 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005269 */
5270 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005271 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
5272 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005273
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005274 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005275 list_del(&dev->unreg_list);
5276 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005277 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005278 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005279 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005280 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005281
Octavian Purdila44345722010-12-13 12:44:07 +00005282 /* If device is running, close it first. */
5283 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005284
Octavian Purdila44345722010-12-13 12:44:07 +00005285 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005286 /* And unlink it from device chain. */
5287 unlist_netdevice(dev);
5288
5289 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005290 }
5291
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005292 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005293
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005294 list_for_each_entry(dev, head, unreg_list) {
5295 /* Shutdown queueing discipline. */
5296 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005297
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005298
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005299 /* Notify protocols, that we are about to destroy
5300 this device. They should clean all the things.
5301 */
5302 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5303
Patrick McHardya2835762010-02-26 06:34:51 +00005304 if (!dev->rtnl_link_ops ||
5305 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5306 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5307
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005308 /*
5309 * Flush the unicast and multicast chains
5310 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005311 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00005312 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005313
5314 if (dev->netdev_ops->ndo_uninit)
5315 dev->netdev_ops->ndo_uninit(dev);
5316
5317 /* Notifier chain MUST detach us from master device. */
5318 WARN_ON(dev->master);
5319
5320 /* Remove entries from kobject tree */
5321 netdev_unregister_kobject(dev);
5322 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005323
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005324 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00005325 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005326 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5327
Eric W. Biederman850a5452011-10-13 22:25:23 +00005328 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005329
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005330 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005331 dev_put(dev);
5332}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005333
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005334static void rollback_registered(struct net_device *dev)
5335{
5336 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005337
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005338 list_add(&dev->unreg_list, &single);
5339 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005340 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005341}
5342
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005343static netdev_features_t netdev_fix_features(struct net_device *dev,
5344 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07005345{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005346 /* Fix illegal checksum combinations */
5347 if ((features & NETIF_F_HW_CSUM) &&
5348 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005349 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005350 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5351 }
5352
Herbert Xub63365a2008-10-23 01:11:29 -07005353 /* Fix illegal SG+CSUM combinations. */
5354 if ((features & NETIF_F_SG) &&
5355 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005356 netdev_dbg(dev,
5357 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005358 features &= ~NETIF_F_SG;
5359 }
5360
5361 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005362 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005363 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005364 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005365 }
5366
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005367 /* TSO ECN requires that TSO is present as well. */
5368 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5369 features &= ~NETIF_F_TSO_ECN;
5370
Michał Mirosław212b5732011-02-15 16:59:16 +00005371 /* Software GSO depends on SG. */
5372 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005373 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005374 features &= ~NETIF_F_GSO;
5375 }
5376
Michał Mirosławacd11302011-01-24 15:45:15 -08005377 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005378 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005379 /* maybe split UFO into V4 and V6? */
5380 if (!((features & NETIF_F_GEN_CSUM) ||
5381 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5382 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005383 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005384 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005385 features &= ~NETIF_F_UFO;
5386 }
5387
5388 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005389 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005390 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005391 features &= ~NETIF_F_UFO;
5392 }
5393 }
5394
5395 return features;
5396}
Herbert Xub63365a2008-10-23 01:11:29 -07005397
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005398int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005399{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005400 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00005401 int err = 0;
5402
Michał Mirosław87267482011-04-12 09:56:38 +00005403 ASSERT_RTNL();
5404
Michał Mirosław5455c692011-02-15 16:59:17 +00005405 features = netdev_get_wanted_features(dev);
5406
5407 if (dev->netdev_ops->ndo_fix_features)
5408 features = dev->netdev_ops->ndo_fix_features(dev, features);
5409
5410 /* driver might be less strict about feature dependencies */
5411 features = netdev_fix_features(dev, features);
5412
5413 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005414 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005415
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005416 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5417 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00005418
5419 if (dev->netdev_ops->ndo_set_features)
5420 err = dev->netdev_ops->ndo_set_features(dev, features);
5421
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005422 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005423 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005424 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5425 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005426 return -1;
5427 }
5428
5429 if (!err)
5430 dev->features = features;
5431
5432 return 1;
5433}
5434
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005435/**
5436 * netdev_update_features - recalculate device features
5437 * @dev: the device to check
5438 *
5439 * Recalculate dev->features set and send notifications if it
5440 * has changed. Should be called after driver or hardware dependent
5441 * conditions might have changed that influence the features.
5442 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005443void netdev_update_features(struct net_device *dev)
5444{
5445 if (__netdev_update_features(dev))
5446 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005447}
5448EXPORT_SYMBOL(netdev_update_features);
5449
Linus Torvalds1da177e2005-04-16 15:20:36 -07005450/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005451 * netdev_change_features - recalculate device features
5452 * @dev: the device to check
5453 *
5454 * Recalculate dev->features set and send notifications even
5455 * if they have not changed. Should be called instead of
5456 * netdev_update_features() if also dev->vlan_features might
5457 * have changed to allow the changes to be propagated to stacked
5458 * VLAN devices.
5459 */
5460void netdev_change_features(struct net_device *dev)
5461{
5462 __netdev_update_features(dev);
5463 netdev_features_change(dev);
5464}
5465EXPORT_SYMBOL(netdev_change_features);
5466
5467/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005468 * netif_stacked_transfer_operstate - transfer operstate
5469 * @rootdev: the root or lower level device to transfer state from
5470 * @dev: the device to transfer operstate to
5471 *
5472 * Transfer operational state from root to device. This is normally
5473 * called when a stacking relationship exists between the root
5474 * device and the device(a leaf device).
5475 */
5476void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5477 struct net_device *dev)
5478{
5479 if (rootdev->operstate == IF_OPER_DORMANT)
5480 netif_dormant_on(dev);
5481 else
5482 netif_dormant_off(dev);
5483
5484 if (netif_carrier_ok(rootdev)) {
5485 if (!netif_carrier_ok(dev))
5486 netif_carrier_on(dev);
5487 } else {
5488 if (netif_carrier_ok(dev))
5489 netif_carrier_off(dev);
5490 }
5491}
5492EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5493
Tom Herbertbf264142010-11-26 08:36:09 +00005494#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005495static int netif_alloc_rx_queues(struct net_device *dev)
5496{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005497 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005498 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005499
Tom Herbertbd25fa72010-10-18 18:00:16 +00005500 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005501
Tom Herbertbd25fa72010-10-18 18:00:16 +00005502 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5503 if (!rx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005504 pr_err("netdev: Unable to allocate %u rx queues\n", count);
Tom Herbertbd25fa72010-10-18 18:00:16 +00005505 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005506 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005507 dev->_rx = rx;
5508
Tom Herbertbd25fa72010-10-18 18:00:16 +00005509 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005510 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005511 return 0;
5512}
Tom Herbertbf264142010-11-26 08:36:09 +00005513#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005514
Changli Gaoaa942102010-12-04 02:31:41 +00005515static void netdev_init_one_queue(struct net_device *dev,
5516 struct netdev_queue *queue, void *_unused)
5517{
5518 /* Initialize queue lock */
5519 spin_lock_init(&queue->_xmit_lock);
5520 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5521 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005522 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005523 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00005524#ifdef CONFIG_BQL
5525 dql_init(&queue->dql, HZ);
5526#endif
Changli Gaoaa942102010-12-04 02:31:41 +00005527}
5528
Tom Herberte6484932010-10-18 18:04:39 +00005529static int netif_alloc_netdev_queues(struct net_device *dev)
5530{
5531 unsigned int count = dev->num_tx_queues;
5532 struct netdev_queue *tx;
5533
5534 BUG_ON(count < 1);
5535
5536 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5537 if (!tx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005538 pr_err("netdev: Unable to allocate %u tx queues\n", count);
Tom Herberte6484932010-10-18 18:04:39 +00005539 return -ENOMEM;
5540 }
5541 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005542
Tom Herberte6484932010-10-18 18:04:39 +00005543 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5544 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005545
5546 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005547}
5548
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005549/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005550 * register_netdevice - register a network device
5551 * @dev: device to register
5552 *
5553 * Take a completed network device structure and add it to the kernel
5554 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5555 * chain. 0 is returned on success. A negative errno code is returned
5556 * on a failure to set up the device, or if the name is a duplicate.
5557 *
5558 * Callers must hold the rtnl semaphore. You may want
5559 * register_netdev() instead of this.
5560 *
5561 * BUGS:
5562 * The locking appears insufficient to guarantee two parallel registers
5563 * will not get the same name.
5564 */
5565
5566int register_netdevice(struct net_device *dev)
5567{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005568 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005569 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005570
5571 BUG_ON(dev_boot_phase);
5572 ASSERT_RTNL();
5573
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005574 might_sleep();
5575
Linus Torvalds1da177e2005-04-16 15:20:36 -07005576 /* When net_device's are persistent, this will be fatal. */
5577 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005578 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005579
David S. Millerf1f28aa2008-07-15 00:08:33 -07005580 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005581 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005582
Linus Torvalds1da177e2005-04-16 15:20:36 -07005583 dev->iflink = -1;
5584
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005585 ret = dev_get_valid_name(dev, dev->name);
5586 if (ret < 0)
5587 goto out;
5588
Linus Torvalds1da177e2005-04-16 15:20:36 -07005589 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005590 if (dev->netdev_ops->ndo_init) {
5591 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005592 if (ret) {
5593 if (ret > 0)
5594 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005595 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005596 }
5597 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005598
Eric W. Biederman881d9662007-09-17 11:56:21 -07005599 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005600 if (dev->iflink == -1)
5601 dev->iflink = dev->ifindex;
5602
Michał Mirosław5455c692011-02-15 16:59:17 +00005603 /* Transfer changeable features to wanted_features and enable
5604 * software offloads (GSO and GRO).
5605 */
5606 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005607 dev->features |= NETIF_F_SOFT_FEATURES;
5608 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005609
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005610 /* Turn on no cache copy if HW is doing checksum */
Michał Mirosław34324dc2011-11-15 15:29:55 +00005611 if (!(dev->flags & IFF_LOOPBACK)) {
5612 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5613 if (dev->features & NETIF_F_ALL_CSUM) {
5614 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5615 dev->features |= NETIF_F_NOCACHE_COPY;
5616 }
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005617 }
5618
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005619 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005620 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005621 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005622
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005623 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5624 ret = notifier_to_errno(ret);
5625 if (ret)
5626 goto err_uninit;
5627
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005628 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005629 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005630 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005631 dev->reg_state = NETREG_REGISTERED;
5632
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005633 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005634
Linus Torvalds1da177e2005-04-16 15:20:36 -07005635 /*
5636 * Default initial state at registry is that the
5637 * device is present.
5638 */
5639
5640 set_bit(__LINK_STATE_PRESENT, &dev->state);
5641
Linus Torvalds1da177e2005-04-16 15:20:36 -07005642 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005643 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005644 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005645 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005646
5647 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005648 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005649 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005650 if (ret) {
5651 rollback_registered(dev);
5652 dev->reg_state = NETREG_UNREGISTERED;
5653 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005654 /*
5655 * Prevent userspace races by waiting until the network
5656 * device is fully setup before sending notifications.
5657 */
Patrick McHardya2835762010-02-26 06:34:51 +00005658 if (!dev->rtnl_link_ops ||
5659 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5660 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005661
5662out:
5663 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005664
5665err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005666 if (dev->netdev_ops->ndo_uninit)
5667 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005668 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005669}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005670EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005671
5672/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005673 * init_dummy_netdev - init a dummy network device for NAPI
5674 * @dev: device to init
5675 *
5676 * This takes a network device structure and initialize the minimum
5677 * amount of fields so it can be used to schedule NAPI polls without
5678 * registering a full blown interface. This is to be used by drivers
5679 * that need to tie several hardware interfaces to a single NAPI
5680 * poll scheduler due to HW limitations.
5681 */
5682int init_dummy_netdev(struct net_device *dev)
5683{
5684 /* Clear everything. Note we don't initialize spinlocks
5685 * are they aren't supposed to be taken by any of the
5686 * NAPI code and this dummy netdev is supposed to be
5687 * only ever used for NAPI polls
5688 */
5689 memset(dev, 0, sizeof(struct net_device));
5690
5691 /* make sure we BUG if trying to hit standard
5692 * register/unregister code path
5693 */
5694 dev->reg_state = NETREG_DUMMY;
5695
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005696 /* NAPI wants this */
5697 INIT_LIST_HEAD(&dev->napi_list);
5698
5699 /* a dummy interface is started by default */
5700 set_bit(__LINK_STATE_PRESENT, &dev->state);
5701 set_bit(__LINK_STATE_START, &dev->state);
5702
Eric Dumazet29b44332010-10-11 10:22:12 +00005703 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5704 * because users of this 'device' dont need to change
5705 * its refcount.
5706 */
5707
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005708 return 0;
5709}
5710EXPORT_SYMBOL_GPL(init_dummy_netdev);
5711
5712
5713/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005714 * register_netdev - register a network device
5715 * @dev: device to register
5716 *
5717 * Take a completed network device structure and add it to the kernel
5718 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5719 * chain. 0 is returned on success. A negative errno code is returned
5720 * on a failure to set up the device, or if the name is a duplicate.
5721 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005722 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005723 * and expands the device name if you passed a format string to
5724 * alloc_netdev.
5725 */
5726int register_netdev(struct net_device *dev)
5727{
5728 int err;
5729
5730 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005731 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005732 rtnl_unlock();
5733 return err;
5734}
5735EXPORT_SYMBOL(register_netdev);
5736
Eric Dumazet29b44332010-10-11 10:22:12 +00005737int netdev_refcnt_read(const struct net_device *dev)
5738{
5739 int i, refcnt = 0;
5740
5741 for_each_possible_cpu(i)
5742 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5743 return refcnt;
5744}
5745EXPORT_SYMBOL(netdev_refcnt_read);
5746
Ben Hutchings2c530402012-07-10 10:55:09 +00005747/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005748 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00005749 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005750 *
5751 * This is called when unregistering network devices.
5752 *
5753 * Any protocol or device that holds a reference should register
5754 * for netdevice notification, and cleanup and put back the
5755 * reference if they receive an UNREGISTER event.
5756 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005757 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005758 */
5759static void netdev_wait_allrefs(struct net_device *dev)
5760{
5761 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005762 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005763
Eric Dumazete014deb2009-11-17 05:59:21 +00005764 linkwatch_forget_dev(dev);
5765
Linus Torvalds1da177e2005-04-16 15:20:36 -07005766 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005767 refcnt = netdev_refcnt_read(dev);
5768
5769 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005770 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005771 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005772
5773 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005774 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005775 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005776 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005777
5778 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5779 &dev->state)) {
5780 /* We must not have linkwatch events
5781 * pending on unregister. If this
5782 * happens, we simply run the queue
5783 * unscheduled, resulting in a noop
5784 * for this device.
5785 */
5786 linkwatch_run_queue();
5787 }
5788
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005789 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005790
5791 rebroadcast_time = jiffies;
5792 }
5793
5794 msleep(250);
5795
Eric Dumazet29b44332010-10-11 10:22:12 +00005796 refcnt = netdev_refcnt_read(dev);
5797
Linus Torvalds1da177e2005-04-16 15:20:36 -07005798 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005799 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5800 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005801 warning_time = jiffies;
5802 }
5803 }
5804}
5805
5806/* The sequence is:
5807 *
5808 * rtnl_lock();
5809 * ...
5810 * register_netdevice(x1);
5811 * register_netdevice(x2);
5812 * ...
5813 * unregister_netdevice(y1);
5814 * unregister_netdevice(y2);
5815 * ...
5816 * rtnl_unlock();
5817 * free_netdev(y1);
5818 * free_netdev(y2);
5819 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005820 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005821 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005822 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005823 * without deadlocking with linkwatch via keventd.
5824 * 2) Since we run with the RTNL semaphore not held, we can sleep
5825 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005826 *
5827 * We must not return until all unregister events added during
5828 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005829 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830void netdev_run_todo(void)
5831{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005832 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005833
Linus Torvalds1da177e2005-04-16 15:20:36 -07005834 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005835 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005836
5837 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005838
Eric W. Biederman850a5452011-10-13 22:25:23 +00005839 /* Wait for rcu callbacks to finish before attempting to drain
5840 * the device list. This usually avoids a 250ms wait.
5841 */
5842 if (!list_empty(&list))
5843 rcu_barrier();
5844
Linus Torvalds1da177e2005-04-16 15:20:36 -07005845 while (!list_empty(&list)) {
5846 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005847 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005848 list_del(&dev->todo_list);
5849
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005850 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005851 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07005852 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005853 dump_stack();
5854 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005855 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005856
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005857 dev->reg_state = NETREG_UNREGISTERED;
5858
Changli Gao152102c2010-03-30 20:16:22 +00005859 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005860
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005861 netdev_wait_allrefs(dev);
5862
5863 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005864 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00005865 WARN_ON(rcu_access_pointer(dev->ip_ptr));
5866 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005867 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005868
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005869 if (dev->destructor)
5870 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005871
5872 /* Free network device */
5873 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005874 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005875}
5876
Ben Hutchings3cfde792010-07-09 09:11:52 +00005877/* Convert net_device_stats to rtnl_link_stats64. They have the same
5878 * fields in the same order, with only the type differing.
5879 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005880void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5881 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00005882{
5883#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005884 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5885 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00005886#else
5887 size_t i, n = sizeof(*stats64) / sizeof(u64);
5888 const unsigned long *src = (const unsigned long *)netdev_stats;
5889 u64 *dst = (u64 *)stats64;
5890
5891 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5892 sizeof(*stats64) / sizeof(u64));
5893 for (i = 0; i < n; i++)
5894 dst[i] = src[i];
5895#endif
5896}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00005897EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00005898
Eric Dumazetd83345a2009-11-16 03:36:51 +00005899/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005900 * dev_get_stats - get network device statistics
5901 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005902 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005903 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005904 * Get network statistics from device. Return @storage.
5905 * The device driver may provide its own method by setting
5906 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5907 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005908 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005909struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5910 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005911{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005912 const struct net_device_ops *ops = dev->netdev_ops;
5913
Eric Dumazet28172732010-07-07 14:58:56 -07005914 if (ops->ndo_get_stats64) {
5915 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005916 ops->ndo_get_stats64(dev, storage);
5917 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005918 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005919 } else {
5920 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005921 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005922 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005923 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005924}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005925EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005926
Eric Dumazet24824a02010-10-02 06:11:55 +00005927struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005928{
Eric Dumazet24824a02010-10-02 06:11:55 +00005929 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005930
Eric Dumazet24824a02010-10-02 06:11:55 +00005931#ifdef CONFIG_NET_CLS_ACT
5932 if (queue)
5933 return queue;
5934 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5935 if (!queue)
5936 return NULL;
5937 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005938 queue->qdisc = &noop_qdisc;
5939 queue->qdisc_sleeping = &noop_qdisc;
5940 rcu_assign_pointer(dev->ingress_queue, queue);
5941#endif
5942 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005943}
5944
Linus Torvalds1da177e2005-04-16 15:20:36 -07005945/**
Tom Herbert36909ea2011-01-09 19:36:31 +00005946 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005947 * @sizeof_priv: size of private data to allocate space for
5948 * @name: device name format string
5949 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00005950 * @txqs: the number of TX subqueues to allocate
5951 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005952 *
5953 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005954 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00005955 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005956 */
Tom Herbert36909ea2011-01-09 19:36:31 +00005957struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5958 void (*setup)(struct net_device *),
5959 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005960{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005961 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005962 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005963 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005964
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005965 BUG_ON(strlen(name) >= sizeof(dev->name));
5966
Tom Herbert36909ea2011-01-09 19:36:31 +00005967 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005968 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00005969 return NULL;
5970 }
5971
Tom Herbert36909ea2011-01-09 19:36:31 +00005972#ifdef CONFIG_RPS
5973 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005974 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00005975 return NULL;
5976 }
5977#endif
5978
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005979 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005980 if (sizeof_priv) {
5981 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005982 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005983 alloc_size += sizeof_priv;
5984 }
5985 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005986 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005987
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005988 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005989 if (!p) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005990 pr_err("alloc_netdev: Unable to allocate device\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005991 return NULL;
5992 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005993
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005994 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005995 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005996
Eric Dumazet29b44332010-10-11 10:22:12 +00005997 dev->pcpu_refcnt = alloc_percpu(int);
5998 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00005999 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006000
Linus Torvalds1da177e2005-04-16 15:20:36 -07006001 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006002 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006003
Jiri Pirko22bedad2010-04-01 21:22:57 +00006004 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006005 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006006
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006007 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006008
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006009 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00006010 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006011
Herbert Xud565b0a2008-12-15 23:38:52 -08006012 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006013 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006014 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07006015 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006016 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006017
6018 dev->num_tx_queues = txqs;
6019 dev->real_num_tx_queues = txqs;
6020 if (netif_alloc_netdev_queues(dev))
6021 goto free_all;
6022
6023#ifdef CONFIG_RPS
6024 dev->num_rx_queues = rxqs;
6025 dev->real_num_rx_queues = rxqs;
6026 if (netif_alloc_rx_queues(dev))
6027 goto free_all;
6028#endif
6029
Linus Torvalds1da177e2005-04-16 15:20:36 -07006030 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006031 dev->group = INIT_NETDEV_GROUP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006032 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006033
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006034free_all:
6035 free_netdev(dev);
6036 return NULL;
6037
Eric Dumazet29b44332010-10-11 10:22:12 +00006038free_pcpu:
6039 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00006040 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006041#ifdef CONFIG_RPS
6042 kfree(dev->_rx);
6043#endif
6044
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006045free_p:
6046 kfree(p);
6047 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006048}
Tom Herbert36909ea2011-01-09 19:36:31 +00006049EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006050
6051/**
6052 * free_netdev - free network device
6053 * @dev: device
6054 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006055 * This function does the last stage of destroying an allocated device
6056 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006057 * If this is the last reference then it will be freed.
6058 */
6059void free_netdev(struct net_device *dev)
6060{
Herbert Xud565b0a2008-12-15 23:38:52 -08006061 struct napi_struct *p, *n;
6062
Denis V. Lunevf3005d72008-04-16 02:02:18 -07006063 release_net(dev_net(dev));
6064
David S. Millere8a04642008-07-17 00:34:19 -07006065 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006066#ifdef CONFIG_RPS
6067 kfree(dev->_rx);
6068#endif
David S. Millere8a04642008-07-17 00:34:19 -07006069
Eric Dumazet33d480c2011-08-11 19:30:52 +00006070 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006071
Jiri Pirkof001fde2009-05-05 02:48:28 +00006072 /* Flush device addresses */
6073 dev_addr_flush(dev);
6074
Herbert Xud565b0a2008-12-15 23:38:52 -08006075 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6076 netif_napi_del(p);
6077
Eric Dumazet29b44332010-10-11 10:22:12 +00006078 free_percpu(dev->pcpu_refcnt);
6079 dev->pcpu_refcnt = NULL;
6080
Stephen Hemminger3041a062006-05-26 13:25:24 -07006081 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006082 if (dev->reg_state == NETREG_UNINITIALIZED) {
6083 kfree((char *)dev - dev->padded);
6084 return;
6085 }
6086
6087 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6088 dev->reg_state = NETREG_RELEASED;
6089
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006090 /* will free via device release */
6091 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006092}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006093EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006094
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006095/**
6096 * synchronize_net - Synchronize with packet receive processing
6097 *
6098 * Wait for packets currently being received to be done.
6099 * Does not block later packets from starting.
6100 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006101void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006102{
6103 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006104 if (rtnl_is_locked())
6105 synchronize_rcu_expedited();
6106 else
6107 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006108}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006109EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006110
6111/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006112 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006113 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006114 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006115 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006116 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006117 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006118 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006119 *
6120 * Callers must hold the rtnl semaphore. You may want
6121 * unregister_netdev() instead of this.
6122 */
6123
Eric Dumazet44a08732009-10-27 07:03:04 +00006124void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006125{
Herbert Xua6620712007-12-12 19:21:56 -08006126 ASSERT_RTNL();
6127
Eric Dumazet44a08732009-10-27 07:03:04 +00006128 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006129 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006130 } else {
6131 rollback_registered(dev);
6132 /* Finish processing unregister after unlock */
6133 net_set_todo(dev);
6134 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006135}
Eric Dumazet44a08732009-10-27 07:03:04 +00006136EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006137
6138/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006139 * unregister_netdevice_many - unregister many devices
6140 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006141 */
6142void unregister_netdevice_many(struct list_head *head)
6143{
6144 struct net_device *dev;
6145
6146 if (!list_empty(head)) {
6147 rollback_registered_many(head);
6148 list_for_each_entry(dev, head, unreg_list)
6149 net_set_todo(dev);
6150 }
6151}
Eric Dumazet63c80992009-10-27 07:06:49 +00006152EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006153
6154/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006155 * unregister_netdev - remove device from the kernel
6156 * @dev: device
6157 *
6158 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006159 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006160 *
6161 * This is just a wrapper for unregister_netdevice that takes
6162 * the rtnl semaphore. In general you want to use this and not
6163 * unregister_netdevice.
6164 */
6165void unregister_netdev(struct net_device *dev)
6166{
6167 rtnl_lock();
6168 unregister_netdevice(dev);
6169 rtnl_unlock();
6170}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006171EXPORT_SYMBOL(unregister_netdev);
6172
Eric W. Biedermance286d32007-09-12 13:53:49 +02006173/**
6174 * dev_change_net_namespace - move device to different nethost namespace
6175 * @dev: device
6176 * @net: network namespace
6177 * @pat: If not NULL name pattern to try if the current device name
6178 * is already taken in the destination network namespace.
6179 *
6180 * This function shuts down a device interface and moves it
6181 * to a new network namespace. On success 0 is returned, on
6182 * a failure a netagive errno code is returned.
6183 *
6184 * Callers must hold the rtnl semaphore.
6185 */
6186
6187int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6188{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006189 int err;
6190
6191 ASSERT_RTNL();
6192
6193 /* Don't allow namespace local devices to be moved. */
6194 err = -EINVAL;
6195 if (dev->features & NETIF_F_NETNS_LOCAL)
6196 goto out;
6197
6198 /* Ensure the device has been registrered */
6199 err = -EINVAL;
6200 if (dev->reg_state != NETREG_REGISTERED)
6201 goto out;
6202
6203 /* Get out if there is nothing todo */
6204 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006205 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006206 goto out;
6207
6208 /* Pick the destination device name, and ensure
6209 * we can use it in the destination network namespace.
6210 */
6211 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006212 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006213 /* We get here if we can't use the current device name */
6214 if (!pat)
6215 goto out;
Jiri Pirko1c5cae82011-04-30 01:21:32 +00006216 if (dev_get_valid_name(dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006217 goto out;
6218 }
6219
6220 /*
6221 * And now a mini version of register_netdevice unregister_netdevice.
6222 */
6223
6224 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006225 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006226
6227 /* And unlink it from device chain */
6228 err = -ENODEV;
6229 unlist_netdevice(dev);
6230
6231 synchronize_net();
6232
6233 /* Shutdown queueing discipline. */
6234 dev_shutdown(dev);
6235
6236 /* Notify protocols, that we are about to destroy
6237 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006238
6239 Note that dev->reg_state stays at NETREG_REGISTERED.
6240 This is wanted because this way 8021q and macvlan know
6241 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006242 */
6243 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006244 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00006245 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006246
6247 /*
6248 * Flush the unicast and multicast chains
6249 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006250 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00006251 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006252
6253 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006254 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006255
Eric W. Biedermance286d32007-09-12 13:53:49 +02006256 /* If there is an ifindex conflict assign a new one */
6257 if (__dev_get_by_index(net, dev->ifindex)) {
6258 int iflink = (dev->iflink == dev->ifindex);
6259 dev->ifindex = dev_new_index(net);
6260 if (iflink)
6261 dev->iflink = dev->ifindex;
6262 }
6263
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006264 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006265 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006266 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006267
6268 /* Add the device back in the hashes */
6269 list_netdevice(dev);
6270
6271 /* Notify protocols, that a new device appeared. */
6272 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6273
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006274 /*
6275 * Prevent userspace races by waiting until the network
6276 * device is fully setup before sending notifications.
6277 */
6278 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6279
Eric W. Biedermance286d32007-09-12 13:53:49 +02006280 synchronize_net();
6281 err = 0;
6282out:
6283 return err;
6284}
Johannes Berg463d0182009-07-14 00:33:35 +02006285EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006286
Linus Torvalds1da177e2005-04-16 15:20:36 -07006287static int dev_cpu_callback(struct notifier_block *nfb,
6288 unsigned long action,
6289 void *ocpu)
6290{
6291 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006292 struct sk_buff *skb;
6293 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6294 struct softnet_data *sd, *oldsd;
6295
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006296 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006297 return NOTIFY_OK;
6298
6299 local_irq_disable();
6300 cpu = smp_processor_id();
6301 sd = &per_cpu(softnet_data, cpu);
6302 oldsd = &per_cpu(softnet_data, oldcpu);
6303
6304 /* Find end of our completion_queue. */
6305 list_skb = &sd->completion_queue;
6306 while (*list_skb)
6307 list_skb = &(*list_skb)->next;
6308 /* Append completion queue from offline CPU. */
6309 *list_skb = oldsd->completion_queue;
6310 oldsd->completion_queue = NULL;
6311
Linus Torvalds1da177e2005-04-16 15:20:36 -07006312 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006313 if (oldsd->output_queue) {
6314 *sd->output_queue_tailp = oldsd->output_queue;
6315 sd->output_queue_tailp = oldsd->output_queue_tailp;
6316 oldsd->output_queue = NULL;
6317 oldsd->output_queue_tailp = &oldsd->output_queue;
6318 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006319 /* Append NAPI poll list from offline CPU. */
6320 if (!list_empty(&oldsd->poll_list)) {
6321 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6322 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6323 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006324
6325 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6326 local_irq_enable();
6327
6328 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006329 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6330 netif_rx(skb);
6331 input_queue_head_incr(oldsd);
6332 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006333 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006334 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006335 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006336 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006337
6338 return NOTIFY_OK;
6339}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006340
6341
Herbert Xu7f353bf2007-08-10 15:47:58 -07006342/**
Herbert Xub63365a2008-10-23 01:11:29 -07006343 * netdev_increment_features - increment feature set by one
6344 * @all: current feature set
6345 * @one: new feature set
6346 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006347 *
6348 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006349 * @one to the master device with current feature set @all. Will not
6350 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006351 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006352netdev_features_t netdev_increment_features(netdev_features_t all,
6353 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006354{
Michał Mirosław1742f182011-04-22 06:31:16 +00006355 if (mask & NETIF_F_GEN_CSUM)
6356 mask |= NETIF_F_ALL_CSUM;
6357 mask |= NETIF_F_VLAN_CHALLENGED;
6358
6359 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6360 all &= one | ~NETIF_F_ALL_FOR_ALL;
6361
Michał Mirosław1742f182011-04-22 06:31:16 +00006362 /* If one device supports hw checksumming, set for all. */
6363 if (all & NETIF_F_GEN_CSUM)
6364 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006365
6366 return all;
6367}
Herbert Xub63365a2008-10-23 01:11:29 -07006368EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006369
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006370static struct hlist_head *netdev_create_hash(void)
6371{
6372 int i;
6373 struct hlist_head *hash;
6374
6375 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6376 if (hash != NULL)
6377 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6378 INIT_HLIST_HEAD(&hash[i]);
6379
6380 return hash;
6381}
6382
Eric W. Biederman881d9662007-09-17 11:56:21 -07006383/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006384static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006385{
Rustad, Mark D734b6542012-07-18 09:06:07 +00006386 if (net != &init_net)
6387 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006388
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006389 net->dev_name_head = netdev_create_hash();
6390 if (net->dev_name_head == NULL)
6391 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006392
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006393 net->dev_index_head = netdev_create_hash();
6394 if (net->dev_index_head == NULL)
6395 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006396
6397 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006398
6399err_idx:
6400 kfree(net->dev_name_head);
6401err_name:
6402 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006403}
6404
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006405/**
6406 * netdev_drivername - network driver for the device
6407 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006408 *
6409 * Determine network driver for device.
6410 */
David S. Miller3019de12011-06-06 16:41:33 -07006411const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006412{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006413 const struct device_driver *driver;
6414 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006415 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006416
6417 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006418 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006419 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006420
6421 driver = parent->driver;
6422 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006423 return driver->name;
6424 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006425}
6426
Jason Baronffa10cb2011-08-11 14:36:48 -04006427int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006428 struct va_format *vaf)
6429{
6430 int r;
6431
6432 if (dev && dev->dev.parent)
6433 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6434 netdev_name(dev), vaf);
6435 else if (dev)
6436 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6437 else
6438 r = printk("%s(NULL net_device): %pV", level, vaf);
6439
6440 return r;
6441}
Jason Baronffa10cb2011-08-11 14:36:48 -04006442EXPORT_SYMBOL(__netdev_printk);
Joe Perches256df2f2010-06-27 01:02:35 +00006443
6444int netdev_printk(const char *level, const struct net_device *dev,
6445 const char *format, ...)
6446{
6447 struct va_format vaf;
6448 va_list args;
6449 int r;
6450
6451 va_start(args, format);
6452
6453 vaf.fmt = format;
6454 vaf.va = &args;
6455
6456 r = __netdev_printk(level, dev, &vaf);
6457 va_end(args);
6458
6459 return r;
6460}
6461EXPORT_SYMBOL(netdev_printk);
6462
6463#define define_netdev_printk_level(func, level) \
6464int func(const struct net_device *dev, const char *fmt, ...) \
6465{ \
6466 int r; \
6467 struct va_format vaf; \
6468 va_list args; \
6469 \
6470 va_start(args, fmt); \
6471 \
6472 vaf.fmt = fmt; \
6473 vaf.va = &args; \
6474 \
6475 r = __netdev_printk(level, dev, &vaf); \
6476 va_end(args); \
6477 \
6478 return r; \
6479} \
6480EXPORT_SYMBOL(func);
6481
6482define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6483define_netdev_printk_level(netdev_alert, KERN_ALERT);
6484define_netdev_printk_level(netdev_crit, KERN_CRIT);
6485define_netdev_printk_level(netdev_err, KERN_ERR);
6486define_netdev_printk_level(netdev_warn, KERN_WARNING);
6487define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6488define_netdev_printk_level(netdev_info, KERN_INFO);
6489
Pavel Emelyanov46650792007-10-08 20:38:39 -07006490static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006491{
6492 kfree(net->dev_name_head);
6493 kfree(net->dev_index_head);
6494}
6495
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006496static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006497 .init = netdev_init,
6498 .exit = netdev_exit,
6499};
6500
Pavel Emelyanov46650792007-10-08 20:38:39 -07006501static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006502{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006503 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006504 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006505 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006506 * initial network namespace
6507 */
6508 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006509 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006510 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006511 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006512
6513 /* Ignore unmoveable devices (i.e. loopback) */
6514 if (dev->features & NETIF_F_NETNS_LOCAL)
6515 continue;
6516
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006517 /* Leave virtual devices for the generic cleanup */
6518 if (dev->rtnl_link_ops)
6519 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006520
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006521 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006522 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6523 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006524 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006525 pr_emerg("%s: failed to move %s to init_net: %d\n",
6526 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006527 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006528 }
6529 }
6530 rtnl_unlock();
6531}
6532
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006533static void __net_exit default_device_exit_batch(struct list_head *net_list)
6534{
6535 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006536 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006537 * Do this across as many network namespaces as possible to
6538 * improve batching efficiency.
6539 */
6540 struct net_device *dev;
6541 struct net *net;
6542 LIST_HEAD(dev_kill_list);
6543
6544 rtnl_lock();
6545 list_for_each_entry(net, net_list, exit_list) {
6546 for_each_netdev_reverse(net, dev) {
6547 if (dev->rtnl_link_ops)
6548 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6549 else
6550 unregister_netdevice_queue(dev, &dev_kill_list);
6551 }
6552 }
6553 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006554 list_del(&dev_kill_list);
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006555 rtnl_unlock();
6556}
6557
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006558static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006559 .exit = default_device_exit,
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006560 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006561};
6562
Linus Torvalds1da177e2005-04-16 15:20:36 -07006563/*
6564 * Initialize the DEV module. At boot time this walks the device list and
6565 * unhooks any devices that fail to initialise (normally hardware not
6566 * present) and leaves us with a valid list of present and active devices.
6567 *
6568 */
6569
6570/*
6571 * This is called single threaded during boot, so no need
6572 * to take the rtnl semaphore.
6573 */
6574static int __init net_dev_init(void)
6575{
6576 int i, rc = -ENOMEM;
6577
6578 BUG_ON(!dev_boot_phase);
6579
Linus Torvalds1da177e2005-04-16 15:20:36 -07006580 if (dev_proc_init())
6581 goto out;
6582
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006583 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006584 goto out;
6585
6586 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006587 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006588 INIT_LIST_HEAD(&ptype_base[i]);
6589
Eric W. Biederman881d9662007-09-17 11:56:21 -07006590 if (register_pernet_subsys(&netdev_net_ops))
6591 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006592
6593 /*
6594 * Initialise the packet receive queues.
6595 */
6596
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006597 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006598 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006599
Changli Gaodee42872010-05-02 05:42:16 +00006600 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006601 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006602 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006603 sd->completion_queue = NULL;
6604 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006605 sd->output_queue = NULL;
6606 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006607#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006608 sd->csd.func = rps_trigger_softirq;
6609 sd->csd.info = sd;
6610 sd->csd.flags = 0;
6611 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006612#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006613
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006614 sd->backlog.poll = process_backlog;
6615 sd->backlog.weight = weight_p;
6616 sd->backlog.gro_list = NULL;
6617 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006618 }
6619
Linus Torvalds1da177e2005-04-16 15:20:36 -07006620 dev_boot_phase = 0;
6621
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006622 /* The loopback device is special if any other network devices
6623 * is present in a network namespace the loopback device must
6624 * be present. Since we now dynamically allocate and free the
6625 * loopback device ensure this invariant is maintained by
6626 * keeping the loopback device as the first device on the
6627 * list of network devices. Ensuring the loopback devices
6628 * is the first device that appears and the last network device
6629 * that disappears.
6630 */
6631 if (register_pernet_device(&loopback_net_ops))
6632 goto out;
6633
6634 if (register_pernet_device(&default_device_ops))
6635 goto out;
6636
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006637 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6638 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006639
6640 hotcpu_notifier(dev_cpu_callback, 0);
6641 dst_init();
6642 dev_mcast_init();
6643 rc = 0;
6644out:
6645 return rc;
6646}
6647
6648subsys_initcall(net_dev_init);
6649
Krishna Kumare88721f2009-02-18 17:55:02 -08006650static int __init initialize_hashrnd(void)
6651{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006652 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006653 return 0;
6654}
6655
6656late_initcall_sync(initialize_hashrnd);
6657