blob: 2a5f5586642923005e3730cb05a9c05edd83db9f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
100#include <linux/proc_fs.h>
101#include <linux/seq_file.h>
102#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/dst.h>
104#include <net/pkt_sched.h>
105#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000106#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/highmem.h>
108#include <linux/init.h>
109#include <linux/kmod.h>
110#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700114#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500117#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700118#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700119#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700120#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700121#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700122#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700123#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700124#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ipv6.h>
126#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700127#include <linux/jhash.h>
128#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700129#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900130#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900131#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000132#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700133#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000134#include <linux/cpu_rmap.h>
Richard Cochran4dc360c2011-10-19 17:00:35 -0400135#include <linux/net_tstamp.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100136#include <linux/static_key.h>
Eric Dumazet4504b862011-11-28 05:23:23 +0000137#include <net/flow_keys.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700139#include "net-sysfs.h"
140
Herbert Xud565b0a2008-12-15 23:38:52 -0800141/* Instead of increasing this, you should create a hash table. */
142#define MAX_GRO_SKBS 8
143
Herbert Xu5d38a072009-01-04 16:13:40 -0800144/* This should be increased if a protocol with a bigger head is added. */
145#define GRO_MAX_HEAD (MAX_HEADER + 128)
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147/*
148 * The list of packet types we will receive (as opposed to discard)
149 * and the routines to invoke.
150 *
151 * Why 16. Because with 16 the only overlap we get on a hash of the
152 * low nibble of the protocol value is RARP/SNAP/X.25.
153 *
154 * NOTE: That is no longer true with the addition of VLAN tags. Not
155 * sure which should go first, but I bet it won't make much
156 * difference if we are running VLANs. The good news is that
157 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700158 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 * --BLG
160 *
161 * 0800 IP
162 * 8100 802.1Q VLAN
163 * 0001 802.3
164 * 0002 AX.25
165 * 0004 802.2
166 * 8035 RARP
167 * 0005 SNAP
168 * 0805 X.25
169 * 0806 ARP
170 * 8137 IPX
171 * 0009 Localtalk
172 * 86DD IPv6
173 */
174
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800175#define PTYPE_HASH_SIZE (16)
176#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
177
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000179static DEFINE_SPINLOCK(offload_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800180static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700181static struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000182static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700185 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 * semaphore.
187 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800188 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 *
190 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700191 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 * actual updates. This allows pure readers to access the list even
193 * while a writer is preparing to update it.
194 *
195 * To put it another way, dev_base_lock is held for writing only to
196 * protect against pure readers; the rtnl semaphore provides the
197 * protection against other writers.
198 *
199 * See, for example usages, register_netdevice() and
200 * unregister_netdevice(), which must be called with the rtnl
201 * semaphore held.
202 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204EXPORT_SYMBOL(dev_base_lock);
205
Brian Haleyc91f6df2012-11-26 05:21:08 +0000206DEFINE_SEQLOCK(devnet_rename_seq);
207
Thomas Graf4e985ad2011-06-21 03:11:20 +0000208static inline void dev_base_seq_inc(struct net *net)
209{
210 while (++net->dev_base_seq == 0);
211}
212
Eric W. Biederman881d9662007-09-17 11:56:21 -0700213static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
Eric Dumazet95c96172012-04-15 05:58:06 +0000215 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
216
stephen hemminger08e98972009-11-10 07:20:34 +0000217 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218}
219
Eric W. Biederman881d9662007-09-17 11:56:21 -0700220static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700222 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
224
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000225static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000226{
227#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000229#endif
230}
231
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000232static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000233{
234#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000235 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000236#endif
237}
238
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239/* Device list insertion */
240static int list_netdevice(struct net_device *dev)
241{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900242 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200243
244 ASSERT_RTNL();
245
246 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800247 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000248 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000249 hlist_add_head_rcu(&dev->index_hlist,
250 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200251 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000252
253 dev_base_seq_inc(net);
254
Eric W. Biedermance286d32007-09-12 13:53:49 +0200255 return 0;
256}
257
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000258/* Device list removal
259 * caller must respect a RCU grace period before freeing/reusing dev
260 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200261static void unlist_netdevice(struct net_device *dev)
262{
263 ASSERT_RTNL();
264
265 /* Unlink dev from the device chain */
266 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800267 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000268 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000269 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200270 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000271
272 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200273}
274
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275/*
276 * Our notifier list
277 */
278
Alan Sternf07d5b92006-05-09 15:23:03 -0700279static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280
281/*
282 * Device drivers call our routines to queue packets here. We empty the
283 * queue in the local softnet handler.
284 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700285
Eric Dumazet9958da02010-04-17 04:17:02 +0000286DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700287EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
David S. Millercf508b12008-07-22 14:16:42 -0700289#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700290/*
David S. Millerc773e842008-07-08 23:13:53 -0700291 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700292 * according to dev->type
293 */
294static const unsigned short netdev_lock_type[] =
295 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
296 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
297 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
298 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
299 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
300 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
301 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
302 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
303 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
304 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
305 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
306 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400307 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
308 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
309 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700310
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700311static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700312 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
313 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
314 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
315 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
316 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
317 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
318 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
319 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
320 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
321 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
322 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
323 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400324 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
325 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
326 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700327
328static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700329static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700330
331static inline unsigned short netdev_lock_pos(unsigned short dev_type)
332{
333 int i;
334
335 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
336 if (netdev_lock_type[i] == dev_type)
337 return i;
338 /* the last key is used by default */
339 return ARRAY_SIZE(netdev_lock_type) - 1;
340}
341
David S. Millercf508b12008-07-22 14:16:42 -0700342static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
343 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700344{
345 int i;
346
347 i = netdev_lock_pos(dev_type);
348 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
349 netdev_lock_name[i]);
350}
David S. Millercf508b12008-07-22 14:16:42 -0700351
352static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
353{
354 int i;
355
356 i = netdev_lock_pos(dev->type);
357 lockdep_set_class_and_name(&dev->addr_list_lock,
358 &netdev_addr_lock_key[i],
359 netdev_lock_name[i]);
360}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700361#else
David S. Millercf508b12008-07-22 14:16:42 -0700362static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
363 unsigned short dev_type)
364{
365}
366static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700367{
368}
369#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
371/*******************************************************************************
372
373 Protocol management and registration routines
374
375*******************************************************************************/
376
377/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 * Add a protocol ID to the list. Now that the input handler is
379 * smarter we can dispense with all the messy stuff that used to be
380 * here.
381 *
382 * BEWARE!!! Protocol handlers, mangling input packets,
383 * MUST BE last in hash buckets and checking protocol handlers
384 * MUST start from promiscuous ptype_all chain in net_bh.
385 * It is true now, do not change it.
386 * Explanation follows: if protocol handler, mangling packet, will
387 * be the first on list, it is not able to sense, that packet
388 * is cloned and should be copied-on-write, so that it will
389 * change it and subsequent readers will get broken packet.
390 * --ANK (980803)
391 */
392
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000393static inline struct list_head *ptype_head(const struct packet_type *pt)
394{
395 if (pt->type == htons(ETH_P_ALL))
396 return &ptype_all;
397 else
398 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
399}
400
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401/**
402 * dev_add_pack - add packet handler
403 * @pt: packet type declaration
404 *
405 * Add a protocol handler to the networking stack. The passed &packet_type
406 * is linked into kernel lists and may not be freed until it has been
407 * removed from the kernel lists.
408 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900409 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 * guarantee all CPU's that are in middle of receiving packets
411 * will see the new packet type (until the next received packet).
412 */
413
414void dev_add_pack(struct packet_type *pt)
415{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000416 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000418 spin_lock(&ptype_lock);
419 list_add_rcu(&pt->list, head);
420 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700422EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424/**
425 * __dev_remove_pack - remove packet handler
426 * @pt: packet type declaration
427 *
428 * Remove a protocol handler that was previously added to the kernel
429 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
430 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900431 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 *
433 * The packet type might still be in use by receivers
434 * and must not be freed until after all the CPU's have gone
435 * through a quiescent state.
436 */
437void __dev_remove_pack(struct packet_type *pt)
438{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000439 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 struct packet_type *pt1;
441
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000442 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
444 list_for_each_entry(pt1, head, list) {
445 if (pt == pt1) {
446 list_del_rcu(&pt->list);
447 goto out;
448 }
449 }
450
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000451 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000453 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700455EXPORT_SYMBOL(__dev_remove_pack);
456
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457/**
458 * dev_remove_pack - remove packet handler
459 * @pt: packet type declaration
460 *
461 * Remove a protocol handler that was previously added to the kernel
462 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
463 * from the kernel lists and can be freed or reused once this function
464 * returns.
465 *
466 * This call sleeps to guarantee that no CPU is looking at the packet
467 * type after return.
468 */
469void dev_remove_pack(struct packet_type *pt)
470{
471 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900472
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 synchronize_net();
474}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700475EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
Vlad Yasevich62532da2012-11-15 08:49:10 +0000477
478/**
479 * dev_add_offload - register offload handlers
480 * @po: protocol offload declaration
481 *
482 * Add protocol offload handlers to the networking stack. The passed
483 * &proto_offload is linked into kernel lists and may not be freed until
484 * it has been removed from the kernel lists.
485 *
486 * This call does not sleep therefore it can not
487 * guarantee all CPU's that are in middle of receiving packets
488 * will see the new offload handlers (until the next received packet).
489 */
490void dev_add_offload(struct packet_offload *po)
491{
492 struct list_head *head = &offload_base;
493
494 spin_lock(&offload_lock);
495 list_add_rcu(&po->list, head);
496 spin_unlock(&offload_lock);
497}
498EXPORT_SYMBOL(dev_add_offload);
499
500/**
501 * __dev_remove_offload - remove offload handler
502 * @po: packet offload declaration
503 *
504 * Remove a protocol offload handler that was previously added to the
505 * kernel offload handlers by dev_add_offload(). The passed &offload_type
506 * is removed from the kernel lists and can be freed or reused once this
507 * function returns.
508 *
509 * The packet type might still be in use by receivers
510 * and must not be freed until after all the CPU's have gone
511 * through a quiescent state.
512 */
513void __dev_remove_offload(struct packet_offload *po)
514{
515 struct list_head *head = &offload_base;
516 struct packet_offload *po1;
517
Eric Dumazetc53aa502012-11-16 08:08:23 +0000518 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000519
520 list_for_each_entry(po1, head, list) {
521 if (po == po1) {
522 list_del_rcu(&po->list);
523 goto out;
524 }
525 }
526
527 pr_warn("dev_remove_offload: %p not found\n", po);
528out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000529 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000530}
531EXPORT_SYMBOL(__dev_remove_offload);
532
533/**
534 * dev_remove_offload - remove packet offload handler
535 * @po: packet offload declaration
536 *
537 * Remove a packet offload handler that was previously added to the kernel
538 * offload handlers by dev_add_offload(). The passed &offload_type is
539 * removed from the kernel lists and can be freed or reused once this
540 * function returns.
541 *
542 * This call sleeps to guarantee that no CPU is looking at the packet
543 * type after return.
544 */
545void dev_remove_offload(struct packet_offload *po)
546{
547 __dev_remove_offload(po);
548
549 synchronize_net();
550}
551EXPORT_SYMBOL(dev_remove_offload);
552
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553/******************************************************************************
554
555 Device Boot-time Settings Routines
556
557*******************************************************************************/
558
559/* Boot time configuration table */
560static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
561
562/**
563 * netdev_boot_setup_add - add new setup entry
564 * @name: name of the device
565 * @map: configured settings for the device
566 *
567 * Adds new setup entry to the dev_boot_setup list. The function
568 * returns 0 on error and 1 on success. This is a generic routine to
569 * all netdevices.
570 */
571static int netdev_boot_setup_add(char *name, struct ifmap *map)
572{
573 struct netdev_boot_setup *s;
574 int i;
575
576 s = dev_boot_setup;
577 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
578 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
579 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700580 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 memcpy(&s[i].map, map, sizeof(s[i].map));
582 break;
583 }
584 }
585
586 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
587}
588
589/**
590 * netdev_boot_setup_check - check boot time settings
591 * @dev: the netdevice
592 *
593 * Check boot time settings for the device.
594 * The found settings are set for the device to be used
595 * later in the device probing.
596 * Returns 0 if no settings found, 1 if they are.
597 */
598int netdev_boot_setup_check(struct net_device *dev)
599{
600 struct netdev_boot_setup *s = dev_boot_setup;
601 int i;
602
603 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
604 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700605 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 dev->irq = s[i].map.irq;
607 dev->base_addr = s[i].map.base_addr;
608 dev->mem_start = s[i].map.mem_start;
609 dev->mem_end = s[i].map.mem_end;
610 return 1;
611 }
612 }
613 return 0;
614}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700615EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
617
618/**
619 * netdev_boot_base - get address from boot time settings
620 * @prefix: prefix for network device
621 * @unit: id for network device
622 *
623 * Check boot time settings for the base address of device.
624 * The found settings are set for the device to be used
625 * later in the device probing.
626 * Returns 0 if no settings found.
627 */
628unsigned long netdev_boot_base(const char *prefix, int unit)
629{
630 const struct netdev_boot_setup *s = dev_boot_setup;
631 char name[IFNAMSIZ];
632 int i;
633
634 sprintf(name, "%s%d", prefix, unit);
635
636 /*
637 * If device already registered then return base of 1
638 * to indicate not to probe for this interface
639 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700640 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 return 1;
642
643 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
644 if (!strcmp(name, s[i].name))
645 return s[i].map.base_addr;
646 return 0;
647}
648
649/*
650 * Saves at boot time configured settings for any netdevice.
651 */
652int __init netdev_boot_setup(char *str)
653{
654 int ints[5];
655 struct ifmap map;
656
657 str = get_options(str, ARRAY_SIZE(ints), ints);
658 if (!str || !*str)
659 return 0;
660
661 /* Save settings */
662 memset(&map, 0, sizeof(map));
663 if (ints[0] > 0)
664 map.irq = ints[1];
665 if (ints[0] > 1)
666 map.base_addr = ints[2];
667 if (ints[0] > 2)
668 map.mem_start = ints[3];
669 if (ints[0] > 3)
670 map.mem_end = ints[4];
671
672 /* Add new entry to the list */
673 return netdev_boot_setup_add(str, &map);
674}
675
676__setup("netdev=", netdev_boot_setup);
677
678/*******************************************************************************
679
680 Device Interface Subroutines
681
682*******************************************************************************/
683
684/**
685 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700686 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 * @name: name to find
688 *
689 * Find an interface by name. Must be called under RTNL semaphore
690 * or @dev_base_lock. If the name is found a pointer to the device
691 * is returned. If the name is not found then %NULL is returned. The
692 * reference counters are not incremented so the caller must be
693 * careful with locks.
694 */
695
Eric W. Biederman881d9662007-09-17 11:56:21 -0700696struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
698 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700699 struct net_device *dev;
700 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700702 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 if (!strncmp(dev->name, name, IFNAMSIZ))
704 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700705
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 return NULL;
707}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700708EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709
710/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000711 * dev_get_by_name_rcu - find a device by its name
712 * @net: the applicable net namespace
713 * @name: name to find
714 *
715 * Find an interface by name.
716 * If the name is found a pointer to the device is returned.
717 * If the name is not found then %NULL is returned.
718 * The reference counters are not incremented so the caller must be
719 * careful with locks. The caller must hold RCU lock.
720 */
721
722struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
723{
724 struct hlist_node *p;
725 struct net_device *dev;
726 struct hlist_head *head = dev_name_hash(net, name);
727
728 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
729 if (!strncmp(dev->name, name, IFNAMSIZ))
730 return dev;
731
732 return NULL;
733}
734EXPORT_SYMBOL(dev_get_by_name_rcu);
735
736/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700738 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 * @name: name to find
740 *
741 * Find an interface by name. This can be called from any
742 * context and does its own locking. The returned handle has
743 * the usage count incremented and the caller must use dev_put() to
744 * release it when it is no longer needed. %NULL is returned if no
745 * matching device is found.
746 */
747
Eric W. Biederman881d9662007-09-17 11:56:21 -0700748struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749{
750 struct net_device *dev;
751
Eric Dumazet72c95282009-10-30 07:11:27 +0000752 rcu_read_lock();
753 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 if (dev)
755 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000756 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 return dev;
758}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700759EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
761/**
762 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700763 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 * @ifindex: index of device
765 *
766 * Search for an interface by index. Returns %NULL if the device
767 * is not found or a pointer to the device. The device has not
768 * had its reference counter increased so the caller must be careful
769 * about locking. The caller must hold either the RTNL semaphore
770 * or @dev_base_lock.
771 */
772
Eric W. Biederman881d9662007-09-17 11:56:21 -0700773struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
775 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700776 struct net_device *dev;
777 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700779 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 if (dev->ifindex == ifindex)
781 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700782
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 return NULL;
784}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700785EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000787/**
788 * dev_get_by_index_rcu - find a device by its ifindex
789 * @net: the applicable net namespace
790 * @ifindex: index of device
791 *
792 * Search for an interface by index. Returns %NULL if the device
793 * is not found or a pointer to the device. The device has not
794 * had its reference counter increased so the caller must be careful
795 * about locking. The caller must hold RCU lock.
796 */
797
798struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
799{
800 struct hlist_node *p;
801 struct net_device *dev;
802 struct hlist_head *head = dev_index_hash(net, ifindex);
803
804 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
805 if (dev->ifindex == ifindex)
806 return dev;
807
808 return NULL;
809}
810EXPORT_SYMBOL(dev_get_by_index_rcu);
811
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
813/**
814 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700815 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 * @ifindex: index of device
817 *
818 * Search for an interface by index. Returns NULL if the device
819 * is not found or a pointer to the device. The device returned has
820 * had a reference added and the pointer is safe until the user calls
821 * dev_put to indicate they have finished with it.
822 */
823
Eric W. Biederman881d9662007-09-17 11:56:21 -0700824struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825{
826 struct net_device *dev;
827
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000828 rcu_read_lock();
829 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 if (dev)
831 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000832 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 return dev;
834}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700835EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836
837/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000838 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700839 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 * @type: media type of device
841 * @ha: hardware address
842 *
843 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800844 * is not found or a pointer to the device.
845 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000846 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 * and the caller must therefore be careful about locking
848 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 */
850
Eric Dumazet941666c2010-12-05 01:23:53 +0000851struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
852 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853{
854 struct net_device *dev;
855
Eric Dumazet941666c2010-12-05 01:23:53 +0000856 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 if (dev->type == type &&
858 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700859 return dev;
860
861 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862}
Eric Dumazet941666c2010-12-05 01:23:53 +0000863EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300864
Eric W. Biederman881d9662007-09-17 11:56:21 -0700865struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700866{
867 struct net_device *dev;
868
869 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700870 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700871 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700872 return dev;
873
874 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700875}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700876EXPORT_SYMBOL(__dev_getfirstbyhwtype);
877
Eric W. Biederman881d9662007-09-17 11:56:21 -0700878struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000880 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000882 rcu_read_lock();
883 for_each_netdev_rcu(net, dev)
884 if (dev->type == type) {
885 dev_hold(dev);
886 ret = dev;
887 break;
888 }
889 rcu_read_unlock();
890 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892EXPORT_SYMBOL(dev_getfirstbyhwtype);
893
894/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000895 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700896 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 * @if_flags: IFF_* values
898 * @mask: bitmask of bits in if_flags to check
899 *
900 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000901 * is not found or a pointer to the device. Must be called inside
902 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 */
904
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000905struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700906 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700908 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909
Pavel Emelianov7562f872007-05-03 15:13:45 -0700910 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800911 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700913 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 break;
915 }
916 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700917 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000919EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
921/**
922 * dev_valid_name - check if name is okay for network device
923 * @name: name string
924 *
925 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700926 * to allow sysfs to work. We also disallow any kind of
927 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 */
David S. Miller95f050b2012-03-06 16:12:15 -0500929bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700931 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500932 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700933 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500934 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700935 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500936 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700937
938 while (*name) {
939 if (*name == '/' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500940 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700941 name++;
942 }
David S. Miller95f050b2012-03-06 16:12:15 -0500943 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700945EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946
947/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200948 * __dev_alloc_name - allocate a name for a device
949 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200951 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 *
953 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700954 * id. It scans list of devices to build up a free map, then chooses
955 * the first empty slot. The caller must hold the dev_base or rtnl lock
956 * while allocating the name and adding the device in order to avoid
957 * duplicates.
958 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
959 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 */
961
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200962static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963{
964 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 const char *p;
966 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700967 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 struct net_device *d;
969
970 p = strnchr(name, IFNAMSIZ-1, '%');
971 if (p) {
972 /*
973 * Verify the string as this thing may have come from
974 * the user. There must be either one "%d" and no other "%"
975 * characters.
976 */
977 if (p[1] != 'd' || strchr(p + 2, '%'))
978 return -EINVAL;
979
980 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700981 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 if (!inuse)
983 return -ENOMEM;
984
Eric W. Biederman881d9662007-09-17 11:56:21 -0700985 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 if (!sscanf(d->name, name, &i))
987 continue;
988 if (i < 0 || i >= max_netdevices)
989 continue;
990
991 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200992 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 if (!strncmp(buf, d->name, IFNAMSIZ))
994 set_bit(i, inuse);
995 }
996
997 i = find_first_zero_bit(inuse, max_netdevices);
998 free_page((unsigned long) inuse);
999 }
1000
Octavian Purdilad9031022009-11-18 02:36:59 +00001001 if (buf != name)
1002 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001003 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
1006 /* It is possible to run out of possible slots
1007 * when the name is long and there isn't enough space left
1008 * for the digits, or if all bits are used.
1009 */
1010 return -ENFILE;
1011}
1012
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001013/**
1014 * dev_alloc_name - allocate a name for a device
1015 * @dev: device
1016 * @name: name format string
1017 *
1018 * Passed a format string - eg "lt%d" it will try and find a suitable
1019 * id. It scans list of devices to build up a free map, then chooses
1020 * the first empty slot. The caller must hold the dev_base or rtnl lock
1021 * while allocating the name and adding the device in order to avoid
1022 * duplicates.
1023 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1024 * Returns the number of the unit assigned or a negative errno code.
1025 */
1026
1027int dev_alloc_name(struct net_device *dev, const char *name)
1028{
1029 char buf[IFNAMSIZ];
1030 struct net *net;
1031 int ret;
1032
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001033 BUG_ON(!dev_net(dev));
1034 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001035 ret = __dev_alloc_name(net, name, buf);
1036 if (ret >= 0)
1037 strlcpy(dev->name, buf, IFNAMSIZ);
1038 return ret;
1039}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001040EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001041
Gao feng828de4f2012-09-13 20:58:27 +00001042static int dev_alloc_name_ns(struct net *net,
1043 struct net_device *dev,
1044 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001045{
Gao feng828de4f2012-09-13 20:58:27 +00001046 char buf[IFNAMSIZ];
1047 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001048
Gao feng828de4f2012-09-13 20:58:27 +00001049 ret = __dev_alloc_name(net, name, buf);
1050 if (ret >= 0)
1051 strlcpy(dev->name, buf, IFNAMSIZ);
1052 return ret;
1053}
1054
1055static int dev_get_valid_name(struct net *net,
1056 struct net_device *dev,
1057 const char *name)
1058{
1059 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001060
Octavian Purdilad9031022009-11-18 02:36:59 +00001061 if (!dev_valid_name(name))
1062 return -EINVAL;
1063
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001064 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001065 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001066 else if (__dev_get_by_name(net, name))
1067 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001068 else if (dev->name != name)
1069 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001070
1071 return 0;
1072}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073
1074/**
1075 * dev_change_name - change name of a device
1076 * @dev: device
1077 * @newname: name (or format string) must be at least IFNAMSIZ
1078 *
1079 * Change name of a device, can pass format strings "eth%d".
1080 * for wildcarding.
1081 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001082int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083{
Herbert Xufcc5a032007-07-30 17:03:38 -07001084 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001086 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001087 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088
1089 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001090 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001092 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 if (dev->flags & IFF_UP)
1094 return -EBUSY;
1095
Brian Haleyc91f6df2012-11-26 05:21:08 +00001096 write_seqlock(&devnet_rename_seq);
1097
1098 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1099 write_sequnlock(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001100 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001101 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001102
Herbert Xufcc5a032007-07-30 17:03:38 -07001103 memcpy(oldname, dev->name, IFNAMSIZ);
1104
Gao feng828de4f2012-09-13 20:58:27 +00001105 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001106 if (err < 0) {
1107 write_sequnlock(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001108 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001109 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Herbert Xufcc5a032007-07-30 17:03:38 -07001111rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001112 ret = device_rename(&dev->dev, dev->name);
1113 if (ret) {
1114 memcpy(dev->name, oldname, IFNAMSIZ);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001115 write_sequnlock(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001116 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001117 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001118
Brian Haleyc91f6df2012-11-26 05:21:08 +00001119 write_sequnlock(&devnet_rename_seq);
1120
Herbert Xu7f988ea2007-07-30 16:35:46 -07001121 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001122 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001123 write_unlock_bh(&dev_base_lock);
1124
1125 synchronize_rcu();
1126
1127 write_lock_bh(&dev_base_lock);
1128 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001129 write_unlock_bh(&dev_base_lock);
1130
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001131 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001132 ret = notifier_to_errno(ret);
1133
1134 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001135 /* err >= 0 after dev_alloc_name() or stores the first errno */
1136 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001137 err = ret;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001138 write_seqlock(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001139 memcpy(dev->name, oldname, IFNAMSIZ);
1140 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001141 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001142 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001143 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001144 }
1145 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146
1147 return err;
1148}
1149
1150/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001151 * dev_set_alias - change ifalias of a device
1152 * @dev: device
1153 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001154 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001155 *
1156 * Set ifalias for a device,
1157 */
1158int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1159{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001160 char *new_ifalias;
1161
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001162 ASSERT_RTNL();
1163
1164 if (len >= IFALIASZ)
1165 return -EINVAL;
1166
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001167 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001168 kfree(dev->ifalias);
1169 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001170 return 0;
1171 }
1172
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001173 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1174 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001175 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001176 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001177
1178 strlcpy(dev->ifalias, alias, len+1);
1179 return len;
1180}
1181
1182
1183/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001184 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001185 * @dev: device to cause notification
1186 *
1187 * Called to indicate a device has changed features.
1188 */
1189void netdev_features_change(struct net_device *dev)
1190{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001191 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001192}
1193EXPORT_SYMBOL(netdev_features_change);
1194
1195/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 * netdev_state_change - device changes state
1197 * @dev: device to cause notification
1198 *
1199 * Called to indicate a device has changed state. This function calls
1200 * the notifier chains for netdev_chain and sends a NEWLINK message
1201 * to the routing socket.
1202 */
1203void netdev_state_change(struct net_device *dev)
1204{
1205 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001206 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1208 }
1209}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001210EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211
Amerigo Wangee89bab2012-08-09 22:14:56 +00001212/**
1213 * netdev_notify_peers - notify network peers about existence of @dev
1214 * @dev: network device
1215 *
1216 * Generate traffic such that interested network peers are aware of
1217 * @dev, such as by generating a gratuitous ARP. This may be used when
1218 * a device wants to inform the rest of the network about some sort of
1219 * reconfiguration such as a failover event or virtual machine
1220 * migration.
1221 */
1222void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001223{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001224 rtnl_lock();
1225 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1226 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001227}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001228EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001229
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230/**
1231 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001232 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 * @name: name of interface
1234 *
1235 * If a network interface is not present and the process has suitable
1236 * privileges this function loads the module. If module loading is not
1237 * available in this kernel then it becomes a nop.
1238 */
1239
Eric W. Biederman881d9662007-09-17 11:56:21 -07001240void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001242 struct net_device *dev;
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001243 int no_module;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244
Eric Dumazet72c95282009-10-30 07:11:27 +00001245 rcu_read_lock();
1246 dev = dev_get_by_name_rcu(net, name);
1247 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001249 no_module = !dev;
1250 if (no_module && capable(CAP_NET_ADMIN))
1251 no_module = request_module("netdev-%s", name);
1252 if (no_module && capable(CAP_SYS_MODULE)) {
1253 if (!request_module("%s", name))
Vinson Lee7cecb522012-06-27 14:32:07 +00001254 pr_warn("Loading kernel module for a network device with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s instead.\n",
1255 name);
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001256 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001258EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259
Patrick McHardybd380812010-02-26 06:34:53 +00001260static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001262 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001263 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001265 ASSERT_RTNL();
1266
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267 if (!netif_device_present(dev))
1268 return -ENODEV;
1269
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001270 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1271 ret = notifier_to_errno(ret);
1272 if (ret)
1273 return ret;
1274
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001276
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001277 if (ops->ndo_validate_addr)
1278 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001279
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001280 if (!ret && ops->ndo_open)
1281 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
Jeff Garzikbada3392007-10-23 20:19:37 -07001283 if (ret)
1284 clear_bit(__LINK_STATE_START, &dev->state);
1285 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001287 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001288 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001290 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001292
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293 return ret;
1294}
Patrick McHardybd380812010-02-26 06:34:53 +00001295
1296/**
1297 * dev_open - prepare an interface for use.
1298 * @dev: device to open
1299 *
1300 * Takes a device from down to up state. The device's private open
1301 * function is invoked and then the multicast lists are loaded. Finally
1302 * the device is moved into the up state and a %NETDEV_UP message is
1303 * sent to the netdev notifier chain.
1304 *
1305 * Calling this function on an active interface is a nop. On a failure
1306 * a negative errno code is returned.
1307 */
1308int dev_open(struct net_device *dev)
1309{
1310 int ret;
1311
Patrick McHardybd380812010-02-26 06:34:53 +00001312 if (dev->flags & IFF_UP)
1313 return 0;
1314
Patrick McHardybd380812010-02-26 06:34:53 +00001315 ret = __dev_open(dev);
1316 if (ret < 0)
1317 return ret;
1318
Patrick McHardybd380812010-02-26 06:34:53 +00001319 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1320 call_netdevice_notifiers(NETDEV_UP, dev);
1321
1322 return ret;
1323}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001324EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325
Octavian Purdila44345722010-12-13 12:44:07 +00001326static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327{
Octavian Purdila44345722010-12-13 12:44:07 +00001328 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001329
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001330 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001331 might_sleep();
1332
Octavian Purdila44345722010-12-13 12:44:07 +00001333 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001334 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335
Octavian Purdila44345722010-12-13 12:44:07 +00001336 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337
Octavian Purdila44345722010-12-13 12:44:07 +00001338 /* Synchronize to scheduled poll. We cannot touch poll list, it
1339 * can be even on different cpu. So just clear netif_running().
1340 *
1341 * dev->stop() will invoke napi_disable() on all of it's
1342 * napi_struct instances on this device.
1343 */
1344 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1345 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346
Octavian Purdila44345722010-12-13 12:44:07 +00001347 dev_deactivate_many(head);
1348
1349 list_for_each_entry(dev, head, unreg_list) {
1350 const struct net_device_ops *ops = dev->netdev_ops;
1351
1352 /*
1353 * Call the device specific close. This cannot fail.
1354 * Only if device is UP
1355 *
1356 * We allow it to be called even after a DETACH hot-plug
1357 * event.
1358 */
1359 if (ops->ndo_stop)
1360 ops->ndo_stop(dev);
1361
Octavian Purdila44345722010-12-13 12:44:07 +00001362 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001363 net_dmaengine_put();
1364 }
1365
1366 return 0;
1367}
1368
1369static int __dev_close(struct net_device *dev)
1370{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001371 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001372 LIST_HEAD(single);
1373
1374 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001375 retval = __dev_close_many(&single);
1376 list_del(&single);
1377 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001378}
1379
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001380static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001381{
1382 struct net_device *dev, *tmp;
1383 LIST_HEAD(tmp_list);
1384
1385 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1386 if (!(dev->flags & IFF_UP))
1387 list_move(&dev->unreg_list, &tmp_list);
1388
1389 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001390
Octavian Purdila44345722010-12-13 12:44:07 +00001391 list_for_each_entry(dev, head, unreg_list) {
1392 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1393 call_netdevice_notifiers(NETDEV_DOWN, dev);
1394 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395
Octavian Purdila44345722010-12-13 12:44:07 +00001396 /* rollback_registered_many needs the complete original list */
1397 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398 return 0;
1399}
Patrick McHardybd380812010-02-26 06:34:53 +00001400
1401/**
1402 * dev_close - shutdown an interface.
1403 * @dev: device to shutdown
1404 *
1405 * This function moves an active device into down state. A
1406 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1407 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1408 * chain.
1409 */
1410int dev_close(struct net_device *dev)
1411{
Eric Dumazete14a5992011-05-10 12:26:06 -07001412 if (dev->flags & IFF_UP) {
1413 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001414
Eric Dumazete14a5992011-05-10 12:26:06 -07001415 list_add(&dev->unreg_list, &single);
1416 dev_close_many(&single);
1417 list_del(&single);
1418 }
Patrick McHardybd380812010-02-26 06:34:53 +00001419 return 0;
1420}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001421EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422
1423
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001424/**
1425 * dev_disable_lro - disable Large Receive Offload on a device
1426 * @dev: device
1427 *
1428 * Disable Large Receive Offload (LRO) on a net device. Must be
1429 * called under RTNL. This is needed if received packets may be
1430 * forwarded to another interface.
1431 */
1432void dev_disable_lro(struct net_device *dev)
1433{
Neil Hormanf11970e2011-05-24 08:31:09 +00001434 /*
1435 * If we're trying to disable lro on a vlan device
1436 * use the underlying physical device instead
1437 */
1438 if (is_vlan_dev(dev))
1439 dev = vlan_dev_real_dev(dev);
1440
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001441 dev->wanted_features &= ~NETIF_F_LRO;
1442 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001443
Michał Mirosław22d59692011-04-21 12:42:15 +00001444 if (unlikely(dev->features & NETIF_F_LRO))
1445 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001446}
1447EXPORT_SYMBOL(dev_disable_lro);
1448
1449
Eric W. Biederman881d9662007-09-17 11:56:21 -07001450static int dev_boot_phase = 1;
1451
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452/**
1453 * register_netdevice_notifier - register a network notifier block
1454 * @nb: notifier
1455 *
1456 * Register a notifier to be called when network device events occur.
1457 * The notifier passed is linked into the kernel structures and must
1458 * not be reused until it has been unregistered. A negative errno code
1459 * is returned on a failure.
1460 *
1461 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001462 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463 * view of the network device list.
1464 */
1465
1466int register_netdevice_notifier(struct notifier_block *nb)
1467{
1468 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001469 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001470 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471 int err;
1472
1473 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001474 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001475 if (err)
1476 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001477 if (dev_boot_phase)
1478 goto unlock;
1479 for_each_net(net) {
1480 for_each_netdev(net, dev) {
1481 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1482 err = notifier_to_errno(err);
1483 if (err)
1484 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485
Eric W. Biederman881d9662007-09-17 11:56:21 -07001486 if (!(dev->flags & IFF_UP))
1487 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001488
Eric W. Biederman881d9662007-09-17 11:56:21 -07001489 nb->notifier_call(nb, NETDEV_UP, dev);
1490 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001492
1493unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494 rtnl_unlock();
1495 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001496
1497rollback:
1498 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001499 for_each_net(net) {
1500 for_each_netdev(net, dev) {
1501 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001502 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001503
Eric W. Biederman881d9662007-09-17 11:56:21 -07001504 if (dev->flags & IFF_UP) {
1505 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1506 nb->notifier_call(nb, NETDEV_DOWN, dev);
1507 }
1508 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001509 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001510 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001511
RongQing.Li8f891482011-11-30 23:43:07 -05001512outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001513 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001514 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001516EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
1518/**
1519 * unregister_netdevice_notifier - unregister a network notifier block
1520 * @nb: notifier
1521 *
1522 * Unregister a notifier previously registered by
1523 * register_netdevice_notifier(). The notifier is unlinked into the
1524 * kernel structures and may then be reused. A negative errno code
1525 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001526 *
1527 * After unregistering unregister and down device events are synthesized
1528 * for all devices on the device list to the removed notifier to remove
1529 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 */
1531
1532int unregister_netdevice_notifier(struct notifier_block *nb)
1533{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001534 struct net_device *dev;
1535 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001536 int err;
1537
1538 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001539 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001540 if (err)
1541 goto unlock;
1542
1543 for_each_net(net) {
1544 for_each_netdev(net, dev) {
1545 if (dev->flags & IFF_UP) {
1546 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1547 nb->notifier_call(nb, NETDEV_DOWN, dev);
1548 }
1549 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001550 }
1551 }
1552unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001553 rtnl_unlock();
1554 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001556EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557
1558/**
1559 * call_netdevice_notifiers - call all network notifier blocks
1560 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001561 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562 *
1563 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001564 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565 */
1566
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001567int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568{
Jiri Pirkoab930472010-04-20 01:45:37 -07001569 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001570 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001572EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573
Ingo Molnarc5905af2012-02-24 08:31:31 +01001574static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001575#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001576/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001577 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001578 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001579 */
1580static atomic_t netstamp_needed_deferred;
1581#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001582
1583void net_enable_timestamp(void)
1584{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001585#ifdef HAVE_JUMP_LABEL
1586 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1587
1588 if (deferred) {
1589 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001590 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001591 return;
1592 }
1593#endif
1594 WARN_ON(in_interrupt());
Ingo Molnarc5905af2012-02-24 08:31:31 +01001595 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001597EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598
1599void net_disable_timestamp(void)
1600{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001601#ifdef HAVE_JUMP_LABEL
1602 if (in_interrupt()) {
1603 atomic_inc(&netstamp_needed_deferred);
1604 return;
1605 }
1606#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001607 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001609EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610
Eric Dumazet3b098e22010-05-15 23:57:10 -07001611static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612{
Eric Dumazet588f0332011-11-15 04:12:55 +00001613 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001614 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001615 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616}
1617
Eric Dumazet588f0332011-11-15 04:12:55 +00001618#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001619 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001620 if ((COND) && !(SKB)->tstamp.tv64) \
1621 __net_timestamp(SKB); \
1622 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001623
Richard Cochran4dc360c2011-10-19 17:00:35 -04001624static int net_hwtstamp_validate(struct ifreq *ifr)
1625{
1626 struct hwtstamp_config cfg;
1627 enum hwtstamp_tx_types tx_type;
1628 enum hwtstamp_rx_filters rx_filter;
1629 int tx_type_valid = 0;
1630 int rx_filter_valid = 0;
1631
1632 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1633 return -EFAULT;
1634
1635 if (cfg.flags) /* reserved for future extensions */
1636 return -EINVAL;
1637
1638 tx_type = cfg.tx_type;
1639 rx_filter = cfg.rx_filter;
1640
1641 switch (tx_type) {
1642 case HWTSTAMP_TX_OFF:
1643 case HWTSTAMP_TX_ON:
1644 case HWTSTAMP_TX_ONESTEP_SYNC:
1645 tx_type_valid = 1;
1646 break;
1647 }
1648
1649 switch (rx_filter) {
1650 case HWTSTAMP_FILTER_NONE:
1651 case HWTSTAMP_FILTER_ALL:
1652 case HWTSTAMP_FILTER_SOME:
1653 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1654 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1655 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1656 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1657 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1658 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1659 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1660 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1661 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1662 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1663 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1664 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1665 rx_filter_valid = 1;
1666 break;
1667 }
1668
1669 if (!tx_type_valid || !rx_filter_valid)
1670 return -ERANGE;
1671
1672 return 0;
1673}
1674
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001675static inline bool is_skb_forwardable(struct net_device *dev,
1676 struct sk_buff *skb)
1677{
1678 unsigned int len;
1679
1680 if (!(dev->flags & IFF_UP))
1681 return false;
1682
1683 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1684 if (skb->len <= len)
1685 return true;
1686
1687 /* if TSO is enabled, we don't care about the length as the packet
1688 * could be forwarded without being segmented before
1689 */
1690 if (skb_is_gso(skb))
1691 return true;
1692
1693 return false;
1694}
1695
Arnd Bergmann44540962009-11-26 06:07:08 +00001696/**
1697 * dev_forward_skb - loopback an skb to another netif
1698 *
1699 * @dev: destination network device
1700 * @skb: buffer to forward
1701 *
1702 * return values:
1703 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001704 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001705 *
1706 * dev_forward_skb can be used for injecting an skb from the
1707 * start_xmit function of one device into the receive queue
1708 * of another device.
1709 *
1710 * The receiving device may be in another namespace, so
1711 * we have to clear all information in the skb that could
1712 * impact namespace isolation.
1713 */
1714int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1715{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001716 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1717 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1718 atomic_long_inc(&dev->rx_dropped);
1719 kfree_skb(skb);
1720 return NET_RX_DROP;
1721 }
1722 }
1723
Arnd Bergmann44540962009-11-26 06:07:08 +00001724 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001725 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001726
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001727 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001728 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001729 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001730 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001731 }
Benjamin LaHaise3b9785c2012-03-27 15:55:44 +00001732 skb->skb_iif = 0;
David S. Miller59b99972012-05-10 23:03:34 -04001733 skb->dev = dev;
1734 skb_dst_drop(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001735 skb->tstamp.tv64 = 0;
1736 skb->pkt_type = PACKET_HOST;
1737 skb->protocol = eth_type_trans(skb, dev);
David S. Miller59b99972012-05-10 23:03:34 -04001738 skb->mark = 0;
1739 secpath_reset(skb);
1740 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001741 return netif_rx(skb);
1742}
1743EXPORT_SYMBOL_GPL(dev_forward_skb);
1744
Changli Gao71d9dec2010-12-15 19:57:25 +00001745static inline int deliver_skb(struct sk_buff *skb,
1746 struct packet_type *pt_prev,
1747 struct net_device *orig_dev)
1748{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001749 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1750 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001751 atomic_inc(&skb->users);
1752 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1753}
1754
Eric Leblondc0de08d2012-08-16 22:02:58 +00001755static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1756{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001757 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001758 return false;
1759
1760 if (ptype->id_match)
1761 return ptype->id_match(ptype, skb->sk);
1762 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1763 return true;
1764
1765 return false;
1766}
1767
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768/*
1769 * Support routine. Sends outgoing frames to any network
1770 * taps currently in use.
1771 */
1772
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001773static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774{
1775 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001776 struct sk_buff *skb2 = NULL;
1777 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001778
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779 rcu_read_lock();
1780 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1781 /* Never send packets back to the socket
1782 * they originated from - MvS (miquels@drinkel.ow.org)
1783 */
1784 if ((ptype->dev == dev || !ptype->dev) &&
Eric Leblondc0de08d2012-08-16 22:02:58 +00001785 (!skb_loop_sk(ptype, skb))) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001786 if (pt_prev) {
1787 deliver_skb(skb2, pt_prev, skb->dev);
1788 pt_prev = ptype;
1789 continue;
1790 }
1791
1792 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793 if (!skb2)
1794 break;
1795
Eric Dumazet70978182010-12-20 21:22:51 +00001796 net_timestamp_set(skb2);
1797
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 /* skb->nh should be correctly
1799 set by sender, so that the second statement is
1800 just protection against buggy protocols.
1801 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001802 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001804 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001805 skb2->network_header > skb2->tail) {
Joe Perchese87cc472012-05-13 21:56:26 +00001806 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1807 ntohs(skb2->protocol),
1808 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001809 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001810 }
1811
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001812 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001814 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 }
1816 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001817 if (pt_prev)
1818 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819 rcu_read_unlock();
1820}
1821
Ben Hutchings2c530402012-07-10 10:55:09 +00001822/**
1823 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001824 * @dev: Network device
1825 * @txq: number of queues available
1826 *
1827 * If real_num_tx_queues is changed the tc mappings may no longer be
1828 * valid. To resolve this verify the tc mapping remains valid and if
1829 * not NULL the mapping. With no priorities mapping to this
1830 * offset/count pair it will no longer be used. In the worst case TC0
1831 * is invalid nothing can be done so disable priority mappings. If is
1832 * expected that drivers will fix this mapping if they can before
1833 * calling netif_set_real_num_tx_queues.
1834 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001835static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001836{
1837 int i;
1838 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1839
1840 /* If TC0 is invalidated disable TC mapping */
1841 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001842 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001843 dev->num_tc = 0;
1844 return;
1845 }
1846
1847 /* Invalidated prio to tc mappings set to TC0 */
1848 for (i = 1; i < TC_BITMASK + 1; i++) {
1849 int q = netdev_get_prio_tc_map(dev, i);
1850
1851 tc = &dev->tc_to_txq[q];
1852 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001853 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1854 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001855 netdev_set_prio_tc_map(dev, i, 0);
1856 }
1857 }
1858}
1859
John Fastabendf0796d52010-07-01 13:21:57 +00001860/*
1861 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1862 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1863 */
Tom Herberte6484932010-10-18 18:04:39 +00001864int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001865{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001866 int rc;
1867
Tom Herberte6484932010-10-18 18:04:39 +00001868 if (txq < 1 || txq > dev->num_tx_queues)
1869 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001870
Ben Hutchings5c565802011-02-15 19:39:21 +00001871 if (dev->reg_state == NETREG_REGISTERED ||
1872 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001873 ASSERT_RTNL();
1874
Tom Herbert1d24eb42010-11-21 13:17:27 +00001875 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1876 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001877 if (rc)
1878 return rc;
1879
John Fastabend4f57c082011-01-17 08:06:04 +00001880 if (dev->num_tc)
1881 netif_setup_tc(dev, txq);
1882
Tom Herberte6484932010-10-18 18:04:39 +00001883 if (txq < dev->real_num_tx_queues)
1884 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001885 }
Tom Herberte6484932010-10-18 18:04:39 +00001886
1887 dev->real_num_tx_queues = txq;
1888 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001889}
1890EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001891
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001892#ifdef CONFIG_RPS
1893/**
1894 * netif_set_real_num_rx_queues - set actual number of RX queues used
1895 * @dev: Network device
1896 * @rxq: Actual number of RX queues
1897 *
1898 * This must be called either with the rtnl_lock held or before
1899 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001900 * negative error code. If called before registration, it always
1901 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001902 */
1903int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1904{
1905 int rc;
1906
Tom Herbertbd25fa72010-10-18 18:00:16 +00001907 if (rxq < 1 || rxq > dev->num_rx_queues)
1908 return -EINVAL;
1909
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001910 if (dev->reg_state == NETREG_REGISTERED) {
1911 ASSERT_RTNL();
1912
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001913 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1914 rxq);
1915 if (rc)
1916 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001917 }
1918
1919 dev->real_num_rx_queues = rxq;
1920 return 0;
1921}
1922EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1923#endif
1924
Ben Hutchings2c530402012-07-10 10:55:09 +00001925/**
1926 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00001927 *
1928 * This routine should set an upper limit on the number of RSS queues
1929 * used by default by multiqueue devices.
1930 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00001931int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00001932{
1933 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
1934}
1935EXPORT_SYMBOL(netif_get_num_default_rss_queues);
1936
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001937static inline void __netif_reschedule(struct Qdisc *q)
1938{
1939 struct softnet_data *sd;
1940 unsigned long flags;
1941
1942 local_irq_save(flags);
1943 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001944 q->next_sched = NULL;
1945 *sd->output_queue_tailp = q;
1946 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001947 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1948 local_irq_restore(flags);
1949}
1950
David S. Miller37437bb2008-07-16 02:15:04 -07001951void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001952{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001953 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1954 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001955}
1956EXPORT_SYMBOL(__netif_schedule);
1957
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001958void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001959{
David S. Miller3578b0c2010-08-03 00:24:04 -07001960 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001961 struct softnet_data *sd;
1962 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001963
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001964 local_irq_save(flags);
1965 sd = &__get_cpu_var(softnet_data);
1966 skb->next = sd->completion_queue;
1967 sd->completion_queue = skb;
1968 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1969 local_irq_restore(flags);
1970 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001971}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001972EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001973
1974void dev_kfree_skb_any(struct sk_buff *skb)
1975{
1976 if (in_irq() || irqs_disabled())
1977 dev_kfree_skb_irq(skb);
1978 else
1979 dev_kfree_skb(skb);
1980}
1981EXPORT_SYMBOL(dev_kfree_skb_any);
1982
1983
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001984/**
1985 * netif_device_detach - mark device as removed
1986 * @dev: network device
1987 *
1988 * Mark device as removed from system and therefore no longer available.
1989 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001990void netif_device_detach(struct net_device *dev)
1991{
1992 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1993 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001994 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001995 }
1996}
1997EXPORT_SYMBOL(netif_device_detach);
1998
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001999/**
2000 * netif_device_attach - mark device as attached
2001 * @dev: network device
2002 *
2003 * Mark device as attached from system and restart if needed.
2004 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002005void netif_device_attach(struct net_device *dev)
2006{
2007 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2008 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002009 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002010 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002011 }
2012}
2013EXPORT_SYMBOL(netif_device_attach);
2014
Ben Hutchings36c92472012-01-17 07:57:56 +00002015static void skb_warn_bad_offload(const struct sk_buff *skb)
2016{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002017 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002018 struct net_device *dev = skb->dev;
2019 const char *driver = "";
2020
2021 if (dev && dev->dev.parent)
2022 driver = dev_driver_string(dev->dev.parent);
2023
2024 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2025 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002026 driver, dev ? &dev->features : &null_features,
2027 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002028 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2029 skb_shinfo(skb)->gso_type, skb->ip_summed);
2030}
2031
Linus Torvalds1da177e2005-04-16 15:20:36 -07002032/*
2033 * Invalidate hardware checksum when packet is to be mangled, and
2034 * complete checksum manually on outgoing path.
2035 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002036int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037{
Al Virod3bc23e2006-11-14 21:24:49 -08002038 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002039 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002040
Patrick McHardy84fa7932006-08-29 16:44:56 -07002041 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002042 goto out_set_summed;
2043
2044 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002045 skb_warn_bad_offload(skb);
2046 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002047 }
2048
Michał Mirosław55508d62010-12-14 15:24:08 +00002049 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002050 BUG_ON(offset >= skb_headlen(skb));
2051 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2052
2053 offset += skb->csum_offset;
2054 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2055
2056 if (skb_cloned(skb) &&
2057 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2059 if (ret)
2060 goto out;
2061 }
2062
Herbert Xua0308472007-10-15 01:47:15 -07002063 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002064out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002066out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 return ret;
2068}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002069EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002071/**
2072 * skb_gso_segment - Perform segmentation on skb.
2073 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002074 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002075 *
2076 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002077 *
2078 * It may return NULL if the skb requires no segmentation. This is
2079 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002080 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002081struct sk_buff *skb_gso_segment(struct sk_buff *skb,
2082 netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002083{
2084 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
Vlad Yasevich22061d82012-11-15 08:49:11 +00002085 struct packet_offload *ptype;
Al Viro252e3342006-11-14 20:48:11 -08002086 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002087 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07002088 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002089
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002090 while (type == htons(ETH_P_8021Q)) {
2091 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00002092
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002093 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00002094 return ERR_PTR(-EINVAL);
2095
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002096 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2097 type = vh->h_vlan_encapsulated_proto;
2098 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00002099 }
2100
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07002101 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07002102 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002103 __skb_pull(skb, skb->mac_len);
2104
Herbert Xu67fd1a72009-01-19 16:26:44 -08002105 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002106 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002107
Herbert Xua430a432006-07-08 13:34:56 -07002108 if (skb_header_cloned(skb) &&
2109 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2110 return ERR_PTR(err);
2111 }
2112
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002113 rcu_read_lock();
Vlad Yasevich22061d82012-11-15 08:49:11 +00002114 list_for_each_entry_rcu(ptype, &offload_base, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002115 if (ptype->type == type && ptype->callbacks.gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07002116 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002117 err = ptype->callbacks.gso_send_check(skb);
Herbert Xua430a432006-07-08 13:34:56 -07002118 segs = ERR_PTR(err);
2119 if (err || skb_gso_ok(skb, features))
2120 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07002121 __skb_push(skb, (skb->data -
2122 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07002123 }
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002124 segs = ptype->callbacks.gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002125 break;
2126 }
2127 }
2128 rcu_read_unlock();
2129
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002130 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07002131
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002132 return segs;
2133}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002134EXPORT_SYMBOL(skb_gso_segment);
2135
Herbert Xufb286bb2005-11-10 13:01:24 -08002136/* Take action when hardware reception checksum errors are detected. */
2137#ifdef CONFIG_BUG
2138void netdev_rx_csum_fault(struct net_device *dev)
2139{
2140 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002141 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002142 dump_stack();
2143 }
2144}
2145EXPORT_SYMBOL(netdev_rx_csum_fault);
2146#endif
2147
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148/* Actually, we should eliminate this check as soon as we know, that:
2149 * 1. IOMMU is present and allows to map all the memory.
2150 * 2. No high memory really exists on this machine.
2151 */
2152
Eric Dumazet9092c652010-04-02 13:34:49 -07002153static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002154{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002155#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002156 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002157 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002158 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2159 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2160 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002161 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002162 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002163 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002164
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002165 if (PCI_DMA_BUS_IS_PHYS) {
2166 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167
Eric Dumazet9092c652010-04-02 13:34:49 -07002168 if (!pdev)
2169 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002170 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002171 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2172 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002173 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2174 return 1;
2175 }
2176 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002177#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002178 return 0;
2179}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002180
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002181struct dev_gso_cb {
2182 void (*destructor)(struct sk_buff *skb);
2183};
2184
2185#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2186
2187static void dev_gso_skb_destructor(struct sk_buff *skb)
2188{
2189 struct dev_gso_cb *cb;
2190
2191 do {
2192 struct sk_buff *nskb = skb->next;
2193
2194 skb->next = nskb->next;
2195 nskb->next = NULL;
2196 kfree_skb(nskb);
2197 } while (skb->next);
2198
2199 cb = DEV_GSO_CB(skb);
2200 if (cb->destructor)
2201 cb->destructor(skb);
2202}
2203
2204/**
2205 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2206 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002207 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002208 *
2209 * This function segments the given skb and stores the list of segments
2210 * in skb->next.
2211 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002212static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002213{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002214 struct sk_buff *segs;
2215
Herbert Xu576a30e2006-06-27 13:22:38 -07002216 segs = skb_gso_segment(skb, features);
2217
2218 /* Verifying header integrity only. */
2219 if (!segs)
2220 return 0;
2221
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002222 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002223 return PTR_ERR(segs);
2224
2225 skb->next = segs;
2226 DEV_GSO_CB(skb)->destructor = skb->destructor;
2227 skb->destructor = dev_gso_skb_destructor;
2228
2229 return 0;
2230}
2231
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002232static bool can_checksum_protocol(netdev_features_t features, __be16 protocol)
Jesse Gross03634662011-01-09 06:23:35 +00002233{
2234 return ((features & NETIF_F_GEN_CSUM) ||
2235 ((features & NETIF_F_V4_CSUM) &&
2236 protocol == htons(ETH_P_IP)) ||
2237 ((features & NETIF_F_V6_CSUM) &&
2238 protocol == htons(ETH_P_IPV6)) ||
2239 ((features & NETIF_F_FCOE_CRC) &&
2240 protocol == htons(ETH_P_FCOE)));
2241}
2242
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002243static netdev_features_t harmonize_features(struct sk_buff *skb,
2244 __be16 protocol, netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002245{
Ed Cashinc0d680e2012-09-19 15:49:00 +00002246 if (skb->ip_summed != CHECKSUM_NONE &&
2247 !can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002248 features &= ~NETIF_F_ALL_CSUM;
2249 features &= ~NETIF_F_SG;
2250 } else if (illegal_highdma(skb->dev, skb)) {
2251 features &= ~NETIF_F_SG;
2252 }
2253
2254 return features;
2255}
2256
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002257netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002258{
2259 __be16 protocol = skb->protocol;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002260 netdev_features_t features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002261
Ben Hutchings30b678d2012-07-30 15:57:00 +00002262 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2263 features &= ~NETIF_F_GSO_MASK;
2264
Jesse Gross58e998c2010-10-29 12:14:55 +00002265 if (protocol == htons(ETH_P_8021Q)) {
2266 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2267 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002268 } else if (!vlan_tx_tag_present(skb)) {
2269 return harmonize_features(skb, protocol, features);
2270 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002271
Jesse Gross6ee400a2011-01-17 20:46:00 +00002272 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002273
2274 if (protocol != htons(ETH_P_8021Q)) {
2275 return harmonize_features(skb, protocol, features);
2276 } else {
2277 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002278 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002279 return harmonize_features(skb, protocol, features);
2280 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002281}
Jesse Grossf01a5232011-01-09 06:23:31 +00002282EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002283
John Fastabend6afff0c2010-06-16 14:18:12 +00002284/*
2285 * Returns true if either:
2286 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
Rami Rosend1a53df2012-08-27 23:39:24 +00002287 * 2. skb is fragmented and the device does not support SG.
John Fastabend6afff0c2010-06-16 14:18:12 +00002288 */
2289static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002290 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002291{
Jesse Gross02932ce2011-01-09 06:23:34 +00002292 return skb_is_nonlinear(skb) &&
2293 ((skb_has_frag_list(skb) &&
2294 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002295 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002296 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002297}
2298
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002299int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2300 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002301{
Stephen Hemminger00829822008-11-20 20:14:53 -08002302 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002303 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002304 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002305
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002306 if (likely(!skb->next)) {
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002307 netdev_features_t features;
Jesse Grossfc741212011-01-09 06:23:32 +00002308
Eric Dumazet93f154b2009-05-18 22:19:19 -07002309 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002310 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002311 * its hot in this cpu cache
2312 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002313 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2314 skb_dst_drop(skb);
2315
Jesse Grossfc741212011-01-09 06:23:32 +00002316 features = netif_skb_features(skb);
2317
Jesse Gross7b9c6092010-10-20 13:56:04 +00002318 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002319 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002320 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2321 if (unlikely(!skb))
2322 goto out;
2323
2324 skb->vlan_tci = 0;
2325 }
2326
Jesse Grossfc741212011-01-09 06:23:32 +00002327 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002328 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002329 goto out_kfree_skb;
2330 if (skb->next)
2331 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002332 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002333 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002334 __skb_linearize(skb))
2335 goto out_kfree_skb;
2336
2337 /* If packet is not checksummed and device does not
2338 * support checksumming for this protocol, complete
2339 * checksumming here.
2340 */
2341 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Michał Mirosław55508d62010-12-14 15:24:08 +00002342 skb_set_transport_header(skb,
2343 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002344 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002345 skb_checksum_help(skb))
2346 goto out_kfree_skb;
2347 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002348 }
2349
Eric Dumazetb40863c2012-09-18 20:44:49 +00002350 if (!list_empty(&ptype_all))
2351 dev_queue_xmit_nit(skb, dev);
2352
Koki Sanagiec764bf2011-05-30 21:48:34 +00002353 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002354 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002355 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002356 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002357 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002358 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002359 }
2360
Herbert Xu576a30e2006-06-27 13:22:38 -07002361gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002362 do {
2363 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002364
2365 skb->next = nskb->next;
2366 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002367
2368 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002369 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002370 * its hot in this cpu cache
2371 */
2372 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2373 skb_dst_drop(nskb);
2374
Eric Dumazetb40863c2012-09-18 20:44:49 +00002375 if (!list_empty(&ptype_all))
2376 dev_queue_xmit_nit(nskb, dev);
2377
Koki Sanagiec764bf2011-05-30 21:48:34 +00002378 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002379 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002380 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002381 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002382 if (rc & ~NETDEV_TX_MASK)
2383 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002384 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002385 skb->next = nskb;
2386 return rc;
2387 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002388 txq_trans_update(txq);
Tom Herbert734664982011-11-28 16:32:44 +00002389 if (unlikely(netif_xmit_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002390 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002391 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002392
Patrick McHardy572a9d72009-11-10 06:14:14 +00002393out_kfree_gso_skb:
2394 if (likely(skb->next == NULL))
2395 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002396out_kfree_skb:
2397 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002398out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002399 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002400}
2401
Tom Herbert0a9627f2010-03-16 08:03:29 +00002402static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002403
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002404/*
2405 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2406 * to be used as a distribution range.
2407 */
2408u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2409 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002410{
David S. Miller70192982009-01-27 16:34:47 -08002411 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002412 u16 qoffset = 0;
2413 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002414
David S. Miller513de112009-05-03 14:43:10 -07002415 if (skb_rx_queue_recorded(skb)) {
2416 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002417 while (unlikely(hash >= num_tx_queues))
2418 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002419 return hash;
2420 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002421
John Fastabend4f57c082011-01-17 08:06:04 +00002422 if (dev->num_tc) {
2423 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2424 qoffset = dev->tc_to_txq[tc].offset;
2425 qcount = dev->tc_to_txq[tc].count;
2426 }
2427
Eric Dumazetec581f62009-05-01 09:05:06 -07002428 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002429 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002430 else
Eric Dumazet62b1a8a2012-06-14 06:42:44 +00002431 hash = (__force u16) skb->protocol;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002432 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002433
John Fastabend4f57c082011-01-17 08:06:04 +00002434 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002435}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002436EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002437
Eric Dumazeted046422009-11-13 21:54:04 +00002438static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2439{
2440 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
Joe Perchese87cc472012-05-13 21:56:26 +00002441 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2442 dev->name, queue_index,
2443 dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002444 return 0;
2445 }
2446 return queue_index;
2447}
2448
Tom Herbert1d24eb42010-11-21 13:17:27 +00002449static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2450{
Tom Herbertbf264142010-11-26 08:36:09 +00002451#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002452 struct xps_dev_maps *dev_maps;
2453 struct xps_map *map;
2454 int queue_index = -1;
2455
2456 rcu_read_lock();
2457 dev_maps = rcu_dereference(dev->xps_maps);
2458 if (dev_maps) {
2459 map = rcu_dereference(
2460 dev_maps->cpu_map[raw_smp_processor_id()]);
2461 if (map) {
2462 if (map->len == 1)
2463 queue_index = map->queues[0];
2464 else {
2465 u32 hash;
2466 if (skb->sk && skb->sk->sk_hash)
2467 hash = skb->sk->sk_hash;
2468 else
2469 hash = (__force u16) skb->protocol ^
2470 skb->rxhash;
2471 hash = jhash_1word(hash, hashrnd);
2472 queue_index = map->queues[
2473 ((u64)hash * map->len) >> 32];
2474 }
2475 if (unlikely(queue_index >= dev->real_num_tx_queues))
2476 queue_index = -1;
2477 }
2478 }
2479 rcu_read_unlock();
2480
2481 return queue_index;
2482#else
2483 return -1;
2484#endif
2485}
2486
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002487struct netdev_queue *netdev_pick_tx(struct net_device *dev,
2488 struct sk_buff *skb)
David S. Millere8a04642008-07-17 00:34:19 -07002489{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002490 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002491 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002492
Tom Herbert3853b582010-11-21 13:17:29 +00002493 if (dev->real_num_tx_queues == 1)
2494 queue_index = 0;
2495 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002496 queue_index = ops->ndo_select_queue(dev, skb);
2497 queue_index = dev_cap_txqueue(dev, queue_index);
2498 } else {
2499 struct sock *sk = skb->sk;
2500 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002501
Tom Herbert3853b582010-11-21 13:17:29 +00002502 if (queue_index < 0 || skb->ooo_okay ||
2503 queue_index >= dev->real_num_tx_queues) {
2504 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002505
Tom Herbert1d24eb42010-11-21 13:17:27 +00002506 queue_index = get_xps_queue(dev, skb);
2507 if (queue_index < 0)
2508 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002509
2510 if (queue_index != old_index && sk) {
2511 struct dst_entry *dst =
2512 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002513
2514 if (dst && skb_dst(skb) == dst)
2515 sk_tx_queue_set(sk, queue_index);
2516 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002517 }
2518 }
David S. Millereae792b2008-07-15 03:03:33 -07002519
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002520 skb_set_queue_mapping(skb, queue_index);
2521 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002522}
2523
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002524static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2525 struct net_device *dev,
2526 struct netdev_queue *txq)
2527{
2528 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002529 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002530 int rc;
2531
Eric Dumazeta2da5702011-01-20 03:48:19 +00002532 qdisc_skb_cb(skb)->pkt_len = skb->len;
2533 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002534 /*
2535 * Heuristic to force contended enqueues to serialize on a
2536 * separate lock before trying to get qdisc main lock.
2537 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2538 * and dequeue packets faster.
2539 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002540 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002541 if (unlikely(contended))
2542 spin_lock(&q->busylock);
2543
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002544 spin_lock(root_lock);
2545 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2546 kfree_skb(skb);
2547 rc = NET_XMIT_DROP;
2548 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002549 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002550 /*
2551 * This is a work-conserving queue; there are no old skbs
2552 * waiting to be sent out; and the qdisc is not running -
2553 * xmit the skb directly.
2554 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002555 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2556 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002557
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002558 qdisc_bstats_update(q, skb);
2559
Eric Dumazet79640a42010-06-02 05:09:29 -07002560 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2561 if (unlikely(contended)) {
2562 spin_unlock(&q->busylock);
2563 contended = false;
2564 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002565 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002566 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002567 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002568
2569 rc = NET_XMIT_SUCCESS;
2570 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002571 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002572 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002573 if (qdisc_run_begin(q)) {
2574 if (unlikely(contended)) {
2575 spin_unlock(&q->busylock);
2576 contended = false;
2577 }
2578 __qdisc_run(q);
2579 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002580 }
2581 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002582 if (unlikely(contended))
2583 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002584 return rc;
2585}
2586
Neil Horman5bc14212011-11-22 05:10:51 +00002587#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2588static void skb_update_prio(struct sk_buff *skb)
2589{
Igor Maravic6977a792011-11-25 07:44:54 +00002590 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002591
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002592 if (!skb->priority && skb->sk && map) {
2593 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2594
2595 if (prioidx < map->priomap_len)
2596 skb->priority = map->priomap[prioidx];
2597 }
Neil Horman5bc14212011-11-22 05:10:51 +00002598}
2599#else
2600#define skb_update_prio(skb)
2601#endif
2602
Eric Dumazet745e20f2010-09-29 13:23:09 -07002603static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002604#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002605
Dave Jonesd29f7492008-07-22 14:09:06 -07002606/**
Michel Machado95603e22012-06-12 10:16:35 +00002607 * dev_loopback_xmit - loop back @skb
2608 * @skb: buffer to transmit
2609 */
2610int dev_loopback_xmit(struct sk_buff *skb)
2611{
2612 skb_reset_mac_header(skb);
2613 __skb_pull(skb, skb_network_offset(skb));
2614 skb->pkt_type = PACKET_LOOPBACK;
2615 skb->ip_summed = CHECKSUM_UNNECESSARY;
2616 WARN_ON(!skb_dst(skb));
2617 skb_dst_force(skb);
2618 netif_rx_ni(skb);
2619 return 0;
2620}
2621EXPORT_SYMBOL(dev_loopback_xmit);
2622
2623/**
Dave Jonesd29f7492008-07-22 14:09:06 -07002624 * dev_queue_xmit - transmit a buffer
2625 * @skb: buffer to transmit
2626 *
2627 * Queue a buffer for transmission to a network device. The caller must
2628 * have set the device and priority and built the buffer before calling
2629 * this function. The function can be called from an interrupt.
2630 *
2631 * A negative errno code is returned on a failure. A success does not
2632 * guarantee the frame will be transmitted as it may be dropped due
2633 * to congestion or traffic shaping.
2634 *
2635 * -----------------------------------------------------------------------------------
2636 * I notice this method can also return errors from the queue disciplines,
2637 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2638 * be positive.
2639 *
2640 * Regardless of the return value, the skb is consumed, so it is currently
2641 * difficult to retry a send to this method. (You can bump the ref count
2642 * before sending to hold a reference for retry if you are careful.)
2643 *
2644 * When calling this method, interrupts MUST be enabled. This is because
2645 * the BH enable code must have IRQs enabled so that it will not deadlock.
2646 * --BLG
2647 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002648int dev_queue_xmit(struct sk_buff *skb)
2649{
2650 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002651 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002652 struct Qdisc *q;
2653 int rc = -ENOMEM;
2654
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002655 /* Disable soft irqs for various locks below. Also
2656 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002657 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002658 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002659
Neil Horman5bc14212011-11-22 05:10:51 +00002660 skb_update_prio(skb);
2661
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002662 txq = netdev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002663 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002664
Linus Torvalds1da177e2005-04-16 15:20:36 -07002665#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002666 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002667#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002668 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002669 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002670 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002671 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002672 }
2673
2674 /* The device has no queue. Common case for software devices:
2675 loopback, all the sorts of tunnels...
2676
Herbert Xu932ff272006-06-09 12:20:56 -07002677 Really, it is unlikely that netif_tx_lock protection is necessary
2678 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679 counters.)
2680 However, it is possible, that they rely on protection
2681 made by us here.
2682
2683 Check this and shot the lock. It is not prone from deadlocks.
2684 Either shot noqueue qdisc, it is even simpler 8)
2685 */
2686 if (dev->flags & IFF_UP) {
2687 int cpu = smp_processor_id(); /* ok because BHs are off */
2688
David S. Millerc773e842008-07-08 23:13:53 -07002689 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690
Eric Dumazet745e20f2010-09-29 13:23:09 -07002691 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2692 goto recursion_alert;
2693
David S. Millerc773e842008-07-08 23:13:53 -07002694 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002695
Tom Herbert734664982011-11-28 16:32:44 +00002696 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002697 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002698 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002699 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002700 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002701 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702 goto out;
2703 }
2704 }
David S. Millerc773e842008-07-08 23:13:53 -07002705 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002706 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2707 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002708 } else {
2709 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002710 * unfortunately
2711 */
2712recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00002713 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2714 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002715 }
2716 }
2717
2718 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002719 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720
Linus Torvalds1da177e2005-04-16 15:20:36 -07002721 kfree_skb(skb);
2722 return rc;
2723out:
Herbert Xud4828d82006-06-22 02:28:18 -07002724 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002725 return rc;
2726}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002727EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002728
2729
2730/*=======================================================================
2731 Receiver routines
2732 =======================================================================*/
2733
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002734int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00002735EXPORT_SYMBOL(netdev_max_backlog);
2736
Eric Dumazet3b098e22010-05-15 23:57:10 -07002737int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002738int netdev_budget __read_mostly = 300;
2739int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002740
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002741/* Called with irq disabled */
2742static inline void ____napi_schedule(struct softnet_data *sd,
2743 struct napi_struct *napi)
2744{
2745 list_add_tail(&napi->poll_list, &sd->poll_list);
2746 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2747}
2748
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002749/*
2750 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
Tom Herbertbdeab992011-08-14 19:45:55 +00002751 * and src/dst port numbers. Sets rxhash in skb to non-zero hash value
2752 * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
2753 * if hash is a canonical 4-tuple hash over transport ports.
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002754 */
Tom Herbertbdeab992011-08-14 19:45:55 +00002755void __skb_get_rxhash(struct sk_buff *skb)
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002756{
Eric Dumazet4504b862011-11-28 05:23:23 +00002757 struct flow_keys keys;
2758 u32 hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002759
Eric Dumazet4504b862011-11-28 05:23:23 +00002760 if (!skb_flow_dissect(skb, &keys))
2761 return;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002762
Chema Gonzalez68622342012-09-07 13:40:50 +00002763 if (keys.ports)
Eric Dumazet4504b862011-11-28 05:23:23 +00002764 skb->l4_rxhash = 1;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002765
2766 /* get a consistent hash (same value on both flow directions) */
Chema Gonzalez68622342012-09-07 13:40:50 +00002767 if (((__force u32)keys.dst < (__force u32)keys.src) ||
2768 (((__force u32)keys.dst == (__force u32)keys.src) &&
2769 ((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) {
Eric Dumazet4504b862011-11-28 05:23:23 +00002770 swap(keys.dst, keys.src);
Chema Gonzalez68622342012-09-07 13:40:50 +00002771 swap(keys.port16[0], keys.port16[1]);
2772 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002773
Eric Dumazet4504b862011-11-28 05:23:23 +00002774 hash = jhash_3words((__force u32)keys.dst,
2775 (__force u32)keys.src,
2776 (__force u32)keys.ports, hashrnd);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002777 if (!hash)
2778 hash = 1;
2779
Tom Herbertbdeab992011-08-14 19:45:55 +00002780 skb->rxhash = hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002781}
2782EXPORT_SYMBOL(__skb_get_rxhash);
2783
Eric Dumazetdf334542010-03-24 19:13:54 +00002784#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002785
2786/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002787struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002788EXPORT_SYMBOL(rps_sock_flow_table);
2789
Ingo Molnarc5905af2012-02-24 08:31:31 +01002790struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00002791
Ben Hutchingsc4454772011-01-19 11:03:53 +00002792static struct rps_dev_flow *
2793set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2794 struct rps_dev_flow *rflow, u16 next_cpu)
2795{
Ben Hutchings09994d12011-10-03 04:42:46 +00002796 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002797#ifdef CONFIG_RFS_ACCEL
2798 struct netdev_rx_queue *rxqueue;
2799 struct rps_dev_flow_table *flow_table;
2800 struct rps_dev_flow *old_rflow;
2801 u32 flow_id;
2802 u16 rxq_index;
2803 int rc;
2804
2805 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002806 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2807 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002808 goto out;
2809 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2810 if (rxq_index == skb_get_rx_queue(skb))
2811 goto out;
2812
2813 rxqueue = dev->_rx + rxq_index;
2814 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2815 if (!flow_table)
2816 goto out;
2817 flow_id = skb->rxhash & flow_table->mask;
2818 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2819 rxq_index, flow_id);
2820 if (rc < 0)
2821 goto out;
2822 old_rflow = rflow;
2823 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002824 rflow->filter = rc;
2825 if (old_rflow->filter == rflow->filter)
2826 old_rflow->filter = RPS_NO_FILTER;
2827 out:
2828#endif
2829 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002830 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002831 }
2832
Ben Hutchings09994d12011-10-03 04:42:46 +00002833 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002834 return rflow;
2835}
2836
Tom Herbert0a9627f2010-03-16 08:03:29 +00002837/*
2838 * get_rps_cpu is called from netif_receive_skb and returns the target
2839 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002840 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002841 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002842static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2843 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002844{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002845 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002846 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002847 struct rps_dev_flow_table *flow_table;
2848 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002849 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002850 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002851
Tom Herbert0a9627f2010-03-16 08:03:29 +00002852 if (skb_rx_queue_recorded(skb)) {
2853 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002854 if (unlikely(index >= dev->real_num_rx_queues)) {
2855 WARN_ONCE(dev->real_num_rx_queues > 1,
2856 "%s received packet on queue %u, but number "
2857 "of RX queues is %u\n",
2858 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002859 goto done;
2860 }
2861 rxqueue = dev->_rx + index;
2862 } else
2863 rxqueue = dev->_rx;
2864
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002865 map = rcu_dereference(rxqueue->rps_map);
2866 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002867 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002868 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002869 tcpu = map->cpus[0];
2870 if (cpu_online(tcpu))
2871 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002872 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002873 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002874 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002875 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002876 }
2877
Changli Gao2d47b452010-08-17 19:00:56 +00002878 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002879 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002880 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002881
Tom Herbertfec5e652010-04-16 16:01:27 -07002882 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2883 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2884 if (flow_table && sock_flow_table) {
2885 u16 next_cpu;
2886 struct rps_dev_flow *rflow;
2887
2888 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2889 tcpu = rflow->cpu;
2890
2891 next_cpu = sock_flow_table->ents[skb->rxhash &
2892 sock_flow_table->mask];
2893
2894 /*
2895 * If the desired CPU (where last recvmsg was done) is
2896 * different from current CPU (one in the rx-queue flow
2897 * table entry), switch if one of the following holds:
2898 * - Current CPU is unset (equal to RPS_NO_CPU).
2899 * - Current CPU is offline.
2900 * - The current CPU's queue tail has advanced beyond the
2901 * last packet that was enqueued using this table entry.
2902 * This guarantees that all previous packets for the flow
2903 * have been dequeued, thus preserving in order delivery.
2904 */
2905 if (unlikely(tcpu != next_cpu) &&
2906 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2907 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00002908 rflow->last_qtail)) >= 0)) {
2909 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002910 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00002911 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00002912
Tom Herbertfec5e652010-04-16 16:01:27 -07002913 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2914 *rflowp = rflow;
2915 cpu = tcpu;
2916 goto done;
2917 }
2918 }
2919
Tom Herbert0a9627f2010-03-16 08:03:29 +00002920 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002921 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002922
2923 if (cpu_online(tcpu)) {
2924 cpu = tcpu;
2925 goto done;
2926 }
2927 }
2928
2929done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002930 return cpu;
2931}
2932
Ben Hutchingsc4454772011-01-19 11:03:53 +00002933#ifdef CONFIG_RFS_ACCEL
2934
2935/**
2936 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2937 * @dev: Device on which the filter was set
2938 * @rxq_index: RX queue index
2939 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2940 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2941 *
2942 * Drivers that implement ndo_rx_flow_steer() should periodically call
2943 * this function for each installed filter and remove the filters for
2944 * which it returns %true.
2945 */
2946bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2947 u32 flow_id, u16 filter_id)
2948{
2949 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2950 struct rps_dev_flow_table *flow_table;
2951 struct rps_dev_flow *rflow;
2952 bool expire = true;
2953 int cpu;
2954
2955 rcu_read_lock();
2956 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2957 if (flow_table && flow_id <= flow_table->mask) {
2958 rflow = &flow_table->flows[flow_id];
2959 cpu = ACCESS_ONCE(rflow->cpu);
2960 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2961 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2962 rflow->last_qtail) <
2963 (int)(10 * flow_table->mask)))
2964 expire = false;
2965 }
2966 rcu_read_unlock();
2967 return expire;
2968}
2969EXPORT_SYMBOL(rps_may_expire_flow);
2970
2971#endif /* CONFIG_RFS_ACCEL */
2972
Tom Herbert0a9627f2010-03-16 08:03:29 +00002973/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002974static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002975{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002976 struct softnet_data *sd = data;
2977
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002978 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002979 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002980}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002981
Tom Herbertfec5e652010-04-16 16:01:27 -07002982#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002983
2984/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002985 * Check if this softnet_data structure is another cpu one
2986 * If yes, queue it to our IPI list and return 1
2987 * If no, return 0
2988 */
2989static int rps_ipi_queued(struct softnet_data *sd)
2990{
2991#ifdef CONFIG_RPS
2992 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2993
2994 if (sd != mysd) {
2995 sd->rps_ipi_next = mysd->rps_ipi_list;
2996 mysd->rps_ipi_list = sd;
2997
2998 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2999 return 1;
3000 }
3001#endif /* CONFIG_RPS */
3002 return 0;
3003}
3004
3005/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003006 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3007 * queue (may be a remote CPU queue).
3008 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003009static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3010 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003011{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003012 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003013 unsigned long flags;
3014
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003015 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003016
3017 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003018
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003019 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003020 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
3021 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003022enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003023 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003024 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003025 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003026 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003027 return NET_RX_SUCCESS;
3028 }
3029
Eric Dumazetebda37c22010-05-06 23:51:21 +00003030 /* Schedule NAPI for backlog device
3031 * We can use non atomic operation since we own the queue lock
3032 */
3033 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003034 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003035 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003036 }
3037 goto enqueue;
3038 }
3039
Changli Gaodee42872010-05-02 05:42:16 +00003040 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003041 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003042
Tom Herbert0a9627f2010-03-16 08:03:29 +00003043 local_irq_restore(flags);
3044
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003045 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003046 kfree_skb(skb);
3047 return NET_RX_DROP;
3048}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003049
Linus Torvalds1da177e2005-04-16 15:20:36 -07003050/**
3051 * netif_rx - post buffer to the network code
3052 * @skb: buffer to post
3053 *
3054 * This function receives a packet from a device driver and queues it for
3055 * the upper (protocol) levels to process. It always succeeds. The buffer
3056 * may be dropped during processing for congestion control or by the
3057 * protocol layers.
3058 *
3059 * return values:
3060 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003061 * NET_RX_DROP (packet was dropped)
3062 *
3063 */
3064
3065int netif_rx(struct sk_buff *skb)
3066{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003067 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003068
3069 /* if netpoll wants it, pretend we never saw it */
3070 if (netpoll_rx(skb))
3071 return NET_RX_DROP;
3072
Eric Dumazet588f0332011-11-15 04:12:55 +00003073 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074
Koki Sanagicf66ba52010-08-23 18:45:02 +09003075 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003076#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003077 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003078 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003079 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080
Changli Gaocece1942010-08-07 20:35:43 -07003081 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003082 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003083
3084 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003085 if (cpu < 0)
3086 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003087
3088 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3089
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003090 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003091 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003092 } else
3093#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003094 {
3095 unsigned int qtail;
3096 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3097 put_cpu();
3098 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003099 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003100}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003101EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003102
3103int netif_rx_ni(struct sk_buff *skb)
3104{
3105 int err;
3106
3107 preempt_disable();
3108 err = netif_rx(skb);
3109 if (local_softirq_pending())
3110 do_softirq();
3111 preempt_enable();
3112
3113 return err;
3114}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003115EXPORT_SYMBOL(netif_rx_ni);
3116
Linus Torvalds1da177e2005-04-16 15:20:36 -07003117static void net_tx_action(struct softirq_action *h)
3118{
3119 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3120
3121 if (sd->completion_queue) {
3122 struct sk_buff *clist;
3123
3124 local_irq_disable();
3125 clist = sd->completion_queue;
3126 sd->completion_queue = NULL;
3127 local_irq_enable();
3128
3129 while (clist) {
3130 struct sk_buff *skb = clist;
3131 clist = clist->next;
3132
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003133 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09003134 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003135 __kfree_skb(skb);
3136 }
3137 }
3138
3139 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003140 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141
3142 local_irq_disable();
3143 head = sd->output_queue;
3144 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003145 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003146 local_irq_enable();
3147
3148 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003149 struct Qdisc *q = head;
3150 spinlock_t *root_lock;
3151
Linus Torvalds1da177e2005-04-16 15:20:36 -07003152 head = head->next_sched;
3153
David S. Miller5fb66222008-08-02 20:02:43 -07003154 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003155 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003156 smp_mb__before_clear_bit();
3157 clear_bit(__QDISC_STATE_SCHED,
3158 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003159 qdisc_run(q);
3160 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003162 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003163 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003164 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003165 } else {
3166 smp_mb__before_clear_bit();
3167 clear_bit(__QDISC_STATE_SCHED,
3168 &q->state);
3169 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003170 }
3171 }
3172 }
3173}
3174
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003175#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3176 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003177/* This hook is defined here for ATM LANE */
3178int (*br_fdb_test_addr_hook)(struct net_device *dev,
3179 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003180EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003181#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182
Linus Torvalds1da177e2005-04-16 15:20:36 -07003183#ifdef CONFIG_NET_CLS_ACT
3184/* TODO: Maybe we should just force sch_ingress to be compiled in
3185 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3186 * a compare and 2 stores extra right now if we dont have it on
3187 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003188 * NOTE: This doesn't stop any functionality; if you dont have
3189 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003190 *
3191 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003192static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003193{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003194 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003195 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003196 int result = TC_ACT_OK;
3197 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003198
Stephen Hemmingerde384832010-08-01 00:33:23 -07003199 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003200 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3201 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003202 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003203 }
3204
Herbert Xuf697c3e2007-10-14 00:38:47 -07003205 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3206 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3207
David S. Miller83874002008-07-17 00:53:03 -07003208 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003209 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003210 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003211 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3212 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003213 spin_unlock(qdisc_lock(q));
3214 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003215
Linus Torvalds1da177e2005-04-16 15:20:36 -07003216 return result;
3217}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003218
3219static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3220 struct packet_type **pt_prev,
3221 int *ret, struct net_device *orig_dev)
3222{
Eric Dumazet24824a02010-10-02 06:11:55 +00003223 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3224
3225 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003226 goto out;
3227
3228 if (*pt_prev) {
3229 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3230 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003231 }
3232
Eric Dumazet24824a02010-10-02 06:11:55 +00003233 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003234 case TC_ACT_SHOT:
3235 case TC_ACT_STOLEN:
3236 kfree_skb(skb);
3237 return NULL;
3238 }
3239
3240out:
3241 skb->tc_verd = 0;
3242 return skb;
3243}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003244#endif
3245
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003246/**
3247 * netdev_rx_handler_register - register receive handler
3248 * @dev: device to register a handler for
3249 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003250 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003251 *
3252 * Register a receive hander for a device. This handler will then be
3253 * called from __netif_receive_skb. A negative errno code is returned
3254 * on a failure.
3255 *
3256 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003257 *
3258 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003259 */
3260int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003261 rx_handler_func_t *rx_handler,
3262 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003263{
3264 ASSERT_RTNL();
3265
3266 if (dev->rx_handler)
3267 return -EBUSY;
3268
Jiri Pirko93e2c322010-06-10 03:34:59 +00003269 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003270 rcu_assign_pointer(dev->rx_handler, rx_handler);
3271
3272 return 0;
3273}
3274EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3275
3276/**
3277 * netdev_rx_handler_unregister - unregister receive handler
3278 * @dev: device to unregister a handler from
3279 *
3280 * Unregister a receive hander from a device.
3281 *
3282 * The caller must hold the rtnl_mutex.
3283 */
3284void netdev_rx_handler_unregister(struct net_device *dev)
3285{
3286
3287 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003288 RCU_INIT_POINTER(dev->rx_handler, NULL);
3289 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003290}
3291EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3292
Mel Gormanb4b9e352012-07-31 16:44:26 -07003293/*
3294 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3295 * the special handling of PFMEMALLOC skbs.
3296 */
3297static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3298{
3299 switch (skb->protocol) {
3300 case __constant_htons(ETH_P_ARP):
3301 case __constant_htons(ETH_P_IP):
3302 case __constant_htons(ETH_P_IPV6):
3303 case __constant_htons(ETH_P_8021Q):
3304 return true;
3305 default:
3306 return false;
3307 }
3308}
3309
Eric Dumazet10f744d2010-03-28 23:07:20 -07003310static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003311{
3312 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003313 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003314 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003315 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003316 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003317 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003318 __be16 type;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003319 unsigned long pflags = current->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003320
Eric Dumazet588f0332011-11-15 04:12:55 +00003321 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003322
Koki Sanagicf66ba52010-08-23 18:45:02 +09003323 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003324
Mel Gormanb4b9e352012-07-31 16:44:26 -07003325 /*
3326 * PFMEMALLOC skbs are special, they should
3327 * - be delivered to SOCK_MEMALLOC sockets only
3328 * - stay away from userspace
3329 * - have bounded memory usage
3330 *
3331 * Use PF_MEMALLOC as this saves us from propagating the allocation
3332 * context down to all allocation sites.
3333 */
3334 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
3335 current->flags |= PF_MEMALLOC;
3336
Linus Torvalds1da177e2005-04-16 15:20:36 -07003337 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003338 if (netpoll_receive_skb(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003339 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003340
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003341 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003342
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003343 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003344 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003345 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003346
3347 pt_prev = NULL;
3348
3349 rcu_read_lock();
3350
David S. Miller63d8ea72011-02-28 10:48:59 -08003351another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003352 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003353
3354 __this_cpu_inc(softnet_data.processed);
3355
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003356 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3357 skb = vlan_untag(skb);
3358 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003359 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003360 }
3361
Linus Torvalds1da177e2005-04-16 15:20:36 -07003362#ifdef CONFIG_NET_CLS_ACT
3363 if (skb->tc_verd & TC_NCLS) {
3364 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3365 goto ncls;
3366 }
3367#endif
3368
Mel Gormanb4b9e352012-07-31 16:44:26 -07003369 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
3370 goto skip_taps;
3371
Linus Torvalds1da177e2005-04-16 15:20:36 -07003372 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003373 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003374 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003375 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003376 pt_prev = ptype;
3377 }
3378 }
3379
Mel Gormanb4b9e352012-07-31 16:44:26 -07003380skip_taps:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003381#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003382 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3383 if (!skb)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003384 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003385ncls:
3386#endif
3387
Mel Gormanb4b9e352012-07-31 16:44:26 -07003388 if (sk_memalloc_socks() && skb_pfmemalloc(skb)
3389 && !skb_pfmemalloc_protocol(skb))
3390 goto drop;
3391
John Fastabend24257172011-10-10 09:16:41 +00003392 if (vlan_tx_tag_present(skb)) {
3393 if (pt_prev) {
3394 ret = deliver_skb(skb, pt_prev, orig_dev);
3395 pt_prev = NULL;
3396 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003397 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003398 goto another_round;
3399 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003400 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003401 }
3402
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003403 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003404 if (rx_handler) {
3405 if (pt_prev) {
3406 ret = deliver_skb(skb, pt_prev, orig_dev);
3407 pt_prev = NULL;
3408 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003409 switch (rx_handler(&skb)) {
3410 case RX_HANDLER_CONSUMED:
Mel Gormanb4b9e352012-07-31 16:44:26 -07003411 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003412 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003413 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003414 case RX_HANDLER_EXACT:
3415 deliver_exact = true;
3416 case RX_HANDLER_PASS:
3417 break;
3418 default:
3419 BUG();
3420 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003421 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003422
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003423 if (vlan_tx_nonzero_tag_present(skb))
3424 skb->pkt_type = PACKET_OTHERHOST;
3425
David S. Miller63d8ea72011-02-28 10:48:59 -08003426 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003427 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003428
Linus Torvalds1da177e2005-04-16 15:20:36 -07003429 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003430 list_for_each_entry_rcu(ptype,
3431 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003432 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003433 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3434 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003435 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003436 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003437 pt_prev = ptype;
3438 }
3439 }
3440
3441 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003442 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003443 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003444 else
3445 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003446 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003447drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003448 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003449 kfree_skb(skb);
3450 /* Jamal, now you will not able to escape explaining
3451 * me how you were going to use this. :-)
3452 */
3453 ret = NET_RX_DROP;
3454 }
3455
Mel Gormanb4b9e352012-07-31 16:44:26 -07003456unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003457 rcu_read_unlock();
Mel Gormanb4b9e352012-07-31 16:44:26 -07003458out:
3459 tsk_restore_flags(current, pflags, PF_MEMALLOC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003460 return ret;
3461}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003462
3463/**
3464 * netif_receive_skb - process receive buffer from network
3465 * @skb: buffer to process
3466 *
3467 * netif_receive_skb() is the main receive data processing function.
3468 * It always succeeds. The buffer may be dropped during processing
3469 * for congestion control or by the protocol layers.
3470 *
3471 * This function may only be called from softirq context and interrupts
3472 * should be enabled.
3473 *
3474 * Return values (usually ignored):
3475 * NET_RX_SUCCESS: no congestion
3476 * NET_RX_DROP: packet was dropped
3477 */
3478int netif_receive_skb(struct sk_buff *skb)
3479{
Eric Dumazet588f0332011-11-15 04:12:55 +00003480 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003481
Richard Cochranc1f19b52010-07-17 08:49:36 +00003482 if (skb_defer_rx_timestamp(skb))
3483 return NET_RX_SUCCESS;
3484
Eric Dumazetdf334542010-03-24 19:13:54 +00003485#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003486 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003487 struct rps_dev_flow voidflow, *rflow = &voidflow;
3488 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003489
Eric Dumazet3b098e22010-05-15 23:57:10 -07003490 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003491
Eric Dumazet3b098e22010-05-15 23:57:10 -07003492 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003493
Eric Dumazet3b098e22010-05-15 23:57:10 -07003494 if (cpu >= 0) {
3495 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3496 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003497 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003498 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003499 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003500 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003501#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003502 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003503}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003504EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003505
Eric Dumazet88751272010-04-19 05:07:33 +00003506/* Network device is going away, flush any packets still pending
3507 * Called with irqs disabled.
3508 */
Changli Gao152102c2010-03-30 20:16:22 +00003509static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003510{
Changli Gao152102c2010-03-30 20:16:22 +00003511 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003512 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003513 struct sk_buff *skb, *tmp;
3514
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003515 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003516 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003517 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003518 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003519 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003520 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003521 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003522 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003523 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003524
3525 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3526 if (skb->dev == dev) {
3527 __skb_unlink(skb, &sd->process_queue);
3528 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003529 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003530 }
3531 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003532}
3533
Herbert Xud565b0a2008-12-15 23:38:52 -08003534static int napi_gro_complete(struct sk_buff *skb)
3535{
Vlad Yasevich22061d82012-11-15 08:49:11 +00003536 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003537 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003538 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08003539 int err = -ENOENT;
3540
Herbert Xufc59f9a2009-04-14 15:11:06 -07003541 if (NAPI_GRO_CB(skb)->count == 1) {
3542 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003543 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003544 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003545
3546 rcu_read_lock();
3547 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003548 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08003549 continue;
3550
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003551 err = ptype->callbacks.gro_complete(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003552 break;
3553 }
3554 rcu_read_unlock();
3555
3556 if (err) {
3557 WARN_ON(&ptype->list == head);
3558 kfree_skb(skb);
3559 return NET_RX_SUCCESS;
3560 }
3561
3562out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003563 return netif_receive_skb(skb);
3564}
3565
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003566/* napi->gro_list contains packets ordered by age.
3567 * youngest packets at the head of it.
3568 * Complete skbs in reverse order to reduce latencies.
3569 */
3570void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08003571{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003572 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003573
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003574 /* scan list and build reverse chain */
3575 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3576 skb->prev = prev;
3577 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08003578 }
3579
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003580 for (skb = prev; skb; skb = prev) {
3581 skb->next = NULL;
3582
3583 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3584 return;
3585
3586 prev = skb->prev;
3587 napi_gro_complete(skb);
3588 napi->gro_count--;
3589 }
3590
Herbert Xud565b0a2008-12-15 23:38:52 -08003591 napi->gro_list = NULL;
3592}
Eric Dumazet86cac582010-08-31 18:25:32 +00003593EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003594
Ben Hutchings5b252f02009-10-29 07:17:09 +00003595enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003596{
3597 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003598 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003599 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003600 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003601 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003602 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003603 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003604
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003605 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003606 goto normal;
3607
David S. Miller21dc3302010-08-23 00:13:46 -07003608 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003609 goto normal;
3610
Herbert Xud565b0a2008-12-15 23:38:52 -08003611 rcu_read_lock();
3612 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003613 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08003614 continue;
3615
Herbert Xu86911732009-01-29 14:19:50 +00003616 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003617 mac_len = skb->network_header - skb->mac_header;
3618 skb->mac_len = mac_len;
3619 NAPI_GRO_CB(skb)->same_flow = 0;
3620 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003621 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003622
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003623 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003624 break;
3625 }
3626 rcu_read_unlock();
3627
3628 if (&ptype->list == head)
3629 goto normal;
3630
Herbert Xu0da2afd52008-12-26 14:57:42 -08003631 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003632 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003633
Herbert Xud565b0a2008-12-15 23:38:52 -08003634 if (pp) {
3635 struct sk_buff *nskb = *pp;
3636
3637 *pp = nskb->next;
3638 nskb->next = NULL;
3639 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003640 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003641 }
3642
Herbert Xu0da2afd52008-12-26 14:57:42 -08003643 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003644 goto ok;
3645
Herbert Xu4ae55442009-02-08 18:00:36 +00003646 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003647 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003648
Herbert Xu4ae55442009-02-08 18:00:36 +00003649 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003650 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003651 NAPI_GRO_CB(skb)->age = jiffies;
Herbert Xu86911732009-01-29 14:19:50 +00003652 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003653 skb->next = napi->gro_list;
3654 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003655 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003656
Herbert Xuad0f9902009-02-01 01:24:55 -08003657pull:
Herbert Xucb189782009-05-26 18:50:31 +00003658 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3659 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3660
3661 BUG_ON(skb->end - skb->tail < grow);
3662
3663 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3664
3665 skb->tail += grow;
3666 skb->data_len -= grow;
3667
3668 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003669 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003670
Eric Dumazet9e903e02011-10-18 21:00:24 +00003671 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003672 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003673 memmove(skb_shinfo(skb)->frags,
3674 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003675 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003676 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003677 }
3678
Herbert Xud565b0a2008-12-15 23:38:52 -08003679ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003680 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003681
3682normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003683 ret = GRO_NORMAL;
3684 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003685}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003686EXPORT_SYMBOL(dev_gro_receive);
3687
Eric Dumazet40d08022010-08-26 22:03:08 -07003688static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003689__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003690{
3691 struct sk_buff *p;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003692 unsigned int maclen = skb->dev->hard_header_len;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003693
3694 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003695 unsigned long diffs;
3696
3697 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003698 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003699 if (maclen == ETH_HLEN)
3700 diffs |= compare_ether_header(skb_mac_header(p),
3701 skb_gro_mac_header(skb));
3702 else if (!diffs)
3703 diffs = memcmp(skb_mac_header(p),
3704 skb_gro_mac_header(skb),
3705 maclen);
Eric Dumazet40d08022010-08-26 22:03:08 -07003706 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003707 NAPI_GRO_CB(p)->flush = 0;
3708 }
3709
3710 return dev_gro_receive(napi, skb);
3711}
Herbert Xu5d38a072009-01-04 16:13:40 -08003712
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003713gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003714{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003715 switch (ret) {
3716 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003717 if (netif_receive_skb(skb))
3718 ret = GRO_DROP;
3719 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003720
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003721 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08003722 kfree_skb(skb);
3723 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003724
Eric Dumazetdaa86542012-04-19 07:07:40 +00003725 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00003726 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3727 kmem_cache_free(skbuff_head_cache, skb);
3728 else
3729 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00003730 break;
3731
Ben Hutchings5b252f02009-10-29 07:17:09 +00003732 case GRO_HELD:
3733 case GRO_MERGED:
3734 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003735 }
3736
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003737 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003738}
3739EXPORT_SYMBOL(napi_skb_finish);
3740
Eric Dumazetca07e432012-10-06 22:28:06 +00003741static void skb_gro_reset_offset(struct sk_buff *skb)
Herbert Xu78a478d2009-05-26 18:50:21 +00003742{
Eric Dumazetca07e432012-10-06 22:28:06 +00003743 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3744 const skb_frag_t *frag0 = &pinfo->frags[0];
3745
Herbert Xu78a478d2009-05-26 18:50:21 +00003746 NAPI_GRO_CB(skb)->data_offset = 0;
3747 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003748 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003749
Herbert Xu78d3fd02009-05-26 18:50:23 +00003750 if (skb->mac_header == skb->tail &&
Eric Dumazetca07e432012-10-06 22:28:06 +00003751 pinfo->nr_frags &&
3752 !PageHighMem(skb_frag_page(frag0))) {
3753 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3754 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xu74895942009-05-26 18:50:27 +00003755 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003756}
Herbert Xu78a478d2009-05-26 18:50:21 +00003757
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003758gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003759{
Herbert Xu86911732009-01-29 14:19:50 +00003760 skb_gro_reset_offset(skb);
3761
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003762 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003763}
3764EXPORT_SYMBOL(napi_gro_receive);
3765
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003766static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003767{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003768 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00003769 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3770 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003771 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003772 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003773 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003774
3775 napi->skb = skb;
3776}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003777
Herbert Xu76620aa2009-04-16 02:02:07 -07003778struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003779{
Herbert Xu5d38a072009-01-04 16:13:40 -08003780 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003781
3782 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003783 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3784 if (skb)
3785 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003786 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003787 return skb;
3788}
Herbert Xu76620aa2009-04-16 02:02:07 -07003789EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003790
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003791gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3792 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003793{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003794 switch (ret) {
3795 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003796 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003797 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003798
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003799 if (ret == GRO_HELD)
3800 skb_gro_pull(skb, -ETH_HLEN);
3801 else if (netif_receive_skb(skb))
3802 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003803 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003804
3805 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003806 case GRO_MERGED_FREE:
3807 napi_reuse_skb(napi, skb);
3808 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003809
3810 case GRO_MERGED:
3811 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003812 }
3813
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003814 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003815}
3816EXPORT_SYMBOL(napi_frags_finish);
3817
Eric Dumazet4adb9c42012-05-18 20:49:06 +00003818static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003819{
Herbert Xu76620aa2009-04-16 02:02:07 -07003820 struct sk_buff *skb = napi->skb;
3821 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003822 unsigned int hlen;
3823 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003824
3825 napi->skb = NULL;
3826
3827 skb_reset_mac_header(skb);
3828 skb_gro_reset_offset(skb);
3829
Herbert Xua5b1cf22009-05-26 18:50:28 +00003830 off = skb_gro_offset(skb);
3831 hlen = off + sizeof(*eth);
3832 eth = skb_gro_header_fast(skb, off);
3833 if (skb_gro_header_hard(skb, hlen)) {
3834 eth = skb_gro_header_slow(skb, hlen, off);
3835 if (unlikely(!eth)) {
3836 napi_reuse_skb(napi, skb);
3837 skb = NULL;
3838 goto out;
3839 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003840 }
3841
3842 skb_gro_pull(skb, sizeof(*eth));
3843
3844 /*
3845 * This works because the only protocols we care about don't require
3846 * special handling. We'll fix it up properly at the end.
3847 */
3848 skb->protocol = eth->h_proto;
3849
3850out:
3851 return skb;
3852}
Herbert Xu76620aa2009-04-16 02:02:07 -07003853
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003854gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003855{
3856 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003857
3858 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003859 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003860
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003861 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003862}
3863EXPORT_SYMBOL(napi_gro_frags);
3864
Eric Dumazete326bed2010-04-22 00:22:45 -07003865/*
3866 * net_rps_action sends any pending IPI's for rps.
3867 * Note: called with local irq disabled, but exits with local irq enabled.
3868 */
3869static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3870{
3871#ifdef CONFIG_RPS
3872 struct softnet_data *remsd = sd->rps_ipi_list;
3873
3874 if (remsd) {
3875 sd->rps_ipi_list = NULL;
3876
3877 local_irq_enable();
3878
3879 /* Send pending IPI's to kick RPS processing on remote cpus. */
3880 while (remsd) {
3881 struct softnet_data *next = remsd->rps_ipi_next;
3882
3883 if (cpu_online(remsd->cpu))
3884 __smp_call_function_single(remsd->cpu,
3885 &remsd->csd, 0);
3886 remsd = next;
3887 }
3888 } else
3889#endif
3890 local_irq_enable();
3891}
3892
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003893static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003894{
3895 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003896 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003897
Eric Dumazete326bed2010-04-22 00:22:45 -07003898#ifdef CONFIG_RPS
3899 /* Check if we have pending ipi, its better to send them now,
3900 * not waiting net_rx_action() end.
3901 */
3902 if (sd->rps_ipi_list) {
3903 local_irq_disable();
3904 net_rps_action_and_irq_enable(sd);
3905 }
3906#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003907 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003908 local_irq_disable();
3909 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003910 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003911 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003912
Changli Gao6e7676c2010-04-27 15:07:33 -07003913 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003914 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003915 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003916 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003917 input_queue_head_incr(sd);
3918 if (++work >= quota) {
3919 local_irq_enable();
3920 return work;
3921 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003922 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003923
Changli Gao6e7676c2010-04-27 15:07:33 -07003924 rps_lock(sd);
3925 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003926 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003927 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3928 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003929
Changli Gao6e7676c2010-04-27 15:07:33 -07003930 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003931 /*
3932 * Inline a custom version of __napi_complete().
3933 * only current cpu owns and manipulates this napi,
3934 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3935 * we can use a plain write instead of clear_bit(),
3936 * and we dont need an smp_mb() memory barrier.
3937 */
3938 list_del(&napi->poll_list);
3939 napi->state = 0;
3940
Changli Gao6e7676c2010-04-27 15:07:33 -07003941 quota = work + qlen;
3942 }
3943 rps_unlock(sd);
3944 }
3945 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003946
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003947 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003948}
3949
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003950/**
3951 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003952 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003953 *
3954 * The entry's receive function will be scheduled to run
3955 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003956void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003957{
3958 unsigned long flags;
3959
3960 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003961 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003962 local_irq_restore(flags);
3963}
3964EXPORT_SYMBOL(__napi_schedule);
3965
Herbert Xud565b0a2008-12-15 23:38:52 -08003966void __napi_complete(struct napi_struct *n)
3967{
3968 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3969 BUG_ON(n->gro_list);
3970
3971 list_del(&n->poll_list);
3972 smp_mb__before_clear_bit();
3973 clear_bit(NAPI_STATE_SCHED, &n->state);
3974}
3975EXPORT_SYMBOL(__napi_complete);
3976
3977void napi_complete(struct napi_struct *n)
3978{
3979 unsigned long flags;
3980
3981 /*
3982 * don't let napi dequeue from the cpu poll list
3983 * just in case its running on a different cpu
3984 */
3985 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3986 return;
3987
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003988 napi_gro_flush(n, false);
Herbert Xud565b0a2008-12-15 23:38:52 -08003989 local_irq_save(flags);
3990 __napi_complete(n);
3991 local_irq_restore(flags);
3992}
3993EXPORT_SYMBOL(napi_complete);
3994
3995void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3996 int (*poll)(struct napi_struct *, int), int weight)
3997{
3998 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003999 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004000 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004001 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004002 napi->poll = poll;
4003 napi->weight = weight;
4004 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004005 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004006#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004007 spin_lock_init(&napi->poll_lock);
4008 napi->poll_owner = -1;
4009#endif
4010 set_bit(NAPI_STATE_SCHED, &napi->state);
4011}
4012EXPORT_SYMBOL(netif_napi_add);
4013
4014void netif_napi_del(struct napi_struct *napi)
4015{
4016 struct sk_buff *skb, *next;
4017
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004018 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004019 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004020
4021 for (skb = napi->gro_list; skb; skb = next) {
4022 next = skb->next;
4023 skb->next = NULL;
4024 kfree_skb(skb);
4025 }
4026
4027 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004028 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004029}
4030EXPORT_SYMBOL(netif_napi_del);
4031
Linus Torvalds1da177e2005-04-16 15:20:36 -07004032static void net_rx_action(struct softirq_action *h)
4033{
Eric Dumazete326bed2010-04-22 00:22:45 -07004034 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004035 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004036 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07004037 void *have;
4038
Linus Torvalds1da177e2005-04-16 15:20:36 -07004039 local_irq_disable();
4040
Eric Dumazete326bed2010-04-22 00:22:45 -07004041 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004042 struct napi_struct *n;
4043 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004044
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004045 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004046 * Allow this to run for 2 jiffies since which will allow
4047 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004048 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004049 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004050 goto softnet_break;
4051
4052 local_irq_enable();
4053
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004054 /* Even though interrupts have been re-enabled, this
4055 * access is safe because interrupts can only add new
4056 * entries to the tail of this list, and only ->poll()
4057 * calls can remove this head entry from the list.
4058 */
Eric Dumazete326bed2010-04-22 00:22:45 -07004059 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004060
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004061 have = netpoll_poll_lock(n);
4062
4063 weight = n->weight;
4064
David S. Miller0a7606c2007-10-29 21:28:47 -07004065 /* This NAPI_STATE_SCHED test is for avoiding a race
4066 * with netpoll's poll_napi(). Only the entity which
4067 * obtains the lock and sees NAPI_STATE_SCHED set will
4068 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004069 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07004070 */
4071 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00004072 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07004073 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00004074 trace_napi_poll(n);
4075 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004076
4077 WARN_ON_ONCE(work > weight);
4078
4079 budget -= work;
4080
4081 local_irq_disable();
4082
4083 /* Drivers must not modify the NAPI state if they
4084 * consume the entire weight. In such cases this code
4085 * still "owns" the NAPI instance and therefore can
4086 * move the instance around on the list at-will.
4087 */
David S. Millerfed17f32008-01-07 21:00:40 -08004088 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07004089 if (unlikely(napi_disable_pending(n))) {
4090 local_irq_enable();
4091 napi_complete(n);
4092 local_irq_disable();
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004093 } else {
4094 if (n->gro_list) {
4095 /* flush too old packets
4096 * If HZ < 1000, flush all packets.
4097 */
4098 local_irq_enable();
4099 napi_gro_flush(n, HZ >= 1000);
4100 local_irq_disable();
4101 }
Eric Dumazete326bed2010-04-22 00:22:45 -07004102 list_move_tail(&n->poll_list, &sd->poll_list);
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004103 }
David S. Millerfed17f32008-01-07 21:00:40 -08004104 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004105
4106 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004107 }
4108out:
Eric Dumazete326bed2010-04-22 00:22:45 -07004109 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004110
Chris Leechdb217332006-06-17 21:24:58 -07004111#ifdef CONFIG_NET_DMA
4112 /*
4113 * There may not be any more sk_buffs coming right now, so push
4114 * any pending DMA copies to hardware
4115 */
Dan Williams2ba05622009-01-06 11:38:14 -07004116 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07004117#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004118
Linus Torvalds1da177e2005-04-16 15:20:36 -07004119 return;
4120
4121softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00004122 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004123 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4124 goto out;
4125}
4126
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004127static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004128
4129/**
4130 * register_gifconf - register a SIOCGIF handler
4131 * @family: Address family
4132 * @gifconf: Function handler
4133 *
4134 * Register protocol dependent address dumping routines. The handler
4135 * that is passed must not be freed or reused until it has been replaced
4136 * by another handler.
4137 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004138int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004139{
4140 if (family >= NPROTO)
4141 return -EINVAL;
4142 gifconf_list[family] = gifconf;
4143 return 0;
4144}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004145EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004146
4147
4148/*
4149 * Map an interface index to its name (SIOCGIFNAME)
4150 */
4151
4152/*
4153 * We need this ioctl for efficient implementation of the
4154 * if_indextoname() function required by the IPv6 API. Without
4155 * it, we would have to search all the interfaces to find a
4156 * match. --pb
4157 */
4158
Eric W. Biederman881d9662007-09-17 11:56:21 -07004159static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004160{
4161 struct net_device *dev;
4162 struct ifreq ifr;
Brian Haleyc91f6df2012-11-26 05:21:08 +00004163 unsigned seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004164
4165 /*
4166 * Fetch the caller's info block.
4167 */
4168
4169 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4170 return -EFAULT;
4171
Brian Haleyc91f6df2012-11-26 05:21:08 +00004172retry:
4173 seq = read_seqbegin(&devnet_rename_seq);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004174 rcu_read_lock();
4175 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004176 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004177 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004178 return -ENODEV;
4179 }
4180
4181 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004182 rcu_read_unlock();
Brian Haleyc91f6df2012-11-26 05:21:08 +00004183 if (read_seqretry(&devnet_rename_seq, seq))
4184 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004185
4186 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
4187 return -EFAULT;
4188 return 0;
4189}
4190
4191/*
4192 * Perform a SIOCGIFCONF call. This structure will change
4193 * size eventually, and there is nothing I can do about it.
4194 * Thus we will need a 'compatibility mode'.
4195 */
4196
Eric W. Biederman881d9662007-09-17 11:56:21 -07004197static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004198{
4199 struct ifconf ifc;
4200 struct net_device *dev;
4201 char __user *pos;
4202 int len;
4203 int total;
4204 int i;
4205
4206 /*
4207 * Fetch the caller's info block.
4208 */
4209
4210 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
4211 return -EFAULT;
4212
4213 pos = ifc.ifc_buf;
4214 len = ifc.ifc_len;
4215
4216 /*
4217 * Loop over the interfaces, and write an info block for each.
4218 */
4219
4220 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004221 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004222 for (i = 0; i < NPROTO; i++) {
4223 if (gifconf_list[i]) {
4224 int done;
4225 if (!pos)
4226 done = gifconf_list[i](dev, NULL, 0);
4227 else
4228 done = gifconf_list[i](dev, pos + total,
4229 len - total);
4230 if (done < 0)
4231 return -EFAULT;
4232 total += done;
4233 }
4234 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004235 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004236
4237 /*
4238 * All done. Write the updated control block back to the caller.
4239 */
4240 ifc.ifc_len = total;
4241
4242 /*
4243 * Both BSD and Solaris return 0 here, so we do too.
4244 */
4245 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
4246}
4247
4248#ifdef CONFIG_PROC_FS
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004249
Eric Dumazet2def16a2012-04-02 22:33:02 +00004250#define BUCKET_SPACE (32 - NETDEV_HASHBITS - 1)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004251
4252#define get_bucket(x) ((x) >> BUCKET_SPACE)
4253#define get_offset(x) ((x) & ((1 << BUCKET_SPACE) - 1))
4254#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))
4255
Eric Dumazet2def16a2012-04-02 22:33:02 +00004256static inline struct net_device *dev_from_same_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004257{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004258 struct net *net = seq_file_net(seq);
4259 struct net_device *dev;
4260 struct hlist_node *p;
4261 struct hlist_head *h;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004262 unsigned int count = 0, offset = get_offset(*pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004263
Eric Dumazet2def16a2012-04-02 22:33:02 +00004264 h = &net->dev_name_head[get_bucket(*pos)];
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004265 hlist_for_each_entry_rcu(dev, p, h, name_hlist) {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004266 if (++count == offset)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004267 return dev;
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004268 }
4269
4270 return NULL;
4271}
4272
Eric Dumazet2def16a2012-04-02 22:33:02 +00004273static inline struct net_device *dev_from_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004274{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004275 struct net_device *dev;
4276 unsigned int bucket;
4277
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004278 do {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004279 dev = dev_from_same_bucket(seq, pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004280 if (dev)
4281 return dev;
4282
Eric Dumazet2def16a2012-04-02 22:33:02 +00004283 bucket = get_bucket(*pos) + 1;
4284 *pos = set_bucket_offset(bucket, 1);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004285 } while (bucket < NETDEV_HASHENTRIES);
4286
4287 return NULL;
4288}
4289
Linus Torvalds1da177e2005-04-16 15:20:36 -07004290/*
4291 * This is invoked by the /proc filesystem handler to display a device
4292 * in detail.
4293 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004294void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004295 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004296{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004297 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07004298 if (!*pos)
4299 return SEQ_START_TOKEN;
4300
Eric Dumazet2def16a2012-04-02 22:33:02 +00004301 if (get_bucket(*pos) >= NETDEV_HASHENTRIES)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004302 return NULL;
Pavel Emelianov7562f872007-05-03 15:13:45 -07004303
Eric Dumazet2def16a2012-04-02 22:33:02 +00004304 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004305}
4306
4307void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4308{
4309 ++*pos;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004310 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004311}
4312
4313void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004314 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004315{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004316 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004317}
4318
4319static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4320{
Eric Dumazet28172732010-07-07 14:58:56 -07004321 struct rtnl_link_stats64 temp;
4322 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004323
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004324 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4325 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004326 dev->name, stats->rx_bytes, stats->rx_packets,
4327 stats->rx_errors,
4328 stats->rx_dropped + stats->rx_missed_errors,
4329 stats->rx_fifo_errors,
4330 stats->rx_length_errors + stats->rx_over_errors +
4331 stats->rx_crc_errors + stats->rx_frame_errors,
4332 stats->rx_compressed, stats->multicast,
4333 stats->tx_bytes, stats->tx_packets,
4334 stats->tx_errors, stats->tx_dropped,
4335 stats->tx_fifo_errors, stats->collisions,
4336 stats->tx_carrier_errors +
4337 stats->tx_aborted_errors +
4338 stats->tx_window_errors +
4339 stats->tx_heartbeat_errors,
4340 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004341}
4342
4343/*
4344 * Called from the PROCfs module. This now uses the new arbitrary sized
4345 * /proc/net interface to create /proc/net/dev
4346 */
4347static int dev_seq_show(struct seq_file *seq, void *v)
4348{
4349 if (v == SEQ_START_TOKEN)
4350 seq_puts(seq, "Inter-| Receive "
4351 " | Transmit\n"
4352 " face |bytes packets errs drop fifo frame "
4353 "compressed multicast|bytes packets errs "
4354 "drop fifo colls carrier compressed\n");
4355 else
4356 dev_seq_printf_stats(seq, v);
4357 return 0;
4358}
4359
Changli Gaodee42872010-05-02 05:42:16 +00004360static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004361{
Changli Gaodee42872010-05-02 05:42:16 +00004362 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004363
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004364 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004365 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004366 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004367 break;
4368 } else
4369 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004370 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004371}
4372
4373static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4374{
4375 return softnet_get_online(pos);
4376}
4377
4378static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4379{
4380 ++*pos;
4381 return softnet_get_online(pos);
4382}
4383
4384static void softnet_seq_stop(struct seq_file *seq, void *v)
4385{
4386}
4387
4388static int softnet_seq_show(struct seq_file *seq, void *v)
4389{
Changli Gaodee42872010-05-02 05:42:16 +00004390 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004391
Tom Herbert0a9627f2010-03-16 08:03:29 +00004392 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004393 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004394 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004395 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004396 return 0;
4397}
4398
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004399static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004400 .start = dev_seq_start,
4401 .next = dev_seq_next,
4402 .stop = dev_seq_stop,
4403 .show = dev_seq_show,
4404};
4405
4406static int dev_seq_open(struct inode *inode, struct file *file)
4407{
Denis V. Luneve372c412007-11-19 22:31:54 -08004408 return seq_open_net(inode, file, &dev_seq_ops,
Eric Dumazet2def16a2012-04-02 22:33:02 +00004409 sizeof(struct seq_net_private));
Anton Blanchard5cac98d2011-11-27 21:14:46 +00004410}
4411
Arjan van de Ven9a321442007-02-12 00:55:35 -08004412static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004413 .owner = THIS_MODULE,
4414 .open = dev_seq_open,
4415 .read = seq_read,
4416 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004417 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004418};
4419
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004420static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421 .start = softnet_seq_start,
4422 .next = softnet_seq_next,
4423 .stop = softnet_seq_stop,
4424 .show = softnet_seq_show,
4425};
4426
4427static int softnet_seq_open(struct inode *inode, struct file *file)
4428{
4429 return seq_open(file, &softnet_seq_ops);
4430}
4431
Arjan van de Ven9a321442007-02-12 00:55:35 -08004432static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004433 .owner = THIS_MODULE,
4434 .open = softnet_seq_open,
4435 .read = seq_read,
4436 .llseek = seq_lseek,
4437 .release = seq_release,
4438};
4439
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004440static void *ptype_get_idx(loff_t pos)
4441{
4442 struct packet_type *pt = NULL;
4443 loff_t i = 0;
4444 int t;
4445
4446 list_for_each_entry_rcu(pt, &ptype_all, list) {
4447 if (i == pos)
4448 return pt;
4449 ++i;
4450 }
4451
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004452 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004453 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4454 if (i == pos)
4455 return pt;
4456 ++i;
4457 }
4458 }
4459 return NULL;
4460}
4461
4462static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004463 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004464{
4465 rcu_read_lock();
4466 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4467}
4468
4469static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4470{
4471 struct packet_type *pt;
4472 struct list_head *nxt;
4473 int hash;
4474
4475 ++*pos;
4476 if (v == SEQ_START_TOKEN)
4477 return ptype_get_idx(0);
4478
4479 pt = v;
4480 nxt = pt->list.next;
4481 if (pt->type == htons(ETH_P_ALL)) {
4482 if (nxt != &ptype_all)
4483 goto found;
4484 hash = 0;
4485 nxt = ptype_base[0].next;
4486 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004487 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004488
4489 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004490 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004491 return NULL;
4492 nxt = ptype_base[hash].next;
4493 }
4494found:
4495 return list_entry(nxt, struct packet_type, list);
4496}
4497
4498static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004499 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004500{
4501 rcu_read_unlock();
4502}
4503
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004504static int ptype_seq_show(struct seq_file *seq, void *v)
4505{
4506 struct packet_type *pt = v;
4507
4508 if (v == SEQ_START_TOKEN)
4509 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004510 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004511 if (pt->type == htons(ETH_P_ALL))
4512 seq_puts(seq, "ALL ");
4513 else
4514 seq_printf(seq, "%04x", ntohs(pt->type));
4515
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004516 seq_printf(seq, " %-8s %pF\n",
4517 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004518 }
4519
4520 return 0;
4521}
4522
4523static const struct seq_operations ptype_seq_ops = {
4524 .start = ptype_seq_start,
4525 .next = ptype_seq_next,
4526 .stop = ptype_seq_stop,
4527 .show = ptype_seq_show,
4528};
4529
4530static int ptype_seq_open(struct inode *inode, struct file *file)
4531{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004532 return seq_open_net(inode, file, &ptype_seq_ops,
4533 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004534}
4535
4536static const struct file_operations ptype_seq_fops = {
4537 .owner = THIS_MODULE,
4538 .open = ptype_seq_open,
4539 .read = seq_read,
4540 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004541 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004542};
4543
4544
Pavel Emelyanov46650792007-10-08 20:38:39 -07004545static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004546{
4547 int rc = -ENOMEM;
4548
Eric W. Biederman881d9662007-09-17 11:56:21 -07004549 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004550 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004551 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004552 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004553 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004554 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004555
Eric W. Biederman881d9662007-09-17 11:56:21 -07004556 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004557 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004558 rc = 0;
4559out:
4560 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004561out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004562 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004563out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004564 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004565out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004566 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004567 goto out;
4568}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004569
Pavel Emelyanov46650792007-10-08 20:38:39 -07004570static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004571{
4572 wext_proc_exit(net);
4573
4574 proc_net_remove(net, "ptype");
4575 proc_net_remove(net, "softnet_stat");
4576 proc_net_remove(net, "dev");
4577}
4578
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004579static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004580 .init = dev_proc_net_init,
4581 .exit = dev_proc_net_exit,
4582};
4583
4584static int __init dev_proc_init(void)
4585{
4586 return register_pernet_subsys(&dev_proc_ops);
4587}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004588#else
4589#define dev_proc_init() 0
4590#endif /* CONFIG_PROC_FS */
4591
4592
4593/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004594 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004595 * @slave: slave device
4596 * @master: new master device
4597 *
4598 * Changes the master device of the slave. Pass %NULL to break the
4599 * bonding. The caller must hold the RTNL semaphore. On a failure
4600 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004601 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004602 */
4603int netdev_set_master(struct net_device *slave, struct net_device *master)
4604{
4605 struct net_device *old = slave->master;
4606
4607 ASSERT_RTNL();
4608
4609 if (master) {
4610 if (old)
4611 return -EBUSY;
4612 dev_hold(master);
4613 }
4614
4615 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004616
Eric Dumazet6df427f2011-05-19 19:37:40 +00004617 if (old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004618 dev_put(old);
Jiri Pirko1765a572011-02-12 06:48:36 +00004619 return 0;
4620}
4621EXPORT_SYMBOL(netdev_set_master);
4622
4623/**
4624 * netdev_set_bond_master - set up bonding master/slave pair
4625 * @slave: slave device
4626 * @master: new master device
4627 *
4628 * Changes the master device of the slave. Pass %NULL to break the
4629 * bonding. The caller must hold the RTNL semaphore. On a failure
4630 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4631 * to the routing socket and the function returns zero.
4632 */
4633int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4634{
4635 int err;
4636
4637 ASSERT_RTNL();
4638
4639 err = netdev_set_master(slave, master);
4640 if (err)
4641 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004642 if (master)
4643 slave->flags |= IFF_SLAVE;
4644 else
4645 slave->flags &= ~IFF_SLAVE;
4646
4647 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4648 return 0;
4649}
Jiri Pirko1765a572011-02-12 06:48:36 +00004650EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004651
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004652static void dev_change_rx_flags(struct net_device *dev, int flags)
4653{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004654 const struct net_device_ops *ops = dev->netdev_ops;
4655
4656 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4657 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004658}
4659
Wang Chendad9b332008-06-18 01:48:28 -07004660static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004661{
Eric Dumazetb536db92011-11-30 21:42:26 +00004662 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004663 kuid_t uid;
4664 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004665
Patrick McHardy24023452007-07-14 18:51:31 -07004666 ASSERT_RTNL();
4667
Wang Chendad9b332008-06-18 01:48:28 -07004668 dev->flags |= IFF_PROMISC;
4669 dev->promiscuity += inc;
4670 if (dev->promiscuity == 0) {
4671 /*
4672 * Avoid overflow.
4673 * If inc causes overflow, untouch promisc and return error.
4674 */
4675 if (inc < 0)
4676 dev->flags &= ~IFF_PROMISC;
4677 else {
4678 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004679 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4680 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004681 return -EOVERFLOW;
4682 }
4683 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004684 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004685 pr_info("device %s %s promiscuous mode\n",
4686 dev->name,
4687 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11004688 if (audit_enabled) {
4689 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004690 audit_log(current->audit_context, GFP_ATOMIC,
4691 AUDIT_ANOM_PROMISCUOUS,
4692 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4693 dev->name, (dev->flags & IFF_PROMISC),
4694 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07004695 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004696 from_kuid(&init_user_ns, uid),
4697 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004698 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004699 }
Patrick McHardy24023452007-07-14 18:51:31 -07004700
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004701 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004702 }
Wang Chendad9b332008-06-18 01:48:28 -07004703 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004704}
4705
Linus Torvalds1da177e2005-04-16 15:20:36 -07004706/**
4707 * dev_set_promiscuity - update promiscuity count on a device
4708 * @dev: device
4709 * @inc: modifier
4710 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004711 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004712 * remains above zero the interface remains promiscuous. Once it hits zero
4713 * the device reverts back to normal filtering operation. A negative inc
4714 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004715 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004716 */
Wang Chendad9b332008-06-18 01:48:28 -07004717int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004718{
Eric Dumazetb536db92011-11-30 21:42:26 +00004719 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004720 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004721
Wang Chendad9b332008-06-18 01:48:28 -07004722 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004723 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004724 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004725 if (dev->flags != old_flags)
4726 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004727 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004728}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004729EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004730
4731/**
4732 * dev_set_allmulti - update allmulti count on a device
4733 * @dev: device
4734 * @inc: modifier
4735 *
4736 * Add or remove reception of all multicast frames to a device. While the
4737 * count in the device remains above zero the interface remains listening
4738 * to all interfaces. Once it hits zero the device reverts back to normal
4739 * filtering operation. A negative @inc value is used to drop the counter
4740 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004741 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004742 */
4743
Wang Chendad9b332008-06-18 01:48:28 -07004744int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004745{
Eric Dumazetb536db92011-11-30 21:42:26 +00004746 unsigned int old_flags = dev->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004747
Patrick McHardy24023452007-07-14 18:51:31 -07004748 ASSERT_RTNL();
4749
Linus Torvalds1da177e2005-04-16 15:20:36 -07004750 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004751 dev->allmulti += inc;
4752 if (dev->allmulti == 0) {
4753 /*
4754 * Avoid overflow.
4755 * If inc causes overflow, untouch allmulti and return error.
4756 */
4757 if (inc < 0)
4758 dev->flags &= ~IFF_ALLMULTI;
4759 else {
4760 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004761 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4762 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004763 return -EOVERFLOW;
4764 }
4765 }
Patrick McHardy24023452007-07-14 18:51:31 -07004766 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004767 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004768 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004769 }
Wang Chendad9b332008-06-18 01:48:28 -07004770 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004771}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004772EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004773
4774/*
4775 * Upload unicast and multicast address lists to device and
4776 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004777 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004778 * are present.
4779 */
4780void __dev_set_rx_mode(struct net_device *dev)
4781{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004782 const struct net_device_ops *ops = dev->netdev_ops;
4783
Patrick McHardy4417da62007-06-27 01:28:10 -07004784 /* dev_open will call this function so the list will stay sane. */
4785 if (!(dev->flags&IFF_UP))
4786 return;
4787
4788 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004789 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004790
Jiri Pirko01789342011-08-16 06:29:00 +00004791 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004792 /* Unicast addresses changes may only happen under the rtnl,
4793 * therefore calling __dev_set_promiscuity here is safe.
4794 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004795 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004796 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004797 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004798 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004799 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004800 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004801 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004802 }
Jiri Pirko01789342011-08-16 06:29:00 +00004803
4804 if (ops->ndo_set_rx_mode)
4805 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004806}
4807
4808void dev_set_rx_mode(struct net_device *dev)
4809{
David S. Millerb9e40852008-07-15 00:15:08 -07004810 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004811 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004812 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004813}
4814
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004815/**
4816 * dev_get_flags - get flags reported to userspace
4817 * @dev: device
4818 *
4819 * Get the combination of flag bits exported through APIs to userspace.
4820 */
Eric Dumazet95c96172012-04-15 05:58:06 +00004821unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004822{
Eric Dumazet95c96172012-04-15 05:58:06 +00004823 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004824
4825 flags = (dev->flags & ~(IFF_PROMISC |
4826 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004827 IFF_RUNNING |
4828 IFF_LOWER_UP |
4829 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004830 (dev->gflags & (IFF_PROMISC |
4831 IFF_ALLMULTI));
4832
Stefan Rompfb00055a2006-03-20 17:09:11 -08004833 if (netif_running(dev)) {
4834 if (netif_oper_up(dev))
4835 flags |= IFF_RUNNING;
4836 if (netif_carrier_ok(dev))
4837 flags |= IFF_LOWER_UP;
4838 if (netif_dormant(dev))
4839 flags |= IFF_DORMANT;
4840 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004841
4842 return flags;
4843}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004844EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004845
Patrick McHardybd380812010-02-26 06:34:53 +00004846int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004847{
Eric Dumazetb536db92011-11-30 21:42:26 +00004848 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004849 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004850
Patrick McHardy24023452007-07-14 18:51:31 -07004851 ASSERT_RTNL();
4852
Linus Torvalds1da177e2005-04-16 15:20:36 -07004853 /*
4854 * Set the flags on our device.
4855 */
4856
4857 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4858 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4859 IFF_AUTOMEDIA)) |
4860 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4861 IFF_ALLMULTI));
4862
4863 /*
4864 * Load in the correct multicast list now the flags have changed.
4865 */
4866
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004867 if ((old_flags ^ flags) & IFF_MULTICAST)
4868 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004869
Patrick McHardy4417da62007-06-27 01:28:10 -07004870 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004871
4872 /*
4873 * Have we downed the interface. We handle IFF_UP ourselves
4874 * according to user attempts to set it, rather than blindly
4875 * setting it.
4876 */
4877
4878 ret = 0;
4879 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004880 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004881
4882 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004883 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004884 }
4885
Linus Torvalds1da177e2005-04-16 15:20:36 -07004886 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004887 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4888
Linus Torvalds1da177e2005-04-16 15:20:36 -07004889 dev->gflags ^= IFF_PROMISC;
4890 dev_set_promiscuity(dev, inc);
4891 }
4892
4893 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4894 is important. Some (broken) drivers set IFF_PROMISC, when
4895 IFF_ALLMULTI is requested not asking us and not reporting.
4896 */
4897 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004898 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4899
Linus Torvalds1da177e2005-04-16 15:20:36 -07004900 dev->gflags ^= IFF_ALLMULTI;
4901 dev_set_allmulti(dev, inc);
4902 }
4903
Patrick McHardybd380812010-02-26 06:34:53 +00004904 return ret;
4905}
4906
4907void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4908{
4909 unsigned int changes = dev->flags ^ old_flags;
4910
4911 if (changes & IFF_UP) {
4912 if (dev->flags & IFF_UP)
4913 call_netdevice_notifiers(NETDEV_UP, dev);
4914 else
4915 call_netdevice_notifiers(NETDEV_DOWN, dev);
4916 }
4917
4918 if (dev->flags & IFF_UP &&
4919 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4920 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4921}
4922
4923/**
4924 * dev_change_flags - change device settings
4925 * @dev: device
4926 * @flags: device state flags
4927 *
4928 * Change settings on device based state flags. The flags are
4929 * in the userspace exported format.
4930 */
Eric Dumazetb536db92011-11-30 21:42:26 +00004931int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00004932{
Eric Dumazetb536db92011-11-30 21:42:26 +00004933 int ret;
4934 unsigned int changes, old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004935
4936 ret = __dev_change_flags(dev, flags);
4937 if (ret < 0)
4938 return ret;
4939
4940 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004941 if (changes)
4942 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004943
Patrick McHardybd380812010-02-26 06:34:53 +00004944 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004945 return ret;
4946}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004947EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004948
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004949/**
4950 * dev_set_mtu - Change maximum transfer unit
4951 * @dev: device
4952 * @new_mtu: new transfer unit
4953 *
4954 * Change the maximum transfer size of the network device.
4955 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004956int dev_set_mtu(struct net_device *dev, int new_mtu)
4957{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004958 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004959 int err;
4960
4961 if (new_mtu == dev->mtu)
4962 return 0;
4963
4964 /* MTU must be positive. */
4965 if (new_mtu < 0)
4966 return -EINVAL;
4967
4968 if (!netif_device_present(dev))
4969 return -ENODEV;
4970
4971 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004972 if (ops->ndo_change_mtu)
4973 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004974 else
4975 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004976
Linus Torvalds1da177e2005-04-16 15:20:36 -07004977 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004978 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004979 return err;
4980}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004981EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004982
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004983/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004984 * dev_set_group - Change group this device belongs to
4985 * @dev: device
4986 * @new_group: group this device should belong to
4987 */
4988void dev_set_group(struct net_device *dev, int new_group)
4989{
4990 dev->group = new_group;
4991}
4992EXPORT_SYMBOL(dev_set_group);
4993
4994/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004995 * dev_set_mac_address - Change Media Access Control Address
4996 * @dev: device
4997 * @sa: new address
4998 *
4999 * Change the hardware (MAC) address of the device
5000 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005001int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
5002{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005003 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005004 int err;
5005
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005006 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005007 return -EOPNOTSUPP;
5008 if (sa->sa_family != dev->type)
5009 return -EINVAL;
5010 if (!netif_device_present(dev))
5011 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005012 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005013 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005014 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005015 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005016 return err;
5017}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005018EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005019
5020/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00005021 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07005022 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07005023static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005024{
5025 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00005026 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005027
5028 if (!dev)
5029 return -ENODEV;
5030
5031 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005032 case SIOCGIFFLAGS: /* Get interface flags */
5033 ifr->ifr_flags = (short) dev_get_flags(dev);
5034 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005035
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005036 case SIOCGIFMETRIC: /* Get the metric on the interface
5037 (currently unused) */
5038 ifr->ifr_metric = 0;
5039 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005040
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005041 case SIOCGIFMTU: /* Get the MTU of a device */
5042 ifr->ifr_mtu = dev->mtu;
5043 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005044
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005045 case SIOCGIFHWADDR:
5046 if (!dev->addr_len)
5047 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
5048 else
5049 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
5050 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
5051 ifr->ifr_hwaddr.sa_family = dev->type;
5052 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005053
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005054 case SIOCGIFSLAVE:
5055 err = -EINVAL;
5056 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005057
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005058 case SIOCGIFMAP:
5059 ifr->ifr_map.mem_start = dev->mem_start;
5060 ifr->ifr_map.mem_end = dev->mem_end;
5061 ifr->ifr_map.base_addr = dev->base_addr;
5062 ifr->ifr_map.irq = dev->irq;
5063 ifr->ifr_map.dma = dev->dma;
5064 ifr->ifr_map.port = dev->if_port;
5065 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005066
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005067 case SIOCGIFINDEX:
5068 ifr->ifr_ifindex = dev->ifindex;
5069 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005070
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005071 case SIOCGIFTXQLEN:
5072 ifr->ifr_qlen = dev->tx_queue_len;
5073 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005074
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005075 default:
5076 /* dev_ioctl() should ensure this case
5077 * is never reached
5078 */
5079 WARN_ON(1);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005080 err = -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005081 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005082
5083 }
5084 return err;
5085}
5086
5087/*
5088 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
5089 */
5090static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
5091{
5092 int err;
5093 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08005094 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005095
5096 if (!dev)
5097 return -ENODEV;
5098
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08005099 ops = dev->netdev_ops;
5100
Jeff Garzik14e3e072007-10-08 00:06:32 -07005101 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005102 case SIOCSIFFLAGS: /* Set interface flags */
5103 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07005104
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005105 case SIOCSIFMETRIC: /* Set the metric on the interface
5106 (currently unused) */
5107 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005108
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005109 case SIOCSIFMTU: /* Set the MTU of a device */
5110 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07005111
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005112 case SIOCSIFHWADDR:
5113 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005114
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005115 case SIOCSIFHWBROADCAST:
5116 if (ifr->ifr_hwaddr.sa_family != dev->type)
5117 return -EINVAL;
5118 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
5119 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
5120 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
5121 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005122
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005123 case SIOCSIFMAP:
5124 if (ops->ndo_set_config) {
5125 if (!netif_device_present(dev))
5126 return -ENODEV;
5127 return ops->ndo_set_config(dev, &ifr->ifr_map);
5128 }
5129 return -EOPNOTSUPP;
5130
5131 case SIOCADDMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00005132 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005133 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
5134 return -EINVAL;
5135 if (!netif_device_present(dev))
5136 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00005137 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005138
5139 case SIOCDELMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00005140 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005141 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
5142 return -EINVAL;
5143 if (!netif_device_present(dev))
5144 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00005145 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005146
5147 case SIOCSIFTXQLEN:
5148 if (ifr->ifr_qlen < 0)
5149 return -EINVAL;
5150 dev->tx_queue_len = ifr->ifr_qlen;
5151 return 0;
5152
5153 case SIOCSIFNAME:
5154 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
5155 return dev_change_name(dev, ifr->ifr_newname);
5156
Richard Cochran4dc360c2011-10-19 17:00:35 -04005157 case SIOCSHWTSTAMP:
5158 err = net_hwtstamp_validate(ifr);
5159 if (err)
5160 return err;
5161 /* fall through */
5162
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005163 /*
5164 * Unknown or private ioctl
5165 */
5166 default:
5167 if ((cmd >= SIOCDEVPRIVATE &&
5168 cmd <= SIOCDEVPRIVATE + 15) ||
5169 cmd == SIOCBONDENSLAVE ||
5170 cmd == SIOCBONDRELEASE ||
5171 cmd == SIOCBONDSETHWADDR ||
5172 cmd == SIOCBONDSLAVEINFOQUERY ||
5173 cmd == SIOCBONDINFOQUERY ||
5174 cmd == SIOCBONDCHANGEACTIVE ||
5175 cmd == SIOCGMIIPHY ||
5176 cmd == SIOCGMIIREG ||
5177 cmd == SIOCSMIIREG ||
5178 cmd == SIOCBRADDIF ||
5179 cmd == SIOCBRDELIF ||
5180 cmd == SIOCSHWTSTAMP ||
5181 cmd == SIOCWANDEV) {
5182 err = -EOPNOTSUPP;
5183 if (ops->ndo_do_ioctl) {
5184 if (netif_device_present(dev))
5185 err = ops->ndo_do_ioctl(dev, ifr, cmd);
5186 else
5187 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005188 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005189 } else
5190 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005191
5192 }
5193 return err;
5194}
5195
5196/*
5197 * This function handles all "interface"-type I/O control requests. The actual
5198 * 'doing' part of this is dev_ifsioc above.
5199 */
5200
5201/**
5202 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005203 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005204 * @cmd: command to issue
5205 * @arg: pointer to a struct ifreq in user space
5206 *
5207 * Issue ioctl functions to devices. This is normally called by the
5208 * user space syscall interfaces but can sometimes be useful for
5209 * other purposes. The return value is the return from the syscall if
5210 * positive or a negative errno code on error.
5211 */
5212
Eric W. Biederman881d9662007-09-17 11:56:21 -07005213int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005214{
5215 struct ifreq ifr;
5216 int ret;
5217 char *colon;
5218
5219 /* One special case: SIOCGIFCONF takes ifconf argument
5220 and requires shared lock, because it sleeps writing
5221 to user space.
5222 */
5223
5224 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005225 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005226 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005227 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005228 return ret;
5229 }
5230 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005231 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005232
5233 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
5234 return -EFAULT;
5235
5236 ifr.ifr_name[IFNAMSIZ-1] = 0;
5237
5238 colon = strchr(ifr.ifr_name, ':');
5239 if (colon)
5240 *colon = 0;
5241
5242 /*
5243 * See which interface the caller is talking about.
5244 */
5245
5246 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005247 /*
5248 * These ioctl calls:
5249 * - can be done by all.
5250 * - atomic and do not require locking.
5251 * - return a value
5252 */
5253 case SIOCGIFFLAGS:
5254 case SIOCGIFMETRIC:
5255 case SIOCGIFMTU:
5256 case SIOCGIFHWADDR:
5257 case SIOCGIFSLAVE:
5258 case SIOCGIFMAP:
5259 case SIOCGIFINDEX:
5260 case SIOCGIFTXQLEN:
5261 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005262 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005263 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005264 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005265 if (!ret) {
5266 if (colon)
5267 *colon = ':';
5268 if (copy_to_user(arg, &ifr,
5269 sizeof(struct ifreq)))
5270 ret = -EFAULT;
5271 }
5272 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005273
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005274 case SIOCETHTOOL:
5275 dev_load(net, ifr.ifr_name);
5276 rtnl_lock();
5277 ret = dev_ethtool(net, &ifr);
5278 rtnl_unlock();
5279 if (!ret) {
5280 if (colon)
5281 *colon = ':';
5282 if (copy_to_user(arg, &ifr,
5283 sizeof(struct ifreq)))
5284 ret = -EFAULT;
5285 }
5286 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005287
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005288 /*
5289 * These ioctl calls:
5290 * - require superuser power.
5291 * - require strict serialization.
5292 * - return a value
5293 */
5294 case SIOCGMIIPHY:
5295 case SIOCGMIIREG:
5296 case SIOCSIFNAME:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +00005297 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005298 return -EPERM;
5299 dev_load(net, ifr.ifr_name);
5300 rtnl_lock();
5301 ret = dev_ifsioc(net, &ifr, cmd);
5302 rtnl_unlock();
5303 if (!ret) {
5304 if (colon)
5305 *colon = ':';
5306 if (copy_to_user(arg, &ifr,
5307 sizeof(struct ifreq)))
5308 ret = -EFAULT;
5309 }
5310 return ret;
5311
5312 /*
5313 * These ioctl calls:
5314 * - require superuser power.
5315 * - require strict serialization.
5316 * - do not return a value
5317 */
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +00005318 case SIOCSIFMAP:
5319 case SIOCSIFTXQLEN:
5320 if (!capable(CAP_NET_ADMIN))
5321 return -EPERM;
5322 /* fall through */
5323 /*
5324 * These ioctl calls:
5325 * - require local superuser power.
5326 * - require strict serialization.
5327 * - do not return a value
5328 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005329 case SIOCSIFFLAGS:
5330 case SIOCSIFMETRIC:
5331 case SIOCSIFMTU:
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005332 case SIOCSIFHWADDR:
5333 case SIOCSIFSLAVE:
5334 case SIOCADDMULTI:
5335 case SIOCDELMULTI:
5336 case SIOCSIFHWBROADCAST:
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005337 case SIOCSMIIREG:
5338 case SIOCBONDENSLAVE:
5339 case SIOCBONDRELEASE:
5340 case SIOCBONDSETHWADDR:
5341 case SIOCBONDCHANGEACTIVE:
5342 case SIOCBRADDIF:
5343 case SIOCBRDELIF:
5344 case SIOCSHWTSTAMP:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +00005345 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005346 return -EPERM;
5347 /* fall through */
5348 case SIOCBONDSLAVEINFOQUERY:
5349 case SIOCBONDINFOQUERY:
5350 dev_load(net, ifr.ifr_name);
5351 rtnl_lock();
5352 ret = dev_ifsioc(net, &ifr, cmd);
5353 rtnl_unlock();
5354 return ret;
5355
5356 case SIOCGIFMEM:
5357 /* Get the per device memory space. We can add this but
5358 * currently do not support it */
5359 case SIOCSIFMEM:
5360 /* Set the per device memory buffer space.
5361 * Not applicable in our case */
5362 case SIOCSIFLINK:
Lifeng Sun41c31f32011-04-27 22:04:51 +00005363 return -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005364
5365 /*
5366 * Unknown or private ioctl.
5367 */
5368 default:
5369 if (cmd == SIOCWANDEV ||
5370 (cmd >= SIOCDEVPRIVATE &&
5371 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005372 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005373 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005374 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005375 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005376 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005377 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005378 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005379 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005380 }
5381 /* Take care of Wireless Extensions */
5382 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5383 return wext_handle_ioctl(net, &ifr, cmd, arg);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005384 return -ENOTTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005385 }
5386}
5387
5388
5389/**
5390 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005391 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005392 *
5393 * Returns a suitable unique value for a new device interface
5394 * number. The caller must hold the rtnl semaphore or the
5395 * dev_base_lock to be sure it remains unique.
5396 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005397static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005398{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005399 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005400 for (;;) {
5401 if (++ifindex <= 0)
5402 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005403 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005404 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005405 }
5406}
5407
Linus Torvalds1da177e2005-04-16 15:20:36 -07005408/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005409static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005410
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005411static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005412{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005413 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005414}
5415
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005416static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005417{
Krishna Kumare93737b2009-12-08 22:26:02 +00005418 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005419
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005420 BUG_ON(dev_boot_phase);
5421 ASSERT_RTNL();
5422
Krishna Kumare93737b2009-12-08 22:26:02 +00005423 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005424 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005425 * for initialization unwind. Remove those
5426 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005427 */
5428 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005429 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
5430 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005431
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005432 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005433 list_del(&dev->unreg_list);
5434 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005435 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005436 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005437 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005438 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005439
Octavian Purdila44345722010-12-13 12:44:07 +00005440 /* If device is running, close it first. */
5441 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005442
Octavian Purdila44345722010-12-13 12:44:07 +00005443 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005444 /* And unlink it from device chain. */
5445 unlist_netdevice(dev);
5446
5447 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005448 }
5449
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005450 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005451
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005452 list_for_each_entry(dev, head, unreg_list) {
5453 /* Shutdown queueing discipline. */
5454 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005455
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005456
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005457 /* Notify protocols, that we are about to destroy
5458 this device. They should clean all the things.
5459 */
5460 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5461
Patrick McHardya2835762010-02-26 06:34:51 +00005462 if (!dev->rtnl_link_ops ||
5463 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5464 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5465
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005466 /*
5467 * Flush the unicast and multicast chains
5468 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005469 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00005470 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005471
5472 if (dev->netdev_ops->ndo_uninit)
5473 dev->netdev_ops->ndo_uninit(dev);
5474
5475 /* Notifier chain MUST detach us from master device. */
5476 WARN_ON(dev->master);
5477
5478 /* Remove entries from kobject tree */
5479 netdev_unregister_kobject(dev);
5480 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005481
Eric W. Biederman850a5452011-10-13 22:25:23 +00005482 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005483
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005484 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005485 dev_put(dev);
5486}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005487
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005488static void rollback_registered(struct net_device *dev)
5489{
5490 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005491
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005492 list_add(&dev->unreg_list, &single);
5493 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005494 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005495}
5496
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005497static netdev_features_t netdev_fix_features(struct net_device *dev,
5498 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07005499{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005500 /* Fix illegal checksum combinations */
5501 if ((features & NETIF_F_HW_CSUM) &&
5502 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005503 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005504 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5505 }
5506
Herbert Xub63365a2008-10-23 01:11:29 -07005507 /* Fix illegal SG+CSUM combinations. */
5508 if ((features & NETIF_F_SG) &&
5509 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005510 netdev_dbg(dev,
5511 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005512 features &= ~NETIF_F_SG;
5513 }
5514
5515 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005516 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005517 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005518 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005519 }
5520
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005521 /* TSO ECN requires that TSO is present as well. */
5522 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5523 features &= ~NETIF_F_TSO_ECN;
5524
Michał Mirosław212b5732011-02-15 16:59:16 +00005525 /* Software GSO depends on SG. */
5526 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005527 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005528 features &= ~NETIF_F_GSO;
5529 }
5530
Michał Mirosławacd11302011-01-24 15:45:15 -08005531 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005532 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005533 /* maybe split UFO into V4 and V6? */
5534 if (!((features & NETIF_F_GEN_CSUM) ||
5535 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5536 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005537 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005538 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005539 features &= ~NETIF_F_UFO;
5540 }
5541
5542 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005543 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005544 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005545 features &= ~NETIF_F_UFO;
5546 }
5547 }
5548
5549 return features;
5550}
Herbert Xub63365a2008-10-23 01:11:29 -07005551
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005552int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005553{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005554 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00005555 int err = 0;
5556
Michał Mirosław87267482011-04-12 09:56:38 +00005557 ASSERT_RTNL();
5558
Michał Mirosław5455c692011-02-15 16:59:17 +00005559 features = netdev_get_wanted_features(dev);
5560
5561 if (dev->netdev_ops->ndo_fix_features)
5562 features = dev->netdev_ops->ndo_fix_features(dev, features);
5563
5564 /* driver might be less strict about feature dependencies */
5565 features = netdev_fix_features(dev, features);
5566
5567 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005568 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005569
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005570 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5571 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00005572
5573 if (dev->netdev_ops->ndo_set_features)
5574 err = dev->netdev_ops->ndo_set_features(dev, features);
5575
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005576 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005577 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005578 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5579 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005580 return -1;
5581 }
5582
5583 if (!err)
5584 dev->features = features;
5585
5586 return 1;
5587}
5588
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005589/**
5590 * netdev_update_features - recalculate device features
5591 * @dev: the device to check
5592 *
5593 * Recalculate dev->features set and send notifications if it
5594 * has changed. Should be called after driver or hardware dependent
5595 * conditions might have changed that influence the features.
5596 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005597void netdev_update_features(struct net_device *dev)
5598{
5599 if (__netdev_update_features(dev))
5600 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005601}
5602EXPORT_SYMBOL(netdev_update_features);
5603
Linus Torvalds1da177e2005-04-16 15:20:36 -07005604/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005605 * netdev_change_features - recalculate device features
5606 * @dev: the device to check
5607 *
5608 * Recalculate dev->features set and send notifications even
5609 * if they have not changed. Should be called instead of
5610 * netdev_update_features() if also dev->vlan_features might
5611 * have changed to allow the changes to be propagated to stacked
5612 * VLAN devices.
5613 */
5614void netdev_change_features(struct net_device *dev)
5615{
5616 __netdev_update_features(dev);
5617 netdev_features_change(dev);
5618}
5619EXPORT_SYMBOL(netdev_change_features);
5620
5621/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005622 * netif_stacked_transfer_operstate - transfer operstate
5623 * @rootdev: the root or lower level device to transfer state from
5624 * @dev: the device to transfer operstate to
5625 *
5626 * Transfer operational state from root to device. This is normally
5627 * called when a stacking relationship exists between the root
5628 * device and the device(a leaf device).
5629 */
5630void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5631 struct net_device *dev)
5632{
5633 if (rootdev->operstate == IF_OPER_DORMANT)
5634 netif_dormant_on(dev);
5635 else
5636 netif_dormant_off(dev);
5637
5638 if (netif_carrier_ok(rootdev)) {
5639 if (!netif_carrier_ok(dev))
5640 netif_carrier_on(dev);
5641 } else {
5642 if (netif_carrier_ok(dev))
5643 netif_carrier_off(dev);
5644 }
5645}
5646EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5647
Tom Herbertbf264142010-11-26 08:36:09 +00005648#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005649static int netif_alloc_rx_queues(struct net_device *dev)
5650{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005651 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005652 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005653
Tom Herbertbd25fa72010-10-18 18:00:16 +00005654 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005655
Tom Herbertbd25fa72010-10-18 18:00:16 +00005656 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5657 if (!rx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005658 pr_err("netdev: Unable to allocate %u rx queues\n", count);
Tom Herbertbd25fa72010-10-18 18:00:16 +00005659 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005660 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005661 dev->_rx = rx;
5662
Tom Herbertbd25fa72010-10-18 18:00:16 +00005663 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005664 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005665 return 0;
5666}
Tom Herbertbf264142010-11-26 08:36:09 +00005667#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005668
Changli Gaoaa942102010-12-04 02:31:41 +00005669static void netdev_init_one_queue(struct net_device *dev,
5670 struct netdev_queue *queue, void *_unused)
5671{
5672 /* Initialize queue lock */
5673 spin_lock_init(&queue->_xmit_lock);
5674 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5675 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005676 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005677 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00005678#ifdef CONFIG_BQL
5679 dql_init(&queue->dql, HZ);
5680#endif
Changli Gaoaa942102010-12-04 02:31:41 +00005681}
5682
Tom Herberte6484932010-10-18 18:04:39 +00005683static int netif_alloc_netdev_queues(struct net_device *dev)
5684{
5685 unsigned int count = dev->num_tx_queues;
5686 struct netdev_queue *tx;
5687
5688 BUG_ON(count < 1);
5689
5690 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5691 if (!tx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005692 pr_err("netdev: Unable to allocate %u tx queues\n", count);
Tom Herberte6484932010-10-18 18:04:39 +00005693 return -ENOMEM;
5694 }
5695 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005696
Tom Herberte6484932010-10-18 18:04:39 +00005697 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5698 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005699
5700 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005701}
5702
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005703/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005704 * register_netdevice - register a network device
5705 * @dev: device to register
5706 *
5707 * Take a completed network device structure and add it to the kernel
5708 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5709 * chain. 0 is returned on success. A negative errno code is returned
5710 * on a failure to set up the device, or if the name is a duplicate.
5711 *
5712 * Callers must hold the rtnl semaphore. You may want
5713 * register_netdev() instead of this.
5714 *
5715 * BUGS:
5716 * The locking appears insufficient to guarantee two parallel registers
5717 * will not get the same name.
5718 */
5719
5720int register_netdevice(struct net_device *dev)
5721{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005722 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005723 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005724
5725 BUG_ON(dev_boot_phase);
5726 ASSERT_RTNL();
5727
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005728 might_sleep();
5729
Linus Torvalds1da177e2005-04-16 15:20:36 -07005730 /* When net_device's are persistent, this will be fatal. */
5731 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005732 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005733
David S. Millerf1f28aa2008-07-15 00:08:33 -07005734 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005735 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005736
Linus Torvalds1da177e2005-04-16 15:20:36 -07005737 dev->iflink = -1;
5738
Gao feng828de4f2012-09-13 20:58:27 +00005739 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005740 if (ret < 0)
5741 goto out;
5742
Linus Torvalds1da177e2005-04-16 15:20:36 -07005743 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005744 if (dev->netdev_ops->ndo_init) {
5745 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005746 if (ret) {
5747 if (ret > 0)
5748 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005749 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005750 }
5751 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005752
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00005753 ret = -EBUSY;
5754 if (!dev->ifindex)
5755 dev->ifindex = dev_new_index(net);
5756 else if (__dev_get_by_index(net, dev->ifindex))
5757 goto err_uninit;
5758
Linus Torvalds1da177e2005-04-16 15:20:36 -07005759 if (dev->iflink == -1)
5760 dev->iflink = dev->ifindex;
5761
Michał Mirosław5455c692011-02-15 16:59:17 +00005762 /* Transfer changeable features to wanted_features and enable
5763 * software offloads (GSO and GRO).
5764 */
5765 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005766 dev->features |= NETIF_F_SOFT_FEATURES;
5767 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005768
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005769 /* Turn on no cache copy if HW is doing checksum */
Michał Mirosław34324dc2011-11-15 15:29:55 +00005770 if (!(dev->flags & IFF_LOOPBACK)) {
5771 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5772 if (dev->features & NETIF_F_ALL_CSUM) {
5773 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5774 dev->features |= NETIF_F_NOCACHE_COPY;
5775 }
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005776 }
5777
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005778 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005779 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005780 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005781
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005782 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5783 ret = notifier_to_errno(ret);
5784 if (ret)
5785 goto err_uninit;
5786
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005787 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005788 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005789 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005790 dev->reg_state = NETREG_REGISTERED;
5791
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005792 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005793
Linus Torvalds1da177e2005-04-16 15:20:36 -07005794 /*
5795 * Default initial state at registry is that the
5796 * device is present.
5797 */
5798
5799 set_bit(__LINK_STATE_PRESENT, &dev->state);
5800
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01005801 linkwatch_init_dev(dev);
5802
Linus Torvalds1da177e2005-04-16 15:20:36 -07005803 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005804 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005805 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005806 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005807
5808 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005809 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005810 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005811 if (ret) {
5812 rollback_registered(dev);
5813 dev->reg_state = NETREG_UNREGISTERED;
5814 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005815 /*
5816 * Prevent userspace races by waiting until the network
5817 * device is fully setup before sending notifications.
5818 */
Patrick McHardya2835762010-02-26 06:34:51 +00005819 if (!dev->rtnl_link_ops ||
5820 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5821 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005822
5823out:
5824 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005825
5826err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005827 if (dev->netdev_ops->ndo_uninit)
5828 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005829 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005831EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005832
5833/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005834 * init_dummy_netdev - init a dummy network device for NAPI
5835 * @dev: device to init
5836 *
5837 * This takes a network device structure and initialize the minimum
5838 * amount of fields so it can be used to schedule NAPI polls without
5839 * registering a full blown interface. This is to be used by drivers
5840 * that need to tie several hardware interfaces to a single NAPI
5841 * poll scheduler due to HW limitations.
5842 */
5843int init_dummy_netdev(struct net_device *dev)
5844{
5845 /* Clear everything. Note we don't initialize spinlocks
5846 * are they aren't supposed to be taken by any of the
5847 * NAPI code and this dummy netdev is supposed to be
5848 * only ever used for NAPI polls
5849 */
5850 memset(dev, 0, sizeof(struct net_device));
5851
5852 /* make sure we BUG if trying to hit standard
5853 * register/unregister code path
5854 */
5855 dev->reg_state = NETREG_DUMMY;
5856
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005857 /* NAPI wants this */
5858 INIT_LIST_HEAD(&dev->napi_list);
5859
5860 /* a dummy interface is started by default */
5861 set_bit(__LINK_STATE_PRESENT, &dev->state);
5862 set_bit(__LINK_STATE_START, &dev->state);
5863
Eric Dumazet29b44332010-10-11 10:22:12 +00005864 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5865 * because users of this 'device' dont need to change
5866 * its refcount.
5867 */
5868
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005869 return 0;
5870}
5871EXPORT_SYMBOL_GPL(init_dummy_netdev);
5872
5873
5874/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005875 * register_netdev - register a network device
5876 * @dev: device to register
5877 *
5878 * Take a completed network device structure and add it to the kernel
5879 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5880 * chain. 0 is returned on success. A negative errno code is returned
5881 * on a failure to set up the device, or if the name is a duplicate.
5882 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005883 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005884 * and expands the device name if you passed a format string to
5885 * alloc_netdev.
5886 */
5887int register_netdev(struct net_device *dev)
5888{
5889 int err;
5890
5891 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005892 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005893 rtnl_unlock();
5894 return err;
5895}
5896EXPORT_SYMBOL(register_netdev);
5897
Eric Dumazet29b44332010-10-11 10:22:12 +00005898int netdev_refcnt_read(const struct net_device *dev)
5899{
5900 int i, refcnt = 0;
5901
5902 for_each_possible_cpu(i)
5903 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5904 return refcnt;
5905}
5906EXPORT_SYMBOL(netdev_refcnt_read);
5907
Ben Hutchings2c530402012-07-10 10:55:09 +00005908/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005909 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00005910 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005911 *
5912 * This is called when unregistering network devices.
5913 *
5914 * Any protocol or device that holds a reference should register
5915 * for netdevice notification, and cleanup and put back the
5916 * reference if they receive an UNREGISTER event.
5917 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005918 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005919 */
5920static void netdev_wait_allrefs(struct net_device *dev)
5921{
5922 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005923 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005924
Eric Dumazete014deb2009-11-17 05:59:21 +00005925 linkwatch_forget_dev(dev);
5926
Linus Torvalds1da177e2005-04-16 15:20:36 -07005927 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005928 refcnt = netdev_refcnt_read(dev);
5929
5930 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005931 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005932 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005933
5934 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005935 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005936
Eric Dumazet748e2d92012-08-22 21:50:59 +00005937 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005938 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00005939 rtnl_lock();
5940
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005941 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005942 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5943 &dev->state)) {
5944 /* We must not have linkwatch events
5945 * pending on unregister. If this
5946 * happens, we simply run the queue
5947 * unscheduled, resulting in a noop
5948 * for this device.
5949 */
5950 linkwatch_run_queue();
5951 }
5952
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005953 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005954
5955 rebroadcast_time = jiffies;
5956 }
5957
5958 msleep(250);
5959
Eric Dumazet29b44332010-10-11 10:22:12 +00005960 refcnt = netdev_refcnt_read(dev);
5961
Linus Torvalds1da177e2005-04-16 15:20:36 -07005962 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005963 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5964 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005965 warning_time = jiffies;
5966 }
5967 }
5968}
5969
5970/* The sequence is:
5971 *
5972 * rtnl_lock();
5973 * ...
5974 * register_netdevice(x1);
5975 * register_netdevice(x2);
5976 * ...
5977 * unregister_netdevice(y1);
5978 * unregister_netdevice(y2);
5979 * ...
5980 * rtnl_unlock();
5981 * free_netdev(y1);
5982 * free_netdev(y2);
5983 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005984 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005985 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005986 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005987 * without deadlocking with linkwatch via keventd.
5988 * 2) Since we run with the RTNL semaphore not held, we can sleep
5989 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005990 *
5991 * We must not return until all unregister events added during
5992 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005993 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005994void netdev_run_todo(void)
5995{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005996 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005997
Linus Torvalds1da177e2005-04-16 15:20:36 -07005998 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005999 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07006000
6001 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006002
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006003
6004 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00006005 if (!list_empty(&list))
6006 rcu_barrier();
6007
Linus Torvalds1da177e2005-04-16 15:20:36 -07006008 while (!list_empty(&list)) {
6009 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00006010 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006011 list_del(&dev->todo_list);
6012
Eric Dumazet748e2d92012-08-22 21:50:59 +00006013 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006014 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00006015 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006016
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006017 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006018 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07006019 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006020 dump_stack();
6021 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006022 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006023
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006024 dev->reg_state = NETREG_UNREGISTERED;
6025
Changli Gao152102c2010-03-30 20:16:22 +00006026 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07006027
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006028 netdev_wait_allrefs(dev);
6029
6030 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00006031 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00006032 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6033 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07006034 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006035
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006036 if (dev->destructor)
6037 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006038
6039 /* Free network device */
6040 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006041 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006042}
6043
Ben Hutchings3cfde792010-07-09 09:11:52 +00006044/* Convert net_device_stats to rtnl_link_stats64. They have the same
6045 * fields in the same order, with only the type differing.
6046 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006047void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6048 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00006049{
6050#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006051 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6052 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00006053#else
6054 size_t i, n = sizeof(*stats64) / sizeof(u64);
6055 const unsigned long *src = (const unsigned long *)netdev_stats;
6056 u64 *dst = (u64 *)stats64;
6057
6058 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6059 sizeof(*stats64) / sizeof(u64));
6060 for (i = 0; i < n; i++)
6061 dst[i] = src[i];
6062#endif
6063}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006064EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00006065
Eric Dumazetd83345a2009-11-16 03:36:51 +00006066/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006067 * dev_get_stats - get network device statistics
6068 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07006069 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006070 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00006071 * Get network statistics from device. Return @storage.
6072 * The device driver may provide its own method by setting
6073 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6074 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006075 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00006076struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6077 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00006078{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006079 const struct net_device_ops *ops = dev->netdev_ops;
6080
Eric Dumazet28172732010-07-07 14:58:56 -07006081 if (ops->ndo_get_stats64) {
6082 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006083 ops->ndo_get_stats64(dev, storage);
6084 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00006085 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006086 } else {
6087 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07006088 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006089 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07006090 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07006091}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006092EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07006093
Eric Dumazet24824a02010-10-02 06:11:55 +00006094struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07006095{
Eric Dumazet24824a02010-10-02 06:11:55 +00006096 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07006097
Eric Dumazet24824a02010-10-02 06:11:55 +00006098#ifdef CONFIG_NET_CLS_ACT
6099 if (queue)
6100 return queue;
6101 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
6102 if (!queue)
6103 return NULL;
6104 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00006105 queue->qdisc = &noop_qdisc;
6106 queue->qdisc_sleeping = &noop_qdisc;
6107 rcu_assign_pointer(dev->ingress_queue, queue);
6108#endif
6109 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07006110}
6111
Eric Dumazet2c60db02012-09-16 09:17:26 +00006112static const struct ethtool_ops default_ethtool_ops;
6113
Linus Torvalds1da177e2005-04-16 15:20:36 -07006114/**
Tom Herbert36909ea2011-01-09 19:36:31 +00006115 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006116 * @sizeof_priv: size of private data to allocate space for
6117 * @name: device name format string
6118 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00006119 * @txqs: the number of TX subqueues to allocate
6120 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07006121 *
6122 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07006123 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00006124 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006125 */
Tom Herbert36909ea2011-01-09 19:36:31 +00006126struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
6127 void (*setup)(struct net_device *),
6128 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006129{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006130 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07006131 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006132 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006133
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07006134 BUG_ON(strlen(name) >= sizeof(dev->name));
6135
Tom Herbert36909ea2011-01-09 19:36:31 +00006136 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006137 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00006138 return NULL;
6139 }
6140
Tom Herbert36909ea2011-01-09 19:36:31 +00006141#ifdef CONFIG_RPS
6142 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006143 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00006144 return NULL;
6145 }
6146#endif
6147
David S. Millerfd2ea0a2008-07-17 01:56:23 -07006148 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006149 if (sizeof_priv) {
6150 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006151 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006152 alloc_size += sizeof_priv;
6153 }
6154 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006155 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006156
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07006157 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006158 if (!p) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006159 pr_err("alloc_netdev: Unable to allocate device\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07006160 return NULL;
6161 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006162
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006163 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006164 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006165
Eric Dumazet29b44332010-10-11 10:22:12 +00006166 dev->pcpu_refcnt = alloc_percpu(int);
6167 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00006168 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006169
Linus Torvalds1da177e2005-04-16 15:20:36 -07006170 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006171 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006172
Jiri Pirko22bedad2010-04-01 21:22:57 +00006173 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006174 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006175
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006176 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006177
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006178 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00006179 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006180
Herbert Xud565b0a2008-12-15 23:38:52 -08006181 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006182 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006183 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07006184 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006185 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006186
6187 dev->num_tx_queues = txqs;
6188 dev->real_num_tx_queues = txqs;
6189 if (netif_alloc_netdev_queues(dev))
6190 goto free_all;
6191
6192#ifdef CONFIG_RPS
6193 dev->num_rx_queues = rxqs;
6194 dev->real_num_rx_queues = rxqs;
6195 if (netif_alloc_rx_queues(dev))
6196 goto free_all;
6197#endif
6198
Linus Torvalds1da177e2005-04-16 15:20:36 -07006199 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006200 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00006201 if (!dev->ethtool_ops)
6202 dev->ethtool_ops = &default_ethtool_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006203 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006204
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006205free_all:
6206 free_netdev(dev);
6207 return NULL;
6208
Eric Dumazet29b44332010-10-11 10:22:12 +00006209free_pcpu:
6210 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00006211 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006212#ifdef CONFIG_RPS
6213 kfree(dev->_rx);
6214#endif
6215
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006216free_p:
6217 kfree(p);
6218 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006219}
Tom Herbert36909ea2011-01-09 19:36:31 +00006220EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006221
6222/**
6223 * free_netdev - free network device
6224 * @dev: device
6225 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006226 * This function does the last stage of destroying an allocated device
6227 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006228 * If this is the last reference then it will be freed.
6229 */
6230void free_netdev(struct net_device *dev)
6231{
Herbert Xud565b0a2008-12-15 23:38:52 -08006232 struct napi_struct *p, *n;
6233
Denis V. Lunevf3005d72008-04-16 02:02:18 -07006234 release_net(dev_net(dev));
6235
David S. Millere8a04642008-07-17 00:34:19 -07006236 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006237#ifdef CONFIG_RPS
6238 kfree(dev->_rx);
6239#endif
David S. Millere8a04642008-07-17 00:34:19 -07006240
Eric Dumazet33d480c2011-08-11 19:30:52 +00006241 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006242
Jiri Pirkof001fde2009-05-05 02:48:28 +00006243 /* Flush device addresses */
6244 dev_addr_flush(dev);
6245
Herbert Xud565b0a2008-12-15 23:38:52 -08006246 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6247 netif_napi_del(p);
6248
Eric Dumazet29b44332010-10-11 10:22:12 +00006249 free_percpu(dev->pcpu_refcnt);
6250 dev->pcpu_refcnt = NULL;
6251
Stephen Hemminger3041a062006-05-26 13:25:24 -07006252 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006253 if (dev->reg_state == NETREG_UNINITIALIZED) {
6254 kfree((char *)dev - dev->padded);
6255 return;
6256 }
6257
6258 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6259 dev->reg_state = NETREG_RELEASED;
6260
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006261 /* will free via device release */
6262 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006263}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006264EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006265
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006266/**
6267 * synchronize_net - Synchronize with packet receive processing
6268 *
6269 * Wait for packets currently being received to be done.
6270 * Does not block later packets from starting.
6271 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006272void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006273{
6274 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006275 if (rtnl_is_locked())
6276 synchronize_rcu_expedited();
6277 else
6278 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006279}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006280EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006281
6282/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006283 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006284 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006285 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006286 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006287 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006288 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006289 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006290 *
6291 * Callers must hold the rtnl semaphore. You may want
6292 * unregister_netdev() instead of this.
6293 */
6294
Eric Dumazet44a08732009-10-27 07:03:04 +00006295void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006296{
Herbert Xua6620712007-12-12 19:21:56 -08006297 ASSERT_RTNL();
6298
Eric Dumazet44a08732009-10-27 07:03:04 +00006299 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006300 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006301 } else {
6302 rollback_registered(dev);
6303 /* Finish processing unregister after unlock */
6304 net_set_todo(dev);
6305 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006306}
Eric Dumazet44a08732009-10-27 07:03:04 +00006307EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006308
6309/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006310 * unregister_netdevice_many - unregister many devices
6311 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006312 */
6313void unregister_netdevice_many(struct list_head *head)
6314{
6315 struct net_device *dev;
6316
6317 if (!list_empty(head)) {
6318 rollback_registered_many(head);
6319 list_for_each_entry(dev, head, unreg_list)
6320 net_set_todo(dev);
6321 }
6322}
Eric Dumazet63c80992009-10-27 07:06:49 +00006323EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006324
6325/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006326 * unregister_netdev - remove device from the kernel
6327 * @dev: device
6328 *
6329 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006330 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006331 *
6332 * This is just a wrapper for unregister_netdevice that takes
6333 * the rtnl semaphore. In general you want to use this and not
6334 * unregister_netdevice.
6335 */
6336void unregister_netdev(struct net_device *dev)
6337{
6338 rtnl_lock();
6339 unregister_netdevice(dev);
6340 rtnl_unlock();
6341}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006342EXPORT_SYMBOL(unregister_netdev);
6343
Eric W. Biedermance286d32007-09-12 13:53:49 +02006344/**
6345 * dev_change_net_namespace - move device to different nethost namespace
6346 * @dev: device
6347 * @net: network namespace
6348 * @pat: If not NULL name pattern to try if the current device name
6349 * is already taken in the destination network namespace.
6350 *
6351 * This function shuts down a device interface and moves it
6352 * to a new network namespace. On success 0 is returned, on
6353 * a failure a netagive errno code is returned.
6354 *
6355 * Callers must hold the rtnl semaphore.
6356 */
6357
6358int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6359{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006360 int err;
6361
6362 ASSERT_RTNL();
6363
6364 /* Don't allow namespace local devices to be moved. */
6365 err = -EINVAL;
6366 if (dev->features & NETIF_F_NETNS_LOCAL)
6367 goto out;
6368
6369 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02006370 if (dev->reg_state != NETREG_REGISTERED)
6371 goto out;
6372
6373 /* Get out if there is nothing todo */
6374 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006375 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006376 goto out;
6377
6378 /* Pick the destination device name, and ensure
6379 * we can use it in the destination network namespace.
6380 */
6381 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006382 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006383 /* We get here if we can't use the current device name */
6384 if (!pat)
6385 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00006386 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006387 goto out;
6388 }
6389
6390 /*
6391 * And now a mini version of register_netdevice unregister_netdevice.
6392 */
6393
6394 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006395 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006396
6397 /* And unlink it from device chain */
6398 err = -ENODEV;
6399 unlist_netdevice(dev);
6400
6401 synchronize_net();
6402
6403 /* Shutdown queueing discipline. */
6404 dev_shutdown(dev);
6405
6406 /* Notify protocols, that we are about to destroy
6407 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006408
6409 Note that dev->reg_state stays at NETREG_REGISTERED.
6410 This is wanted because this way 8021q and macvlan know
6411 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006412 */
6413 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00006414 rcu_barrier();
6415 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00006416 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006417
6418 /*
6419 * Flush the unicast and multicast chains
6420 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006421 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00006422 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006423
6424 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006425 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006426
Eric W. Biedermance286d32007-09-12 13:53:49 +02006427 /* If there is an ifindex conflict assign a new one */
6428 if (__dev_get_by_index(net, dev->ifindex)) {
6429 int iflink = (dev->iflink == dev->ifindex);
6430 dev->ifindex = dev_new_index(net);
6431 if (iflink)
6432 dev->iflink = dev->ifindex;
6433 }
6434
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006435 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006436 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006437 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006438
6439 /* Add the device back in the hashes */
6440 list_netdevice(dev);
6441
6442 /* Notify protocols, that a new device appeared. */
6443 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6444
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006445 /*
6446 * Prevent userspace races by waiting until the network
6447 * device is fully setup before sending notifications.
6448 */
6449 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6450
Eric W. Biedermance286d32007-09-12 13:53:49 +02006451 synchronize_net();
6452 err = 0;
6453out:
6454 return err;
6455}
Johannes Berg463d0182009-07-14 00:33:35 +02006456EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006457
Linus Torvalds1da177e2005-04-16 15:20:36 -07006458static int dev_cpu_callback(struct notifier_block *nfb,
6459 unsigned long action,
6460 void *ocpu)
6461{
6462 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006463 struct sk_buff *skb;
6464 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6465 struct softnet_data *sd, *oldsd;
6466
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006467 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006468 return NOTIFY_OK;
6469
6470 local_irq_disable();
6471 cpu = smp_processor_id();
6472 sd = &per_cpu(softnet_data, cpu);
6473 oldsd = &per_cpu(softnet_data, oldcpu);
6474
6475 /* Find end of our completion_queue. */
6476 list_skb = &sd->completion_queue;
6477 while (*list_skb)
6478 list_skb = &(*list_skb)->next;
6479 /* Append completion queue from offline CPU. */
6480 *list_skb = oldsd->completion_queue;
6481 oldsd->completion_queue = NULL;
6482
Linus Torvalds1da177e2005-04-16 15:20:36 -07006483 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006484 if (oldsd->output_queue) {
6485 *sd->output_queue_tailp = oldsd->output_queue;
6486 sd->output_queue_tailp = oldsd->output_queue_tailp;
6487 oldsd->output_queue = NULL;
6488 oldsd->output_queue_tailp = &oldsd->output_queue;
6489 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006490 /* Append NAPI poll list from offline CPU. */
6491 if (!list_empty(&oldsd->poll_list)) {
6492 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6493 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6494 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006495
6496 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6497 local_irq_enable();
6498
6499 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006500 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6501 netif_rx(skb);
6502 input_queue_head_incr(oldsd);
6503 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006504 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006505 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006506 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006507 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006508
6509 return NOTIFY_OK;
6510}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006511
6512
Herbert Xu7f353bf2007-08-10 15:47:58 -07006513/**
Herbert Xub63365a2008-10-23 01:11:29 -07006514 * netdev_increment_features - increment feature set by one
6515 * @all: current feature set
6516 * @one: new feature set
6517 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006518 *
6519 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006520 * @one to the master device with current feature set @all. Will not
6521 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006522 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006523netdev_features_t netdev_increment_features(netdev_features_t all,
6524 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006525{
Michał Mirosław1742f182011-04-22 06:31:16 +00006526 if (mask & NETIF_F_GEN_CSUM)
6527 mask |= NETIF_F_ALL_CSUM;
6528 mask |= NETIF_F_VLAN_CHALLENGED;
6529
6530 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6531 all &= one | ~NETIF_F_ALL_FOR_ALL;
6532
Michał Mirosław1742f182011-04-22 06:31:16 +00006533 /* If one device supports hw checksumming, set for all. */
6534 if (all & NETIF_F_GEN_CSUM)
6535 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006536
6537 return all;
6538}
Herbert Xub63365a2008-10-23 01:11:29 -07006539EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006540
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006541static struct hlist_head *netdev_create_hash(void)
6542{
6543 int i;
6544 struct hlist_head *hash;
6545
6546 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6547 if (hash != NULL)
6548 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6549 INIT_HLIST_HEAD(&hash[i]);
6550
6551 return hash;
6552}
6553
Eric W. Biederman881d9662007-09-17 11:56:21 -07006554/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006555static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006556{
Rustad, Mark D734b6542012-07-18 09:06:07 +00006557 if (net != &init_net)
6558 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006559
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006560 net->dev_name_head = netdev_create_hash();
6561 if (net->dev_name_head == NULL)
6562 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006563
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006564 net->dev_index_head = netdev_create_hash();
6565 if (net->dev_index_head == NULL)
6566 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006567
6568 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006569
6570err_idx:
6571 kfree(net->dev_name_head);
6572err_name:
6573 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006574}
6575
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006576/**
6577 * netdev_drivername - network driver for the device
6578 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006579 *
6580 * Determine network driver for device.
6581 */
David S. Miller3019de12011-06-06 16:41:33 -07006582const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006583{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006584 const struct device_driver *driver;
6585 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006586 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006587
6588 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006589 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006590 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006591
6592 driver = parent->driver;
6593 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006594 return driver->name;
6595 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006596}
6597
Joe Perchesb004ff42012-09-12 20:12:19 -07006598static int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006599 struct va_format *vaf)
6600{
6601 int r;
6602
Joe Perchesb004ff42012-09-12 20:12:19 -07006603 if (dev && dev->dev.parent) {
Joe Perches666f3552012-09-12 20:14:11 -07006604 r = dev_printk_emit(level[1] - '0',
6605 dev->dev.parent,
6606 "%s %s %s: %pV",
6607 dev_driver_string(dev->dev.parent),
6608 dev_name(dev->dev.parent),
6609 netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006610 } else if (dev) {
Joe Perches256df2f2010-06-27 01:02:35 +00006611 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006612 } else {
Joe Perches256df2f2010-06-27 01:02:35 +00006613 r = printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006614 }
Joe Perches256df2f2010-06-27 01:02:35 +00006615
6616 return r;
6617}
6618
6619int netdev_printk(const char *level, const struct net_device *dev,
6620 const char *format, ...)
6621{
6622 struct va_format vaf;
6623 va_list args;
6624 int r;
6625
6626 va_start(args, format);
6627
6628 vaf.fmt = format;
6629 vaf.va = &args;
6630
6631 r = __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006632
Joe Perches256df2f2010-06-27 01:02:35 +00006633 va_end(args);
6634
6635 return r;
6636}
6637EXPORT_SYMBOL(netdev_printk);
6638
6639#define define_netdev_printk_level(func, level) \
6640int func(const struct net_device *dev, const char *fmt, ...) \
6641{ \
6642 int r; \
6643 struct va_format vaf; \
6644 va_list args; \
6645 \
6646 va_start(args, fmt); \
6647 \
6648 vaf.fmt = fmt; \
6649 vaf.va = &args; \
6650 \
6651 r = __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07006652 \
Joe Perches256df2f2010-06-27 01:02:35 +00006653 va_end(args); \
6654 \
6655 return r; \
6656} \
6657EXPORT_SYMBOL(func);
6658
6659define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6660define_netdev_printk_level(netdev_alert, KERN_ALERT);
6661define_netdev_printk_level(netdev_crit, KERN_CRIT);
6662define_netdev_printk_level(netdev_err, KERN_ERR);
6663define_netdev_printk_level(netdev_warn, KERN_WARNING);
6664define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6665define_netdev_printk_level(netdev_info, KERN_INFO);
6666
Pavel Emelyanov46650792007-10-08 20:38:39 -07006667static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006668{
6669 kfree(net->dev_name_head);
6670 kfree(net->dev_index_head);
6671}
6672
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006673static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006674 .init = netdev_init,
6675 .exit = netdev_exit,
6676};
6677
Pavel Emelyanov46650792007-10-08 20:38:39 -07006678static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006679{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006680 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006681 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006682 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006683 * initial network namespace
6684 */
6685 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006686 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006687 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006688 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006689
6690 /* Ignore unmoveable devices (i.e. loopback) */
6691 if (dev->features & NETIF_F_NETNS_LOCAL)
6692 continue;
6693
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006694 /* Leave virtual devices for the generic cleanup */
6695 if (dev->rtnl_link_ops)
6696 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006697
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006698 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006699 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6700 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006701 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006702 pr_emerg("%s: failed to move %s to init_net: %d\n",
6703 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006704 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006705 }
6706 }
6707 rtnl_unlock();
6708}
6709
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006710static void __net_exit default_device_exit_batch(struct list_head *net_list)
6711{
6712 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006713 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006714 * Do this across as many network namespaces as possible to
6715 * improve batching efficiency.
6716 */
6717 struct net_device *dev;
6718 struct net *net;
6719 LIST_HEAD(dev_kill_list);
6720
6721 rtnl_lock();
6722 list_for_each_entry(net, net_list, exit_list) {
6723 for_each_netdev_reverse(net, dev) {
6724 if (dev->rtnl_link_ops)
6725 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6726 else
6727 unregister_netdevice_queue(dev, &dev_kill_list);
6728 }
6729 }
6730 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006731 list_del(&dev_kill_list);
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006732 rtnl_unlock();
6733}
6734
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006735static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006736 .exit = default_device_exit,
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006737 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006738};
6739
Linus Torvalds1da177e2005-04-16 15:20:36 -07006740/*
6741 * Initialize the DEV module. At boot time this walks the device list and
6742 * unhooks any devices that fail to initialise (normally hardware not
6743 * present) and leaves us with a valid list of present and active devices.
6744 *
6745 */
6746
6747/*
6748 * This is called single threaded during boot, so no need
6749 * to take the rtnl semaphore.
6750 */
6751static int __init net_dev_init(void)
6752{
6753 int i, rc = -ENOMEM;
6754
6755 BUG_ON(!dev_boot_phase);
6756
Linus Torvalds1da177e2005-04-16 15:20:36 -07006757 if (dev_proc_init())
6758 goto out;
6759
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006760 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006761 goto out;
6762
6763 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006764 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006765 INIT_LIST_HEAD(&ptype_base[i]);
6766
Vlad Yasevich62532da2012-11-15 08:49:10 +00006767 INIT_LIST_HEAD(&offload_base);
6768
Eric W. Biederman881d9662007-09-17 11:56:21 -07006769 if (register_pernet_subsys(&netdev_net_ops))
6770 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006771
6772 /*
6773 * Initialise the packet receive queues.
6774 */
6775
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006776 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006777 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006778
Changli Gaodee42872010-05-02 05:42:16 +00006779 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006780 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006781 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006782 sd->completion_queue = NULL;
6783 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006784 sd->output_queue = NULL;
6785 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006786#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006787 sd->csd.func = rps_trigger_softirq;
6788 sd->csd.info = sd;
6789 sd->csd.flags = 0;
6790 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006791#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006792
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006793 sd->backlog.poll = process_backlog;
6794 sd->backlog.weight = weight_p;
6795 sd->backlog.gro_list = NULL;
6796 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006797 }
6798
Linus Torvalds1da177e2005-04-16 15:20:36 -07006799 dev_boot_phase = 0;
6800
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006801 /* The loopback device is special if any other network devices
6802 * is present in a network namespace the loopback device must
6803 * be present. Since we now dynamically allocate and free the
6804 * loopback device ensure this invariant is maintained by
6805 * keeping the loopback device as the first device on the
6806 * list of network devices. Ensuring the loopback devices
6807 * is the first device that appears and the last network device
6808 * that disappears.
6809 */
6810 if (register_pernet_device(&loopback_net_ops))
6811 goto out;
6812
6813 if (register_pernet_device(&default_device_ops))
6814 goto out;
6815
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006816 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6817 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006818
6819 hotcpu_notifier(dev_cpu_callback, 0);
6820 dst_init();
6821 dev_mcast_init();
6822 rc = 0;
6823out:
6824 return rc;
6825}
6826
6827subsys_initcall(net_dev_init);
6828
Krishna Kumare88721f2009-02-18 17:55:02 -08006829static int __init initialize_hashrnd(void)
6830{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006831 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006832 return 0;
6833}
6834
6835late_initcall_sync(initialize_hashrnd);
6836