blob: a4c4a1bf07d521047bf3401248a174e39978f05a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
100#include <linux/proc_fs.h>
101#include <linux/seq_file.h>
102#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <net/dst.h>
104#include <net/pkt_sched.h>
105#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000106#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107#include <linux/highmem.h>
108#include <linux/init.h>
109#include <linux/kmod.h>
110#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700114#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500117#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700118#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700119#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700120#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700121#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700122#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700123#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700124#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ipv6.h>
126#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700127#include <linux/jhash.h>
128#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700129#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900130#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900131#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000132#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700133#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000134#include <linux/cpu_rmap.h>
Richard Cochran4dc360c2011-10-19 17:00:35 -0400135#include <linux/net_tstamp.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100136#include <linux/static_key.h>
Eric Dumazet4504b862011-11-28 05:23:23 +0000137#include <net/flow_keys.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700139#include "net-sysfs.h"
140
Herbert Xud565b0a2008-12-15 23:38:52 -0800141/* Instead of increasing this, you should create a hash table. */
142#define MAX_GRO_SKBS 8
143
Herbert Xu5d38a072009-01-04 16:13:40 -0800144/* This should be increased if a protocol with a bigger head is added. */
145#define GRO_MAX_HEAD (MAX_HEADER + 128)
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147/*
148 * The list of packet types we will receive (as opposed to discard)
149 * and the routines to invoke.
150 *
151 * Why 16. Because with 16 the only overlap we get on a hash of the
152 * low nibble of the protocol value is RARP/SNAP/X.25.
153 *
154 * NOTE: That is no longer true with the addition of VLAN tags. Not
155 * sure which should go first, but I bet it won't make much
156 * difference if we are running VLANs. The good news is that
157 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700158 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 * --BLG
160 *
161 * 0800 IP
162 * 8100 802.1Q VLAN
163 * 0001 802.3
164 * 0002 AX.25
165 * 0004 802.2
166 * 8035 RARP
167 * 0005 SNAP
168 * 0805 X.25
169 * 0806 ARP
170 * 8137 IPX
171 * 0009 Localtalk
172 * 86DD IPv6
173 */
174
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800175#define PTYPE_HASH_SIZE (16)
176#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
177
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000179static DEFINE_SPINLOCK(offload_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800180static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700181static struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da2012-11-15 08:49:10 +0000182static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700185 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 * semaphore.
187 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800188 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 *
190 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700191 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 * actual updates. This allows pure readers to access the list even
193 * while a writer is preparing to update it.
194 *
195 * To put it another way, dev_base_lock is held for writing only to
196 * protect against pure readers; the rtnl semaphore provides the
197 * protection against other writers.
198 *
199 * See, for example usages, register_netdevice() and
200 * unregister_netdevice(), which must be called with the rtnl
201 * semaphore held.
202 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204EXPORT_SYMBOL(dev_base_lock);
205
Brian Haleyc91f6df2012-11-26 05:21:08 +0000206DEFINE_SEQLOCK(devnet_rename_seq);
207
Thomas Graf4e985ad2011-06-21 03:11:20 +0000208static inline void dev_base_seq_inc(struct net *net)
209{
210 while (++net->dev_base_seq == 0);
211}
212
Eric W. Biederman881d9662007-09-17 11:56:21 -0700213static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
Eric Dumazet95c96172012-04-15 05:58:06 +0000215 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
216
stephen hemminger08e98972009-11-10 07:20:34 +0000217 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218}
219
Eric W. Biederman881d9662007-09-17 11:56:21 -0700220static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700222 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
224
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000225static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000226{
227#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000229#endif
230}
231
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000232static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000233{
234#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000235 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000236#endif
237}
238
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239/* Device list insertion */
240static int list_netdevice(struct net_device *dev)
241{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900242 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200243
244 ASSERT_RTNL();
245
246 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800247 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000248 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000249 hlist_add_head_rcu(&dev->index_hlist,
250 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200251 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000252
253 dev_base_seq_inc(net);
254
Eric W. Biedermance286d32007-09-12 13:53:49 +0200255 return 0;
256}
257
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000258/* Device list removal
259 * caller must respect a RCU grace period before freeing/reusing dev
260 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200261static void unlist_netdevice(struct net_device *dev)
262{
263 ASSERT_RTNL();
264
265 /* Unlink dev from the device chain */
266 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800267 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000268 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000269 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200270 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000271
272 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200273}
274
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275/*
276 * Our notifier list
277 */
278
Alan Sternf07d5b92006-05-09 15:23:03 -0700279static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280
281/*
282 * Device drivers call our routines to queue packets here. We empty the
283 * queue in the local softnet handler.
284 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700285
Eric Dumazet9958da02010-04-17 04:17:02 +0000286DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700287EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
David S. Millercf508b12008-07-22 14:16:42 -0700289#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700290/*
David S. Millerc773e842008-07-08 23:13:53 -0700291 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700292 * according to dev->type
293 */
294static const unsigned short netdev_lock_type[] =
295 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
296 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
297 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
298 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
299 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
300 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
301 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
302 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
303 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
304 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
305 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
306 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400307 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
308 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
309 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700310
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700311static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700312 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
313 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
314 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
315 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
316 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
317 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
318 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
319 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
320 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
321 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
322 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
323 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
Paul Gortmaker211ed862012-05-10 17:14:35 -0400324 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
325 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
326 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700327
328static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700329static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700330
331static inline unsigned short netdev_lock_pos(unsigned short dev_type)
332{
333 int i;
334
335 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
336 if (netdev_lock_type[i] == dev_type)
337 return i;
338 /* the last key is used by default */
339 return ARRAY_SIZE(netdev_lock_type) - 1;
340}
341
David S. Millercf508b12008-07-22 14:16:42 -0700342static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
343 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700344{
345 int i;
346
347 i = netdev_lock_pos(dev_type);
348 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
349 netdev_lock_name[i]);
350}
David S. Millercf508b12008-07-22 14:16:42 -0700351
352static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
353{
354 int i;
355
356 i = netdev_lock_pos(dev->type);
357 lockdep_set_class_and_name(&dev->addr_list_lock,
358 &netdev_addr_lock_key[i],
359 netdev_lock_name[i]);
360}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700361#else
David S. Millercf508b12008-07-22 14:16:42 -0700362static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
363 unsigned short dev_type)
364{
365}
366static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700367{
368}
369#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
371/*******************************************************************************
372
373 Protocol management and registration routines
374
375*******************************************************************************/
376
377/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 * Add a protocol ID to the list. Now that the input handler is
379 * smarter we can dispense with all the messy stuff that used to be
380 * here.
381 *
382 * BEWARE!!! Protocol handlers, mangling input packets,
383 * MUST BE last in hash buckets and checking protocol handlers
384 * MUST start from promiscuous ptype_all chain in net_bh.
385 * It is true now, do not change it.
386 * Explanation follows: if protocol handler, mangling packet, will
387 * be the first on list, it is not able to sense, that packet
388 * is cloned and should be copied-on-write, so that it will
389 * change it and subsequent readers will get broken packet.
390 * --ANK (980803)
391 */
392
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000393static inline struct list_head *ptype_head(const struct packet_type *pt)
394{
395 if (pt->type == htons(ETH_P_ALL))
396 return &ptype_all;
397 else
398 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
399}
400
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401/**
402 * dev_add_pack - add packet handler
403 * @pt: packet type declaration
404 *
405 * Add a protocol handler to the networking stack. The passed &packet_type
406 * is linked into kernel lists and may not be freed until it has been
407 * removed from the kernel lists.
408 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900409 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 * guarantee all CPU's that are in middle of receiving packets
411 * will see the new packet type (until the next received packet).
412 */
413
414void dev_add_pack(struct packet_type *pt)
415{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000416 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000418 spin_lock(&ptype_lock);
419 list_add_rcu(&pt->list, head);
420 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700422EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424/**
425 * __dev_remove_pack - remove packet handler
426 * @pt: packet type declaration
427 *
428 * Remove a protocol handler that was previously added to the kernel
429 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
430 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900431 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 *
433 * The packet type might still be in use by receivers
434 * and must not be freed until after all the CPU's have gone
435 * through a quiescent state.
436 */
437void __dev_remove_pack(struct packet_type *pt)
438{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000439 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 struct packet_type *pt1;
441
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000442 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
444 list_for_each_entry(pt1, head, list) {
445 if (pt == pt1) {
446 list_del_rcu(&pt->list);
447 goto out;
448 }
449 }
450
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000451 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000453 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700455EXPORT_SYMBOL(__dev_remove_pack);
456
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457/**
458 * dev_remove_pack - remove packet handler
459 * @pt: packet type declaration
460 *
461 * Remove a protocol handler that was previously added to the kernel
462 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
463 * from the kernel lists and can be freed or reused once this function
464 * returns.
465 *
466 * This call sleeps to guarantee that no CPU is looking at the packet
467 * type after return.
468 */
469void dev_remove_pack(struct packet_type *pt)
470{
471 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900472
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 synchronize_net();
474}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700475EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
Vlad Yasevich62532da2012-11-15 08:49:10 +0000477
478/**
479 * dev_add_offload - register offload handlers
480 * @po: protocol offload declaration
481 *
482 * Add protocol offload handlers to the networking stack. The passed
483 * &proto_offload is linked into kernel lists and may not be freed until
484 * it has been removed from the kernel lists.
485 *
486 * This call does not sleep therefore it can not
487 * guarantee all CPU's that are in middle of receiving packets
488 * will see the new offload handlers (until the next received packet).
489 */
490void dev_add_offload(struct packet_offload *po)
491{
492 struct list_head *head = &offload_base;
493
494 spin_lock(&offload_lock);
495 list_add_rcu(&po->list, head);
496 spin_unlock(&offload_lock);
497}
498EXPORT_SYMBOL(dev_add_offload);
499
500/**
501 * __dev_remove_offload - remove offload handler
502 * @po: packet offload declaration
503 *
504 * Remove a protocol offload handler that was previously added to the
505 * kernel offload handlers by dev_add_offload(). The passed &offload_type
506 * is removed from the kernel lists and can be freed or reused once this
507 * function returns.
508 *
509 * The packet type might still be in use by receivers
510 * and must not be freed until after all the CPU's have gone
511 * through a quiescent state.
512 */
513void __dev_remove_offload(struct packet_offload *po)
514{
515 struct list_head *head = &offload_base;
516 struct packet_offload *po1;
517
Eric Dumazetc53aa502012-11-16 08:08:23 +0000518 spin_lock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000519
520 list_for_each_entry(po1, head, list) {
521 if (po == po1) {
522 list_del_rcu(&po->list);
523 goto out;
524 }
525 }
526
527 pr_warn("dev_remove_offload: %p not found\n", po);
528out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000529 spin_unlock(&offload_lock);
Vlad Yasevich62532da2012-11-15 08:49:10 +0000530}
531EXPORT_SYMBOL(__dev_remove_offload);
532
533/**
534 * dev_remove_offload - remove packet offload handler
535 * @po: packet offload declaration
536 *
537 * Remove a packet offload handler that was previously added to the kernel
538 * offload handlers by dev_add_offload(). The passed &offload_type is
539 * removed from the kernel lists and can be freed or reused once this
540 * function returns.
541 *
542 * This call sleeps to guarantee that no CPU is looking at the packet
543 * type after return.
544 */
545void dev_remove_offload(struct packet_offload *po)
546{
547 __dev_remove_offload(po);
548
549 synchronize_net();
550}
551EXPORT_SYMBOL(dev_remove_offload);
552
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553/******************************************************************************
554
555 Device Boot-time Settings Routines
556
557*******************************************************************************/
558
559/* Boot time configuration table */
560static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
561
562/**
563 * netdev_boot_setup_add - add new setup entry
564 * @name: name of the device
565 * @map: configured settings for the device
566 *
567 * Adds new setup entry to the dev_boot_setup list. The function
568 * returns 0 on error and 1 on success. This is a generic routine to
569 * all netdevices.
570 */
571static int netdev_boot_setup_add(char *name, struct ifmap *map)
572{
573 struct netdev_boot_setup *s;
574 int i;
575
576 s = dev_boot_setup;
577 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
578 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
579 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700580 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 memcpy(&s[i].map, map, sizeof(s[i].map));
582 break;
583 }
584 }
585
586 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
587}
588
589/**
590 * netdev_boot_setup_check - check boot time settings
591 * @dev: the netdevice
592 *
593 * Check boot time settings for the device.
594 * The found settings are set for the device to be used
595 * later in the device probing.
596 * Returns 0 if no settings found, 1 if they are.
597 */
598int netdev_boot_setup_check(struct net_device *dev)
599{
600 struct netdev_boot_setup *s = dev_boot_setup;
601 int i;
602
603 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
604 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700605 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 dev->irq = s[i].map.irq;
607 dev->base_addr = s[i].map.base_addr;
608 dev->mem_start = s[i].map.mem_start;
609 dev->mem_end = s[i].map.mem_end;
610 return 1;
611 }
612 }
613 return 0;
614}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700615EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
617
618/**
619 * netdev_boot_base - get address from boot time settings
620 * @prefix: prefix for network device
621 * @unit: id for network device
622 *
623 * Check boot time settings for the base address of device.
624 * The found settings are set for the device to be used
625 * later in the device probing.
626 * Returns 0 if no settings found.
627 */
628unsigned long netdev_boot_base(const char *prefix, int unit)
629{
630 const struct netdev_boot_setup *s = dev_boot_setup;
631 char name[IFNAMSIZ];
632 int i;
633
634 sprintf(name, "%s%d", prefix, unit);
635
636 /*
637 * If device already registered then return base of 1
638 * to indicate not to probe for this interface
639 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700640 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641 return 1;
642
643 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
644 if (!strcmp(name, s[i].name))
645 return s[i].map.base_addr;
646 return 0;
647}
648
649/*
650 * Saves at boot time configured settings for any netdevice.
651 */
652int __init netdev_boot_setup(char *str)
653{
654 int ints[5];
655 struct ifmap map;
656
657 str = get_options(str, ARRAY_SIZE(ints), ints);
658 if (!str || !*str)
659 return 0;
660
661 /* Save settings */
662 memset(&map, 0, sizeof(map));
663 if (ints[0] > 0)
664 map.irq = ints[1];
665 if (ints[0] > 1)
666 map.base_addr = ints[2];
667 if (ints[0] > 2)
668 map.mem_start = ints[3];
669 if (ints[0] > 3)
670 map.mem_end = ints[4];
671
672 /* Add new entry to the list */
673 return netdev_boot_setup_add(str, &map);
674}
675
676__setup("netdev=", netdev_boot_setup);
677
678/*******************************************************************************
679
680 Device Interface Subroutines
681
682*******************************************************************************/
683
684/**
685 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700686 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 * @name: name to find
688 *
689 * Find an interface by name. Must be called under RTNL semaphore
690 * or @dev_base_lock. If the name is found a pointer to the device
691 * is returned. If the name is not found then %NULL is returned. The
692 * reference counters are not incremented so the caller must be
693 * careful with locks.
694 */
695
Eric W. Biederman881d9662007-09-17 11:56:21 -0700696struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
698 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700699 struct net_device *dev;
700 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700702 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 if (!strncmp(dev->name, name, IFNAMSIZ))
704 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700705
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 return NULL;
707}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700708EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709
710/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000711 * dev_get_by_name_rcu - find a device by its name
712 * @net: the applicable net namespace
713 * @name: name to find
714 *
715 * Find an interface by name.
716 * If the name is found a pointer to the device is returned.
717 * If the name is not found then %NULL is returned.
718 * The reference counters are not incremented so the caller must be
719 * careful with locks. The caller must hold RCU lock.
720 */
721
722struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
723{
724 struct hlist_node *p;
725 struct net_device *dev;
726 struct hlist_head *head = dev_name_hash(net, name);
727
728 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
729 if (!strncmp(dev->name, name, IFNAMSIZ))
730 return dev;
731
732 return NULL;
733}
734EXPORT_SYMBOL(dev_get_by_name_rcu);
735
736/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700738 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 * @name: name to find
740 *
741 * Find an interface by name. This can be called from any
742 * context and does its own locking. The returned handle has
743 * the usage count incremented and the caller must use dev_put() to
744 * release it when it is no longer needed. %NULL is returned if no
745 * matching device is found.
746 */
747
Eric W. Biederman881d9662007-09-17 11:56:21 -0700748struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749{
750 struct net_device *dev;
751
Eric Dumazet72c95282009-10-30 07:11:27 +0000752 rcu_read_lock();
753 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 if (dev)
755 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000756 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 return dev;
758}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700759EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
761/**
762 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700763 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 * @ifindex: index of device
765 *
766 * Search for an interface by index. Returns %NULL if the device
767 * is not found or a pointer to the device. The device has not
768 * had its reference counter increased so the caller must be careful
769 * about locking. The caller must hold either the RTNL semaphore
770 * or @dev_base_lock.
771 */
772
Eric W. Biederman881d9662007-09-17 11:56:21 -0700773struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
775 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700776 struct net_device *dev;
777 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700779 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 if (dev->ifindex == ifindex)
781 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700782
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 return NULL;
784}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700785EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000787/**
788 * dev_get_by_index_rcu - find a device by its ifindex
789 * @net: the applicable net namespace
790 * @ifindex: index of device
791 *
792 * Search for an interface by index. Returns %NULL if the device
793 * is not found or a pointer to the device. The device has not
794 * had its reference counter increased so the caller must be careful
795 * about locking. The caller must hold RCU lock.
796 */
797
798struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
799{
800 struct hlist_node *p;
801 struct net_device *dev;
802 struct hlist_head *head = dev_index_hash(net, ifindex);
803
804 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
805 if (dev->ifindex == ifindex)
806 return dev;
807
808 return NULL;
809}
810EXPORT_SYMBOL(dev_get_by_index_rcu);
811
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
813/**
814 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700815 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 * @ifindex: index of device
817 *
818 * Search for an interface by index. Returns NULL if the device
819 * is not found or a pointer to the device. The device returned has
820 * had a reference added and the pointer is safe until the user calls
821 * dev_put to indicate they have finished with it.
822 */
823
Eric W. Biederman881d9662007-09-17 11:56:21 -0700824struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825{
826 struct net_device *dev;
827
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000828 rcu_read_lock();
829 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 if (dev)
831 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000832 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 return dev;
834}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700835EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836
837/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000838 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700839 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 * @type: media type of device
841 * @ha: hardware address
842 *
843 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800844 * is not found or a pointer to the device.
845 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000846 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 * and the caller must therefore be careful about locking
848 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 */
850
Eric Dumazet941666c2010-12-05 01:23:53 +0000851struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
852 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853{
854 struct net_device *dev;
855
Eric Dumazet941666c2010-12-05 01:23:53 +0000856 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 if (dev->type == type &&
858 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700859 return dev;
860
861 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862}
Eric Dumazet941666c2010-12-05 01:23:53 +0000863EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300864
Eric W. Biederman881d9662007-09-17 11:56:21 -0700865struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700866{
867 struct net_device *dev;
868
869 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700870 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700871 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700872 return dev;
873
874 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700875}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700876EXPORT_SYMBOL(__dev_getfirstbyhwtype);
877
Eric W. Biederman881d9662007-09-17 11:56:21 -0700878struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000880 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000882 rcu_read_lock();
883 for_each_netdev_rcu(net, dev)
884 if (dev->type == type) {
885 dev_hold(dev);
886 ret = dev;
887 break;
888 }
889 rcu_read_unlock();
890 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892EXPORT_SYMBOL(dev_getfirstbyhwtype);
893
894/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000895 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700896 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 * @if_flags: IFF_* values
898 * @mask: bitmask of bits in if_flags to check
899 *
900 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000901 * is not found or a pointer to the device. Must be called inside
902 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 */
904
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000905struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700906 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700908 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909
Pavel Emelianov7562f872007-05-03 15:13:45 -0700910 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800911 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700913 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 break;
915 }
916 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700917 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000919EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
921/**
922 * dev_valid_name - check if name is okay for network device
923 * @name: name string
924 *
925 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700926 * to allow sysfs to work. We also disallow any kind of
927 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 */
David S. Miller95f050b2012-03-06 16:12:15 -0500929bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700931 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -0500932 return false;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700933 if (strlen(name) >= IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -0500934 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700935 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -0500936 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700937
938 while (*name) {
939 if (*name == '/' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -0500940 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700941 name++;
942 }
David S. Miller95f050b2012-03-06 16:12:15 -0500943 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700945EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946
947/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200948 * __dev_alloc_name - allocate a name for a device
949 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200951 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 *
953 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700954 * id. It scans list of devices to build up a free map, then chooses
955 * the first empty slot. The caller must hold the dev_base or rtnl lock
956 * while allocating the name and adding the device in order to avoid
957 * duplicates.
958 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
959 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 */
961
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200962static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963{
964 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 const char *p;
966 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700967 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 struct net_device *d;
969
970 p = strnchr(name, IFNAMSIZ-1, '%');
971 if (p) {
972 /*
973 * Verify the string as this thing may have come from
974 * the user. There must be either one "%d" and no other "%"
975 * characters.
976 */
977 if (p[1] != 'd' || strchr(p + 2, '%'))
978 return -EINVAL;
979
980 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700981 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 if (!inuse)
983 return -ENOMEM;
984
Eric W. Biederman881d9662007-09-17 11:56:21 -0700985 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 if (!sscanf(d->name, name, &i))
987 continue;
988 if (i < 0 || i >= max_netdevices)
989 continue;
990
991 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200992 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 if (!strncmp(buf, d->name, IFNAMSIZ))
994 set_bit(i, inuse);
995 }
996
997 i = find_first_zero_bit(inuse, max_netdevices);
998 free_page((unsigned long) inuse);
999 }
1000
Octavian Purdilad9031022009-11-18 02:36:59 +00001001 if (buf != name)
1002 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001003 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
1006 /* It is possible to run out of possible slots
1007 * when the name is long and there isn't enough space left
1008 * for the digits, or if all bits are used.
1009 */
1010 return -ENFILE;
1011}
1012
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001013/**
1014 * dev_alloc_name - allocate a name for a device
1015 * @dev: device
1016 * @name: name format string
1017 *
1018 * Passed a format string - eg "lt%d" it will try and find a suitable
1019 * id. It scans list of devices to build up a free map, then chooses
1020 * the first empty slot. The caller must hold the dev_base or rtnl lock
1021 * while allocating the name and adding the device in order to avoid
1022 * duplicates.
1023 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1024 * Returns the number of the unit assigned or a negative errno code.
1025 */
1026
1027int dev_alloc_name(struct net_device *dev, const char *name)
1028{
1029 char buf[IFNAMSIZ];
1030 struct net *net;
1031 int ret;
1032
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001033 BUG_ON(!dev_net(dev));
1034 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001035 ret = __dev_alloc_name(net, name, buf);
1036 if (ret >= 0)
1037 strlcpy(dev->name, buf, IFNAMSIZ);
1038 return ret;
1039}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001040EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +02001041
Gao feng828de4f2012-09-13 20:58:27 +00001042static int dev_alloc_name_ns(struct net *net,
1043 struct net_device *dev,
1044 const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +00001045{
Gao feng828de4f2012-09-13 20:58:27 +00001046 char buf[IFNAMSIZ];
1047 int ret;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001048
Gao feng828de4f2012-09-13 20:58:27 +00001049 ret = __dev_alloc_name(net, name, buf);
1050 if (ret >= 0)
1051 strlcpy(dev->name, buf, IFNAMSIZ);
1052 return ret;
1053}
1054
1055static int dev_get_valid_name(struct net *net,
1056 struct net_device *dev,
1057 const char *name)
1058{
1059 BUG_ON(!net);
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001060
Octavian Purdilad9031022009-11-18 02:36:59 +00001061 if (!dev_valid_name(name))
1062 return -EINVAL;
1063
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001064 if (strchr(name, '%'))
Gao feng828de4f2012-09-13 20:58:27 +00001065 return dev_alloc_name_ns(net, dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +00001066 else if (__dev_get_by_name(net, name))
1067 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +00001068 else if (dev->name != name)
1069 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +00001070
1071 return 0;
1072}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073
1074/**
1075 * dev_change_name - change name of a device
1076 * @dev: device
1077 * @newname: name (or format string) must be at least IFNAMSIZ
1078 *
1079 * Change name of a device, can pass format strings "eth%d".
1080 * for wildcarding.
1081 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001082int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083{
Herbert Xufcc5a032007-07-30 17:03:38 -07001084 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001086 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001087 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088
1089 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001090 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001092 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 if (dev->flags & IFF_UP)
1094 return -EBUSY;
1095
Brian Haleyc91f6df2012-11-26 05:21:08 +00001096 write_seqlock(&devnet_rename_seq);
1097
1098 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1099 write_sequnlock(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001100 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001101 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001102
Herbert Xufcc5a032007-07-30 17:03:38 -07001103 memcpy(oldname, dev->name, IFNAMSIZ);
1104
Gao feng828de4f2012-09-13 20:58:27 +00001105 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001106 if (err < 0) {
1107 write_sequnlock(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001108 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001109 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
Herbert Xufcc5a032007-07-30 17:03:38 -07001111rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001112 ret = device_rename(&dev->dev, dev->name);
1113 if (ret) {
1114 memcpy(dev->name, oldname, IFNAMSIZ);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001115 write_sequnlock(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001116 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001117 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001118
Brian Haleyc91f6df2012-11-26 05:21:08 +00001119 write_sequnlock(&devnet_rename_seq);
1120
Herbert Xu7f988ea2007-07-30 16:35:46 -07001121 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001122 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001123 write_unlock_bh(&dev_base_lock);
1124
1125 synchronize_rcu();
1126
1127 write_lock_bh(&dev_base_lock);
1128 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001129 write_unlock_bh(&dev_base_lock);
1130
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001131 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001132 ret = notifier_to_errno(ret);
1133
1134 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001135 /* err >= 0 after dev_alloc_name() or stores the first errno */
1136 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001137 err = ret;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001138 write_seqlock(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001139 memcpy(dev->name, oldname, IFNAMSIZ);
1140 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001141 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001142 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001143 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001144 }
1145 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146
1147 return err;
1148}
1149
1150/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001151 * dev_set_alias - change ifalias of a device
1152 * @dev: device
1153 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001154 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001155 *
1156 * Set ifalias for a device,
1157 */
1158int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1159{
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001160 char *new_ifalias;
1161
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001162 ASSERT_RTNL();
1163
1164 if (len >= IFALIASZ)
1165 return -EINVAL;
1166
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001167 if (!len) {
Sachin Kamat388dfc22012-11-20 00:57:04 +00001168 kfree(dev->ifalias);
1169 dev->ifalias = NULL;
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001170 return 0;
1171 }
1172
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001173 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1174 if (!new_ifalias)
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001175 return -ENOMEM;
Alexey Khoroshilov7364e442012-08-08 00:33:25 +00001176 dev->ifalias = new_ifalias;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001177
1178 strlcpy(dev->ifalias, alias, len+1);
1179 return len;
1180}
1181
1182
1183/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001184 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001185 * @dev: device to cause notification
1186 *
1187 * Called to indicate a device has changed features.
1188 */
1189void netdev_features_change(struct net_device *dev)
1190{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001191 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001192}
1193EXPORT_SYMBOL(netdev_features_change);
1194
1195/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 * netdev_state_change - device changes state
1197 * @dev: device to cause notification
1198 *
1199 * Called to indicate a device has changed state. This function calls
1200 * the notifier chains for netdev_chain and sends a NEWLINK message
1201 * to the routing socket.
1202 */
1203void netdev_state_change(struct net_device *dev)
1204{
1205 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001206 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1208 }
1209}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001210EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211
Amerigo Wangee89bab2012-08-09 22:14:56 +00001212/**
1213 * netdev_notify_peers - notify network peers about existence of @dev
1214 * @dev: network device
1215 *
1216 * Generate traffic such that interested network peers are aware of
1217 * @dev, such as by generating a gratuitous ARP. This may be used when
1218 * a device wants to inform the rest of the network about some sort of
1219 * reconfiguration such as a failover event or virtual machine
1220 * migration.
1221 */
1222void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001223{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001224 rtnl_lock();
1225 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1226 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001227}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001228EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001229
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230/**
1231 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001232 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 * @name: name of interface
1234 *
1235 * If a network interface is not present and the process has suitable
1236 * privileges this function loads the module. If module loading is not
1237 * available in this kernel then it becomes a nop.
1238 */
1239
Eric W. Biederman881d9662007-09-17 11:56:21 -07001240void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001242 struct net_device *dev;
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001243 int no_module;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244
Eric Dumazet72c95282009-10-30 07:11:27 +00001245 rcu_read_lock();
1246 dev = dev_get_by_name_rcu(net, name);
1247 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001249 no_module = !dev;
1250 if (no_module && capable(CAP_NET_ADMIN))
1251 no_module = request_module("netdev-%s", name);
1252 if (no_module && capable(CAP_SYS_MODULE)) {
1253 if (!request_module("%s", name))
Vinson Lee7cecb522012-06-27 14:32:07 +00001254 pr_warn("Loading kernel module for a network device with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s instead.\n",
1255 name);
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001256 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001258EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259
Patrick McHardybd380812010-02-26 06:34:53 +00001260static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001262 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001263 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001265 ASSERT_RTNL();
1266
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267 if (!netif_device_present(dev))
1268 return -ENODEV;
1269
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001270 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1271 ret = notifier_to_errno(ret);
1272 if (ret)
1273 return ret;
1274
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001276
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001277 if (ops->ndo_validate_addr)
1278 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001279
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001280 if (!ret && ops->ndo_open)
1281 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
Jeff Garzikbada3392007-10-23 20:19:37 -07001283 if (ret)
1284 clear_bit(__LINK_STATE_START, &dev->state);
1285 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001287 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001288 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001290 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001292
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293 return ret;
1294}
Patrick McHardybd380812010-02-26 06:34:53 +00001295
1296/**
1297 * dev_open - prepare an interface for use.
1298 * @dev: device to open
1299 *
1300 * Takes a device from down to up state. The device's private open
1301 * function is invoked and then the multicast lists are loaded. Finally
1302 * the device is moved into the up state and a %NETDEV_UP message is
1303 * sent to the netdev notifier chain.
1304 *
1305 * Calling this function on an active interface is a nop. On a failure
1306 * a negative errno code is returned.
1307 */
1308int dev_open(struct net_device *dev)
1309{
1310 int ret;
1311
Patrick McHardybd380812010-02-26 06:34:53 +00001312 if (dev->flags & IFF_UP)
1313 return 0;
1314
Patrick McHardybd380812010-02-26 06:34:53 +00001315 ret = __dev_open(dev);
1316 if (ret < 0)
1317 return ret;
1318
Patrick McHardybd380812010-02-26 06:34:53 +00001319 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1320 call_netdevice_notifiers(NETDEV_UP, dev);
1321
1322 return ret;
1323}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001324EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325
Octavian Purdila44345722010-12-13 12:44:07 +00001326static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327{
Octavian Purdila44345722010-12-13 12:44:07 +00001328 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001329
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001330 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001331 might_sleep();
1332
Octavian Purdila44345722010-12-13 12:44:07 +00001333 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001334 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335
Octavian Purdila44345722010-12-13 12:44:07 +00001336 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337
Octavian Purdila44345722010-12-13 12:44:07 +00001338 /* Synchronize to scheduled poll. We cannot touch poll list, it
1339 * can be even on different cpu. So just clear netif_running().
1340 *
1341 * dev->stop() will invoke napi_disable() on all of it's
1342 * napi_struct instances on this device.
1343 */
1344 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1345 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346
Octavian Purdila44345722010-12-13 12:44:07 +00001347 dev_deactivate_many(head);
1348
1349 list_for_each_entry(dev, head, unreg_list) {
1350 const struct net_device_ops *ops = dev->netdev_ops;
1351
1352 /*
1353 * Call the device specific close. This cannot fail.
1354 * Only if device is UP
1355 *
1356 * We allow it to be called even after a DETACH hot-plug
1357 * event.
1358 */
1359 if (ops->ndo_stop)
1360 ops->ndo_stop(dev);
1361
Octavian Purdila44345722010-12-13 12:44:07 +00001362 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001363 net_dmaengine_put();
1364 }
1365
1366 return 0;
1367}
1368
1369static int __dev_close(struct net_device *dev)
1370{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001371 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001372 LIST_HEAD(single);
1373
1374 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001375 retval = __dev_close_many(&single);
1376 list_del(&single);
1377 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001378}
1379
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001380static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001381{
1382 struct net_device *dev, *tmp;
1383 LIST_HEAD(tmp_list);
1384
1385 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1386 if (!(dev->flags & IFF_UP))
1387 list_move(&dev->unreg_list, &tmp_list);
1388
1389 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001390
Octavian Purdila44345722010-12-13 12:44:07 +00001391 list_for_each_entry(dev, head, unreg_list) {
1392 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1393 call_netdevice_notifiers(NETDEV_DOWN, dev);
1394 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395
Octavian Purdila44345722010-12-13 12:44:07 +00001396 /* rollback_registered_many needs the complete original list */
1397 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398 return 0;
1399}
Patrick McHardybd380812010-02-26 06:34:53 +00001400
1401/**
1402 * dev_close - shutdown an interface.
1403 * @dev: device to shutdown
1404 *
1405 * This function moves an active device into down state. A
1406 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1407 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1408 * chain.
1409 */
1410int dev_close(struct net_device *dev)
1411{
Eric Dumazete14a5992011-05-10 12:26:06 -07001412 if (dev->flags & IFF_UP) {
1413 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001414
Eric Dumazete14a5992011-05-10 12:26:06 -07001415 list_add(&dev->unreg_list, &single);
1416 dev_close_many(&single);
1417 list_del(&single);
1418 }
Patrick McHardybd380812010-02-26 06:34:53 +00001419 return 0;
1420}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001421EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422
1423
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001424/**
1425 * dev_disable_lro - disable Large Receive Offload on a device
1426 * @dev: device
1427 *
1428 * Disable Large Receive Offload (LRO) on a net device. Must be
1429 * called under RTNL. This is needed if received packets may be
1430 * forwarded to another interface.
1431 */
1432void dev_disable_lro(struct net_device *dev)
1433{
Neil Hormanf11970e2011-05-24 08:31:09 +00001434 /*
1435 * If we're trying to disable lro on a vlan device
1436 * use the underlying physical device instead
1437 */
1438 if (is_vlan_dev(dev))
1439 dev = vlan_dev_real_dev(dev);
1440
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001441 dev->wanted_features &= ~NETIF_F_LRO;
1442 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001443
Michał Mirosław22d59692011-04-21 12:42:15 +00001444 if (unlikely(dev->features & NETIF_F_LRO))
1445 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001446}
1447EXPORT_SYMBOL(dev_disable_lro);
1448
1449
Eric W. Biederman881d9662007-09-17 11:56:21 -07001450static int dev_boot_phase = 1;
1451
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452/**
1453 * register_netdevice_notifier - register a network notifier block
1454 * @nb: notifier
1455 *
1456 * Register a notifier to be called when network device events occur.
1457 * The notifier passed is linked into the kernel structures and must
1458 * not be reused until it has been unregistered. A negative errno code
1459 * is returned on a failure.
1460 *
1461 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001462 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463 * view of the network device list.
1464 */
1465
1466int register_netdevice_notifier(struct notifier_block *nb)
1467{
1468 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001469 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001470 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001471 int err;
1472
1473 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001474 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001475 if (err)
1476 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001477 if (dev_boot_phase)
1478 goto unlock;
1479 for_each_net(net) {
1480 for_each_netdev(net, dev) {
1481 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1482 err = notifier_to_errno(err);
1483 if (err)
1484 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485
Eric W. Biederman881d9662007-09-17 11:56:21 -07001486 if (!(dev->flags & IFF_UP))
1487 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001488
Eric W. Biederman881d9662007-09-17 11:56:21 -07001489 nb->notifier_call(nb, NETDEV_UP, dev);
1490 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001492
1493unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494 rtnl_unlock();
1495 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001496
1497rollback:
1498 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001499 for_each_net(net) {
1500 for_each_netdev(net, dev) {
1501 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001502 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001503
Eric W. Biederman881d9662007-09-17 11:56:21 -07001504 if (dev->flags & IFF_UP) {
1505 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1506 nb->notifier_call(nb, NETDEV_DOWN, dev);
1507 }
1508 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001509 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001510 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001511
RongQing.Li8f891482011-11-30 23:43:07 -05001512outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001513 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001514 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001516EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
1518/**
1519 * unregister_netdevice_notifier - unregister a network notifier block
1520 * @nb: notifier
1521 *
1522 * Unregister a notifier previously registered by
1523 * register_netdevice_notifier(). The notifier is unlinked into the
1524 * kernel structures and may then be reused. A negative errno code
1525 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001526 *
1527 * After unregistering unregister and down device events are synthesized
1528 * for all devices on the device list to the removed notifier to remove
1529 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 */
1531
1532int unregister_netdevice_notifier(struct notifier_block *nb)
1533{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001534 struct net_device *dev;
1535 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001536 int err;
1537
1538 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001539 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001540 if (err)
1541 goto unlock;
1542
1543 for_each_net(net) {
1544 for_each_netdev(net, dev) {
1545 if (dev->flags & IFF_UP) {
1546 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1547 nb->notifier_call(nb, NETDEV_DOWN, dev);
1548 }
1549 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001550 }
1551 }
1552unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001553 rtnl_unlock();
1554 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001556EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557
1558/**
1559 * call_netdevice_notifiers - call all network notifier blocks
1560 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001561 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562 *
1563 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001564 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001565 */
1566
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001567int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568{
Jiri Pirkoab930472010-04-20 01:45:37 -07001569 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001570 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001572EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573
Ingo Molnarc5905af2012-02-24 08:31:31 +01001574static struct static_key netstamp_needed __read_mostly;
Eric Dumazetb90e5792011-11-28 11:16:50 +00001575#ifdef HAVE_JUMP_LABEL
Ingo Molnarc5905af2012-02-24 08:31:31 +01001576/* We are not allowed to call static_key_slow_dec() from irq context
Eric Dumazetb90e5792011-11-28 11:16:50 +00001577 * If net_disable_timestamp() is called from irq context, defer the
Ingo Molnarc5905af2012-02-24 08:31:31 +01001578 * static_key_slow_dec() calls.
Eric Dumazetb90e5792011-11-28 11:16:50 +00001579 */
1580static atomic_t netstamp_needed_deferred;
1581#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001582
1583void net_enable_timestamp(void)
1584{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001585#ifdef HAVE_JUMP_LABEL
1586 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1587
1588 if (deferred) {
1589 while (--deferred)
Ingo Molnarc5905af2012-02-24 08:31:31 +01001590 static_key_slow_dec(&netstamp_needed);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001591 return;
1592 }
1593#endif
1594 WARN_ON(in_interrupt());
Ingo Molnarc5905af2012-02-24 08:31:31 +01001595 static_key_slow_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001597EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598
1599void net_disable_timestamp(void)
1600{
Eric Dumazetb90e5792011-11-28 11:16:50 +00001601#ifdef HAVE_JUMP_LABEL
1602 if (in_interrupt()) {
1603 atomic_inc(&netstamp_needed_deferred);
1604 return;
1605 }
1606#endif
Ingo Molnarc5905af2012-02-24 08:31:31 +01001607 static_key_slow_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001609EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610
Eric Dumazet3b098e22010-05-15 23:57:10 -07001611static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612{
Eric Dumazet588f0332011-11-15 04:12:55 +00001613 skb->tstamp.tv64 = 0;
Ingo Molnarc5905af2012-02-24 08:31:31 +01001614 if (static_key_false(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001615 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616}
1617
Eric Dumazet588f0332011-11-15 04:12:55 +00001618#define net_timestamp_check(COND, SKB) \
Ingo Molnarc5905af2012-02-24 08:31:31 +01001619 if (static_key_false(&netstamp_needed)) { \
Eric Dumazet588f0332011-11-15 04:12:55 +00001620 if ((COND) && !(SKB)->tstamp.tv64) \
1621 __net_timestamp(SKB); \
1622 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001623
Richard Cochran4dc360c2011-10-19 17:00:35 -04001624static int net_hwtstamp_validate(struct ifreq *ifr)
1625{
1626 struct hwtstamp_config cfg;
1627 enum hwtstamp_tx_types tx_type;
1628 enum hwtstamp_rx_filters rx_filter;
1629 int tx_type_valid = 0;
1630 int rx_filter_valid = 0;
1631
1632 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1633 return -EFAULT;
1634
1635 if (cfg.flags) /* reserved for future extensions */
1636 return -EINVAL;
1637
1638 tx_type = cfg.tx_type;
1639 rx_filter = cfg.rx_filter;
1640
1641 switch (tx_type) {
1642 case HWTSTAMP_TX_OFF:
1643 case HWTSTAMP_TX_ON:
1644 case HWTSTAMP_TX_ONESTEP_SYNC:
1645 tx_type_valid = 1;
1646 break;
1647 }
1648
1649 switch (rx_filter) {
1650 case HWTSTAMP_FILTER_NONE:
1651 case HWTSTAMP_FILTER_ALL:
1652 case HWTSTAMP_FILTER_SOME:
1653 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1654 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1655 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1656 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1657 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1658 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1659 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1660 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1661 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1662 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1663 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1664 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1665 rx_filter_valid = 1;
1666 break;
1667 }
1668
1669 if (!tx_type_valid || !rx_filter_valid)
1670 return -ERANGE;
1671
1672 return 0;
1673}
1674
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001675static inline bool is_skb_forwardable(struct net_device *dev,
1676 struct sk_buff *skb)
1677{
1678 unsigned int len;
1679
1680 if (!(dev->flags & IFF_UP))
1681 return false;
1682
1683 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1684 if (skb->len <= len)
1685 return true;
1686
1687 /* if TSO is enabled, we don't care about the length as the packet
1688 * could be forwarded without being segmented before
1689 */
1690 if (skb_is_gso(skb))
1691 return true;
1692
1693 return false;
1694}
1695
Arnd Bergmann44540962009-11-26 06:07:08 +00001696/**
1697 * dev_forward_skb - loopback an skb to another netif
1698 *
1699 * @dev: destination network device
1700 * @skb: buffer to forward
1701 *
1702 * return values:
1703 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001704 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001705 *
1706 * dev_forward_skb can be used for injecting an skb from the
1707 * start_xmit function of one device into the receive queue
1708 * of another device.
1709 *
1710 * The receiving device may be in another namespace, so
1711 * we have to clear all information in the skb that could
1712 * impact namespace isolation.
1713 */
1714int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1715{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001716 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1717 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1718 atomic_long_inc(&dev->rx_dropped);
1719 kfree_skb(skb);
1720 return NET_RX_DROP;
1721 }
1722 }
1723
Arnd Bergmann44540962009-11-26 06:07:08 +00001724 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001725 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001726
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001727 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001728 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001729 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001730 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001731 }
Benjamin LaHaise3b9785c2012-03-27 15:55:44 +00001732 skb->skb_iif = 0;
David S. Miller59b99972012-05-10 23:03:34 -04001733 skb->dev = dev;
1734 skb_dst_drop(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001735 skb->tstamp.tv64 = 0;
1736 skb->pkt_type = PACKET_HOST;
1737 skb->protocol = eth_type_trans(skb, dev);
David S. Miller59b99972012-05-10 23:03:34 -04001738 skb->mark = 0;
1739 secpath_reset(skb);
1740 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001741 return netif_rx(skb);
1742}
1743EXPORT_SYMBOL_GPL(dev_forward_skb);
1744
Changli Gao71d9dec2010-12-15 19:57:25 +00001745static inline int deliver_skb(struct sk_buff *skb,
1746 struct packet_type *pt_prev,
1747 struct net_device *orig_dev)
1748{
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001749 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1750 return -ENOMEM;
Changli Gao71d9dec2010-12-15 19:57:25 +00001751 atomic_inc(&skb->users);
1752 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1753}
1754
Eric Leblondc0de08d2012-08-16 22:02:58 +00001755static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1756{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001757 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001758 return false;
1759
1760 if (ptype->id_match)
1761 return ptype->id_match(ptype, skb->sk);
1762 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1763 return true;
1764
1765 return false;
1766}
1767
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768/*
1769 * Support routine. Sends outgoing frames to any network
1770 * taps currently in use.
1771 */
1772
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001773static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001774{
1775 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001776 struct sk_buff *skb2 = NULL;
1777 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001778
Linus Torvalds1da177e2005-04-16 15:20:36 -07001779 rcu_read_lock();
1780 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1781 /* Never send packets back to the socket
1782 * they originated from - MvS (miquels@drinkel.ow.org)
1783 */
1784 if ((ptype->dev == dev || !ptype->dev) &&
Eric Leblondc0de08d2012-08-16 22:02:58 +00001785 (!skb_loop_sk(ptype, skb))) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001786 if (pt_prev) {
1787 deliver_skb(skb2, pt_prev, skb->dev);
1788 pt_prev = ptype;
1789 continue;
1790 }
1791
1792 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793 if (!skb2)
1794 break;
1795
Eric Dumazet70978182010-12-20 21:22:51 +00001796 net_timestamp_set(skb2);
1797
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798 /* skb->nh should be correctly
1799 set by sender, so that the second statement is
1800 just protection against buggy protocols.
1801 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001802 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001804 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001805 skb2->network_header > skb2->tail) {
Joe Perchese87cc472012-05-13 21:56:26 +00001806 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1807 ntohs(skb2->protocol),
1808 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001809 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001810 }
1811
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001812 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001813 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001814 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 }
1816 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001817 if (pt_prev)
1818 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819 rcu_read_unlock();
1820}
1821
Ben Hutchings2c530402012-07-10 10:55:09 +00001822/**
1823 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00001824 * @dev: Network device
1825 * @txq: number of queues available
1826 *
1827 * If real_num_tx_queues is changed the tc mappings may no longer be
1828 * valid. To resolve this verify the tc mapping remains valid and if
1829 * not NULL the mapping. With no priorities mapping to this
1830 * offset/count pair it will no longer be used. In the worst case TC0
1831 * is invalid nothing can be done so disable priority mappings. If is
1832 * expected that drivers will fix this mapping if they can before
1833 * calling netif_set_real_num_tx_queues.
1834 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001835static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001836{
1837 int i;
1838 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1839
1840 /* If TC0 is invalidated disable TC mapping */
1841 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001842 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00001843 dev->num_tc = 0;
1844 return;
1845 }
1846
1847 /* Invalidated prio to tc mappings set to TC0 */
1848 for (i = 1; i < TC_BITMASK + 1; i++) {
1849 int q = netdev_get_prio_tc_map(dev, i);
1850
1851 tc = &dev->tc_to_txq[q];
1852 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001853 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1854 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00001855 netdev_set_prio_tc_map(dev, i, 0);
1856 }
1857 }
1858}
1859
John Fastabendf0796d52010-07-01 13:21:57 +00001860/*
1861 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1862 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1863 */
Tom Herberte6484932010-10-18 18:04:39 +00001864int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001865{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001866 int rc;
1867
Tom Herberte6484932010-10-18 18:04:39 +00001868 if (txq < 1 || txq > dev->num_tx_queues)
1869 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001870
Ben Hutchings5c565802011-02-15 19:39:21 +00001871 if (dev->reg_state == NETREG_REGISTERED ||
1872 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001873 ASSERT_RTNL();
1874
Tom Herbert1d24eb42010-11-21 13:17:27 +00001875 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1876 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001877 if (rc)
1878 return rc;
1879
John Fastabend4f57c082011-01-17 08:06:04 +00001880 if (dev->num_tc)
1881 netif_setup_tc(dev, txq);
1882
Tom Herberte6484932010-10-18 18:04:39 +00001883 if (txq < dev->real_num_tx_queues)
1884 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001885 }
Tom Herberte6484932010-10-18 18:04:39 +00001886
1887 dev->real_num_tx_queues = txq;
1888 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001889}
1890EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001891
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001892#ifdef CONFIG_RPS
1893/**
1894 * netif_set_real_num_rx_queues - set actual number of RX queues used
1895 * @dev: Network device
1896 * @rxq: Actual number of RX queues
1897 *
1898 * This must be called either with the rtnl_lock held or before
1899 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001900 * negative error code. If called before registration, it always
1901 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001902 */
1903int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1904{
1905 int rc;
1906
Tom Herbertbd25fa72010-10-18 18:00:16 +00001907 if (rxq < 1 || rxq > dev->num_rx_queues)
1908 return -EINVAL;
1909
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001910 if (dev->reg_state == NETREG_REGISTERED) {
1911 ASSERT_RTNL();
1912
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001913 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1914 rxq);
1915 if (rc)
1916 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001917 }
1918
1919 dev->real_num_rx_queues = rxq;
1920 return 0;
1921}
1922EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1923#endif
1924
Ben Hutchings2c530402012-07-10 10:55:09 +00001925/**
1926 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00001927 *
1928 * This routine should set an upper limit on the number of RSS queues
1929 * used by default by multiqueue devices.
1930 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00001931int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00001932{
1933 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
1934}
1935EXPORT_SYMBOL(netif_get_num_default_rss_queues);
1936
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001937static inline void __netif_reschedule(struct Qdisc *q)
1938{
1939 struct softnet_data *sd;
1940 unsigned long flags;
1941
1942 local_irq_save(flags);
1943 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001944 q->next_sched = NULL;
1945 *sd->output_queue_tailp = q;
1946 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001947 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1948 local_irq_restore(flags);
1949}
1950
David S. Miller37437bb2008-07-16 02:15:04 -07001951void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001952{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001953 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1954 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001955}
1956EXPORT_SYMBOL(__netif_schedule);
1957
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001958void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001959{
David S. Miller3578b0c2010-08-03 00:24:04 -07001960 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001961 struct softnet_data *sd;
1962 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001963
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001964 local_irq_save(flags);
1965 sd = &__get_cpu_var(softnet_data);
1966 skb->next = sd->completion_queue;
1967 sd->completion_queue = skb;
1968 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1969 local_irq_restore(flags);
1970 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001971}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001972EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001973
1974void dev_kfree_skb_any(struct sk_buff *skb)
1975{
1976 if (in_irq() || irqs_disabled())
1977 dev_kfree_skb_irq(skb);
1978 else
1979 dev_kfree_skb(skb);
1980}
1981EXPORT_SYMBOL(dev_kfree_skb_any);
1982
1983
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001984/**
1985 * netif_device_detach - mark device as removed
1986 * @dev: network device
1987 *
1988 * Mark device as removed from system and therefore no longer available.
1989 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001990void netif_device_detach(struct net_device *dev)
1991{
1992 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1993 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001994 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001995 }
1996}
1997EXPORT_SYMBOL(netif_device_detach);
1998
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001999/**
2000 * netif_device_attach - mark device as attached
2001 * @dev: network device
2002 *
2003 * Mark device as attached from system and restart if needed.
2004 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002005void netif_device_attach(struct net_device *dev)
2006{
2007 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2008 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002009 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002010 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002011 }
2012}
2013EXPORT_SYMBOL(netif_device_attach);
2014
Ben Hutchings36c92472012-01-17 07:57:56 +00002015static void skb_warn_bad_offload(const struct sk_buff *skb)
2016{
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002017 static const netdev_features_t null_features = 0;
Ben Hutchings36c92472012-01-17 07:57:56 +00002018 struct net_device *dev = skb->dev;
2019 const char *driver = "";
2020
2021 if (dev && dev->dev.parent)
2022 driver = dev_driver_string(dev->dev.parent);
2023
2024 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2025 "gso_type=%d ip_summed=%d\n",
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002026 driver, dev ? &dev->features : &null_features,
2027 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002028 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2029 skb_shinfo(skb)->gso_type, skb->ip_summed);
2030}
2031
Linus Torvalds1da177e2005-04-16 15:20:36 -07002032/*
2033 * Invalidate hardware checksum when packet is to be mangled, and
2034 * complete checksum manually on outgoing path.
2035 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002036int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037{
Al Virod3bc23e2006-11-14 21:24:49 -08002038 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002039 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002040
Patrick McHardy84fa7932006-08-29 16:44:56 -07002041 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002042 goto out_set_summed;
2043
2044 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002045 skb_warn_bad_offload(skb);
2046 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002047 }
2048
Michał Mirosław55508d62010-12-14 15:24:08 +00002049 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002050 BUG_ON(offset >= skb_headlen(skb));
2051 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2052
2053 offset += skb->csum_offset;
2054 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2055
2056 if (skb_cloned(skb) &&
2057 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2059 if (ret)
2060 goto out;
2061 }
2062
Herbert Xua0308472007-10-15 01:47:15 -07002063 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07002064out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002066out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 return ret;
2068}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002069EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002071/**
2072 * skb_gso_segment - Perform segmentation on skb.
2073 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07002074 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002075 *
2076 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07002077 *
2078 * It may return NULL if the skb requires no segmentation. This is
2079 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002080 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002081struct sk_buff *skb_gso_segment(struct sk_buff *skb,
2082 netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002083{
2084 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
Vlad Yasevich22061d82012-11-15 08:49:11 +00002085 struct packet_offload *ptype;
Al Viro252e3342006-11-14 20:48:11 -08002086 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002087 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07002088 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002089
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002090 while (type == htons(ETH_P_8021Q)) {
2091 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00002092
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002093 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00002094 return ERR_PTR(-EINVAL);
2095
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00002096 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2097 type = vh->h_vlan_encapsulated_proto;
2098 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00002099 }
2100
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07002101 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07002102 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002103 __skb_pull(skb, skb->mac_len);
2104
Herbert Xu67fd1a72009-01-19 16:26:44 -08002105 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002106 skb_warn_bad_offload(skb);
Herbert Xu67fd1a72009-01-19 16:26:44 -08002107
Herbert Xua430a432006-07-08 13:34:56 -07002108 if (skb_header_cloned(skb) &&
2109 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2110 return ERR_PTR(err);
2111 }
2112
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002113 rcu_read_lock();
Vlad Yasevich22061d82012-11-15 08:49:11 +00002114 list_for_each_entry_rcu(ptype, &offload_base, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002115 if (ptype->type == type && ptype->callbacks.gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07002116 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002117 err = ptype->callbacks.gso_send_check(skb);
Herbert Xua430a432006-07-08 13:34:56 -07002118 segs = ERR_PTR(err);
2119 if (err || skb_gso_ok(skb, features))
2120 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07002121 __skb_push(skb, (skb->data -
2122 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07002123 }
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00002124 segs = ptype->callbacks.gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002125 break;
2126 }
2127 }
2128 rcu_read_unlock();
2129
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002130 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07002131
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002132 return segs;
2133}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002134EXPORT_SYMBOL(skb_gso_segment);
2135
Herbert Xufb286bb2005-11-10 13:01:24 -08002136/* Take action when hardware reception checksum errors are detected. */
2137#ifdef CONFIG_BUG
2138void netdev_rx_csum_fault(struct net_device *dev)
2139{
2140 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002141 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08002142 dump_stack();
2143 }
2144}
2145EXPORT_SYMBOL(netdev_rx_csum_fault);
2146#endif
2147
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148/* Actually, we should eliminate this check as soon as we know, that:
2149 * 1. IOMMU is present and allows to map all the memory.
2150 * 2. No high memory really exists on this machine.
2151 */
2152
Eric Dumazet9092c652010-04-02 13:34:49 -07002153static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002154{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002155#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002156 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002157 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002158 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2159 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2160 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002161 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002162 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002163 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002164
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002165 if (PCI_DMA_BUS_IS_PHYS) {
2166 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167
Eric Dumazet9092c652010-04-02 13:34:49 -07002168 if (!pdev)
2169 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002170 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002171 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2172 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002173 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2174 return 1;
2175 }
2176 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002177#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002178 return 0;
2179}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002180
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002181struct dev_gso_cb {
2182 void (*destructor)(struct sk_buff *skb);
2183};
2184
2185#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2186
2187static void dev_gso_skb_destructor(struct sk_buff *skb)
2188{
2189 struct dev_gso_cb *cb;
2190
2191 do {
2192 struct sk_buff *nskb = skb->next;
2193
2194 skb->next = nskb->next;
2195 nskb->next = NULL;
2196 kfree_skb(nskb);
2197 } while (skb->next);
2198
2199 cb = DEV_GSO_CB(skb);
2200 if (cb->destructor)
2201 cb->destructor(skb);
2202}
2203
2204/**
2205 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2206 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002207 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002208 *
2209 * This function segments the given skb and stores the list of segments
2210 * in skb->next.
2211 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002212static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002213{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002214 struct sk_buff *segs;
2215
Herbert Xu576a30e2006-06-27 13:22:38 -07002216 segs = skb_gso_segment(skb, features);
2217
2218 /* Verifying header integrity only. */
2219 if (!segs)
2220 return 0;
2221
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002222 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002223 return PTR_ERR(segs);
2224
2225 skb->next = segs;
2226 DEV_GSO_CB(skb)->destructor = skb->destructor;
2227 skb->destructor = dev_gso_skb_destructor;
2228
2229 return 0;
2230}
2231
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002232static bool can_checksum_protocol(netdev_features_t features, __be16 protocol)
Jesse Gross03634662011-01-09 06:23:35 +00002233{
2234 return ((features & NETIF_F_GEN_CSUM) ||
2235 ((features & NETIF_F_V4_CSUM) &&
2236 protocol == htons(ETH_P_IP)) ||
2237 ((features & NETIF_F_V6_CSUM) &&
2238 protocol == htons(ETH_P_IPV6)) ||
2239 ((features & NETIF_F_FCOE_CRC) &&
2240 protocol == htons(ETH_P_FCOE)));
2241}
2242
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002243static netdev_features_t harmonize_features(struct sk_buff *skb,
2244 __be16 protocol, netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002245{
Ed Cashinc0d680e2012-09-19 15:49:00 +00002246 if (skb->ip_summed != CHECKSUM_NONE &&
2247 !can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002248 features &= ~NETIF_F_ALL_CSUM;
2249 features &= ~NETIF_F_SG;
2250 } else if (illegal_highdma(skb->dev, skb)) {
2251 features &= ~NETIF_F_SG;
2252 }
2253
2254 return features;
2255}
2256
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002257netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002258{
2259 __be16 protocol = skb->protocol;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002260 netdev_features_t features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002261
Ben Hutchings30b678d2012-07-30 15:57:00 +00002262 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2263 features &= ~NETIF_F_GSO_MASK;
2264
Jesse Gross58e998c2010-10-29 12:14:55 +00002265 if (protocol == htons(ETH_P_8021Q)) {
2266 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2267 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002268 } else if (!vlan_tx_tag_present(skb)) {
2269 return harmonize_features(skb, protocol, features);
2270 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002271
Jesse Gross6ee400a2011-01-17 20:46:00 +00002272 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002273
2274 if (protocol != htons(ETH_P_8021Q)) {
2275 return harmonize_features(skb, protocol, features);
2276 } else {
2277 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002278 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002279 return harmonize_features(skb, protocol, features);
2280 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002281}
Jesse Grossf01a5232011-01-09 06:23:31 +00002282EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002283
John Fastabend6afff0c2010-06-16 14:18:12 +00002284/*
2285 * Returns true if either:
2286 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
Rami Rosend1a53df2012-08-27 23:39:24 +00002287 * 2. skb is fragmented and the device does not support SG.
John Fastabend6afff0c2010-06-16 14:18:12 +00002288 */
2289static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002290 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002291{
Jesse Gross02932ce2011-01-09 06:23:34 +00002292 return skb_is_nonlinear(skb) &&
2293 ((skb_has_frag_list(skb) &&
2294 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002295 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002296 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002297}
2298
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002299int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2300 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002301{
Stephen Hemminger00829822008-11-20 20:14:53 -08002302 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002303 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002304 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002305
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002306 if (likely(!skb->next)) {
Michał Mirosławc8f44af2011-11-15 15:29:55 +00002307 netdev_features_t features;
Jesse Grossfc741212011-01-09 06:23:32 +00002308
Eric Dumazet93f154b2009-05-18 22:19:19 -07002309 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002310 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002311 * its hot in this cpu cache
2312 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002313 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2314 skb_dst_drop(skb);
2315
Jesse Grossfc741212011-01-09 06:23:32 +00002316 features = netif_skb_features(skb);
2317
Jesse Gross7b9c6092010-10-20 13:56:04 +00002318 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002319 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002320 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2321 if (unlikely(!skb))
2322 goto out;
2323
2324 skb->vlan_tci = 0;
2325 }
2326
Alexander Duyckfc70fb62012-12-07 14:14:15 +00002327 /* If encapsulation offload request, verify we are testing
2328 * hardware encapsulation features instead of standard
2329 * features for the netdev
2330 */
2331 if (skb->encapsulation)
2332 features &= dev->hw_enc_features;
2333
Jesse Grossfc741212011-01-09 06:23:32 +00002334 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002335 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002336 goto out_kfree_skb;
2337 if (skb->next)
2338 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002339 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002340 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002341 __skb_linearize(skb))
2342 goto out_kfree_skb;
2343
2344 /* If packet is not checksummed and device does not
2345 * support checksumming for this protocol, complete
2346 * checksumming here.
2347 */
2348 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Alexander Duyckfc70fb62012-12-07 14:14:15 +00002349 if (skb->encapsulation)
2350 skb_set_inner_transport_header(skb,
2351 skb_checksum_start_offset(skb));
2352 else
2353 skb_set_transport_header(skb,
2354 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002355 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002356 skb_checksum_help(skb))
2357 goto out_kfree_skb;
2358 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002359 }
2360
Eric Dumazetb40863c2012-09-18 20:44:49 +00002361 if (!list_empty(&ptype_all))
2362 dev_queue_xmit_nit(skb, dev);
2363
Koki Sanagiec764bf2011-05-30 21:48:34 +00002364 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002365 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002366 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002367 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002368 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002369 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002370 }
2371
Herbert Xu576a30e2006-06-27 13:22:38 -07002372gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002373 do {
2374 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002375
2376 skb->next = nskb->next;
2377 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002378
2379 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002380 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002381 * its hot in this cpu cache
2382 */
2383 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2384 skb_dst_drop(nskb);
2385
Eric Dumazetb40863c2012-09-18 20:44:49 +00002386 if (!list_empty(&ptype_all))
2387 dev_queue_xmit_nit(nskb, dev);
2388
Koki Sanagiec764bf2011-05-30 21:48:34 +00002389 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002390 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002391 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002392 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002393 if (rc & ~NETDEV_TX_MASK)
2394 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002395 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002396 skb->next = nskb;
2397 return rc;
2398 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002399 txq_trans_update(txq);
Tom Herbert734664982011-11-28 16:32:44 +00002400 if (unlikely(netif_xmit_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002401 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002402 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002403
Patrick McHardy572a9d72009-11-10 06:14:14 +00002404out_kfree_gso_skb:
2405 if (likely(skb->next == NULL))
2406 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002407out_kfree_skb:
2408 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002409out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002410 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002411}
2412
Tom Herbert0a9627f2010-03-16 08:03:29 +00002413static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002414
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002415/*
2416 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2417 * to be used as a distribution range.
2418 */
2419u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2420 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002421{
David S. Miller70192982009-01-27 16:34:47 -08002422 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002423 u16 qoffset = 0;
2424 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002425
David S. Miller513de112009-05-03 14:43:10 -07002426 if (skb_rx_queue_recorded(skb)) {
2427 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002428 while (unlikely(hash >= num_tx_queues))
2429 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002430 return hash;
2431 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002432
John Fastabend4f57c082011-01-17 08:06:04 +00002433 if (dev->num_tc) {
2434 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2435 qoffset = dev->tc_to_txq[tc].offset;
2436 qcount = dev->tc_to_txq[tc].count;
2437 }
2438
Eric Dumazetec581f62009-05-01 09:05:06 -07002439 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002440 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002441 else
Eric Dumazet62b1a8a2012-06-14 06:42:44 +00002442 hash = (__force u16) skb->protocol;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002443 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002444
John Fastabend4f57c082011-01-17 08:06:04 +00002445 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002446}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002447EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002448
Eric Dumazeted046422009-11-13 21:54:04 +00002449static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2450{
2451 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
Joe Perchese87cc472012-05-13 21:56:26 +00002452 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
2453 dev->name, queue_index,
2454 dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002455 return 0;
2456 }
2457 return queue_index;
2458}
2459
Tom Herbert1d24eb42010-11-21 13:17:27 +00002460static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2461{
Tom Herbertbf264142010-11-26 08:36:09 +00002462#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002463 struct xps_dev_maps *dev_maps;
2464 struct xps_map *map;
2465 int queue_index = -1;
2466
2467 rcu_read_lock();
2468 dev_maps = rcu_dereference(dev->xps_maps);
2469 if (dev_maps) {
2470 map = rcu_dereference(
2471 dev_maps->cpu_map[raw_smp_processor_id()]);
2472 if (map) {
2473 if (map->len == 1)
2474 queue_index = map->queues[0];
2475 else {
2476 u32 hash;
2477 if (skb->sk && skb->sk->sk_hash)
2478 hash = skb->sk->sk_hash;
2479 else
2480 hash = (__force u16) skb->protocol ^
2481 skb->rxhash;
2482 hash = jhash_1word(hash, hashrnd);
2483 queue_index = map->queues[
2484 ((u64)hash * map->len) >> 32];
2485 }
2486 if (unlikely(queue_index >= dev->real_num_tx_queues))
2487 queue_index = -1;
2488 }
2489 }
2490 rcu_read_unlock();
2491
2492 return queue_index;
2493#else
2494 return -1;
2495#endif
2496}
2497
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002498struct netdev_queue *netdev_pick_tx(struct net_device *dev,
2499 struct sk_buff *skb)
David S. Millere8a04642008-07-17 00:34:19 -07002500{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002501 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002502 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002503
Tom Herbert3853b582010-11-21 13:17:29 +00002504 if (dev->real_num_tx_queues == 1)
2505 queue_index = 0;
2506 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002507 queue_index = ops->ndo_select_queue(dev, skb);
2508 queue_index = dev_cap_txqueue(dev, queue_index);
2509 } else {
2510 struct sock *sk = skb->sk;
2511 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002512
Tom Herbert3853b582010-11-21 13:17:29 +00002513 if (queue_index < 0 || skb->ooo_okay ||
2514 queue_index >= dev->real_num_tx_queues) {
2515 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002516
Tom Herbert1d24eb42010-11-21 13:17:27 +00002517 queue_index = get_xps_queue(dev, skb);
2518 if (queue_index < 0)
2519 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002520
2521 if (queue_index != old_index && sk) {
2522 struct dst_entry *dst =
2523 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002524
2525 if (dst && skb_dst(skb) == dst)
2526 sk_tx_queue_set(sk, queue_index);
2527 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002528 }
2529 }
David S. Millereae792b2008-07-15 03:03:33 -07002530
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002531 skb_set_queue_mapping(skb, queue_index);
2532 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002533}
2534
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002535static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2536 struct net_device *dev,
2537 struct netdev_queue *txq)
2538{
2539 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002540 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002541 int rc;
2542
Eric Dumazeta2da5702011-01-20 03:48:19 +00002543 qdisc_skb_cb(skb)->pkt_len = skb->len;
2544 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002545 /*
2546 * Heuristic to force contended enqueues to serialize on a
2547 * separate lock before trying to get qdisc main lock.
2548 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2549 * and dequeue packets faster.
2550 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002551 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002552 if (unlikely(contended))
2553 spin_lock(&q->busylock);
2554
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002555 spin_lock(root_lock);
2556 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2557 kfree_skb(skb);
2558 rc = NET_XMIT_DROP;
2559 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002560 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002561 /*
2562 * This is a work-conserving queue; there are no old skbs
2563 * waiting to be sent out; and the qdisc is not running -
2564 * xmit the skb directly.
2565 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002566 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2567 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002568
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002569 qdisc_bstats_update(q, skb);
2570
Eric Dumazet79640a42010-06-02 05:09:29 -07002571 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2572 if (unlikely(contended)) {
2573 spin_unlock(&q->busylock);
2574 contended = false;
2575 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002576 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002577 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002578 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002579
2580 rc = NET_XMIT_SUCCESS;
2581 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002582 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002583 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002584 if (qdisc_run_begin(q)) {
2585 if (unlikely(contended)) {
2586 spin_unlock(&q->busylock);
2587 contended = false;
2588 }
2589 __qdisc_run(q);
2590 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002591 }
2592 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002593 if (unlikely(contended))
2594 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002595 return rc;
2596}
2597
Neil Horman5bc14212011-11-22 05:10:51 +00002598#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
2599static void skb_update_prio(struct sk_buff *skb)
2600{
Igor Maravic6977a792011-11-25 07:44:54 +00002601 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
Neil Horman5bc14212011-11-22 05:10:51 +00002602
Eric Dumazet91c68ce2012-07-08 21:45:10 +00002603 if (!skb->priority && skb->sk && map) {
2604 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2605
2606 if (prioidx < map->priomap_len)
2607 skb->priority = map->priomap[prioidx];
2608 }
Neil Horman5bc14212011-11-22 05:10:51 +00002609}
2610#else
2611#define skb_update_prio(skb)
2612#endif
2613
Eric Dumazet745e20f2010-09-29 13:23:09 -07002614static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002615#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002616
Dave Jonesd29f7492008-07-22 14:09:06 -07002617/**
Michel Machado95603e22012-06-12 10:16:35 +00002618 * dev_loopback_xmit - loop back @skb
2619 * @skb: buffer to transmit
2620 */
2621int dev_loopback_xmit(struct sk_buff *skb)
2622{
2623 skb_reset_mac_header(skb);
2624 __skb_pull(skb, skb_network_offset(skb));
2625 skb->pkt_type = PACKET_LOOPBACK;
2626 skb->ip_summed = CHECKSUM_UNNECESSARY;
2627 WARN_ON(!skb_dst(skb));
2628 skb_dst_force(skb);
2629 netif_rx_ni(skb);
2630 return 0;
2631}
2632EXPORT_SYMBOL(dev_loopback_xmit);
2633
2634/**
Dave Jonesd29f7492008-07-22 14:09:06 -07002635 * dev_queue_xmit - transmit a buffer
2636 * @skb: buffer to transmit
2637 *
2638 * Queue a buffer for transmission to a network device. The caller must
2639 * have set the device and priority and built the buffer before calling
2640 * this function. The function can be called from an interrupt.
2641 *
2642 * A negative errno code is returned on a failure. A success does not
2643 * guarantee the frame will be transmitted as it may be dropped due
2644 * to congestion or traffic shaping.
2645 *
2646 * -----------------------------------------------------------------------------------
2647 * I notice this method can also return errors from the queue disciplines,
2648 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2649 * be positive.
2650 *
2651 * Regardless of the return value, the skb is consumed, so it is currently
2652 * difficult to retry a send to this method. (You can bump the ref count
2653 * before sending to hold a reference for retry if you are careful.)
2654 *
2655 * When calling this method, interrupts MUST be enabled. This is because
2656 * the BH enable code must have IRQs enabled so that it will not deadlock.
2657 * --BLG
2658 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002659int dev_queue_xmit(struct sk_buff *skb)
2660{
2661 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002662 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663 struct Qdisc *q;
2664 int rc = -ENOMEM;
2665
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002666 /* Disable soft irqs for various locks below. Also
2667 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002668 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002669 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002670
Neil Horman5bc14212011-11-22 05:10:51 +00002671 skb_update_prio(skb);
2672
Amerigo Wang8c4c49d2012-09-17 20:16:31 +00002673 txq = netdev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002674 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002675
Linus Torvalds1da177e2005-04-16 15:20:36 -07002676#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002677 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002678#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002679 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002680 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002681 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002682 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002683 }
2684
2685 /* The device has no queue. Common case for software devices:
2686 loopback, all the sorts of tunnels...
2687
Herbert Xu932ff272006-06-09 12:20:56 -07002688 Really, it is unlikely that netif_tx_lock protection is necessary
2689 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690 counters.)
2691 However, it is possible, that they rely on protection
2692 made by us here.
2693
2694 Check this and shot the lock. It is not prone from deadlocks.
2695 Either shot noqueue qdisc, it is even simpler 8)
2696 */
2697 if (dev->flags & IFF_UP) {
2698 int cpu = smp_processor_id(); /* ok because BHs are off */
2699
David S. Millerc773e842008-07-08 23:13:53 -07002700 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701
Eric Dumazet745e20f2010-09-29 13:23:09 -07002702 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2703 goto recursion_alert;
2704
David S. Millerc773e842008-07-08 23:13:53 -07002705 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002706
Tom Herbert734664982011-11-28 16:32:44 +00002707 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002708 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002709 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002710 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002711 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002712 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002713 goto out;
2714 }
2715 }
David S. Millerc773e842008-07-08 23:13:53 -07002716 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00002717 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2718 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002719 } else {
2720 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002721 * unfortunately
2722 */
2723recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00002724 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2725 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002726 }
2727 }
2728
2729 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002730 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002731
Linus Torvalds1da177e2005-04-16 15:20:36 -07002732 kfree_skb(skb);
2733 return rc;
2734out:
Herbert Xud4828d82006-06-22 02:28:18 -07002735 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002736 return rc;
2737}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002738EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002739
2740
2741/*=======================================================================
2742 Receiver routines
2743 =======================================================================*/
2744
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002745int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00002746EXPORT_SYMBOL(netdev_max_backlog);
2747
Eric Dumazet3b098e22010-05-15 23:57:10 -07002748int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002749int netdev_budget __read_mostly = 300;
2750int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002751
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002752/* Called with irq disabled */
2753static inline void ____napi_schedule(struct softnet_data *sd,
2754 struct napi_struct *napi)
2755{
2756 list_add_tail(&napi->poll_list, &sd->poll_list);
2757 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2758}
2759
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002760/*
2761 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
Tom Herbertbdeab992011-08-14 19:45:55 +00002762 * and src/dst port numbers. Sets rxhash in skb to non-zero hash value
2763 * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
2764 * if hash is a canonical 4-tuple hash over transport ports.
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002765 */
Tom Herbertbdeab992011-08-14 19:45:55 +00002766void __skb_get_rxhash(struct sk_buff *skb)
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002767{
Eric Dumazet4504b862011-11-28 05:23:23 +00002768 struct flow_keys keys;
2769 u32 hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002770
Eric Dumazet4504b862011-11-28 05:23:23 +00002771 if (!skb_flow_dissect(skb, &keys))
2772 return;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002773
Chema Gonzalez68622342012-09-07 13:40:50 +00002774 if (keys.ports)
Eric Dumazet4504b862011-11-28 05:23:23 +00002775 skb->l4_rxhash = 1;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002776
2777 /* get a consistent hash (same value on both flow directions) */
Chema Gonzalez68622342012-09-07 13:40:50 +00002778 if (((__force u32)keys.dst < (__force u32)keys.src) ||
2779 (((__force u32)keys.dst == (__force u32)keys.src) &&
2780 ((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) {
Eric Dumazet4504b862011-11-28 05:23:23 +00002781 swap(keys.dst, keys.src);
Chema Gonzalez68622342012-09-07 13:40:50 +00002782 swap(keys.port16[0], keys.port16[1]);
2783 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002784
Eric Dumazet4504b862011-11-28 05:23:23 +00002785 hash = jhash_3words((__force u32)keys.dst,
2786 (__force u32)keys.src,
2787 (__force u32)keys.ports, hashrnd);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002788 if (!hash)
2789 hash = 1;
2790
Tom Herbertbdeab992011-08-14 19:45:55 +00002791 skb->rxhash = hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002792}
2793EXPORT_SYMBOL(__skb_get_rxhash);
2794
Eric Dumazetdf334542010-03-24 19:13:54 +00002795#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002796
2797/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002798struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002799EXPORT_SYMBOL(rps_sock_flow_table);
2800
Ingo Molnarc5905af2012-02-24 08:31:31 +01002801struct static_key rps_needed __read_mostly;
Eric Dumazetadc93002011-11-17 03:13:26 +00002802
Ben Hutchingsc4454772011-01-19 11:03:53 +00002803static struct rps_dev_flow *
2804set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2805 struct rps_dev_flow *rflow, u16 next_cpu)
2806{
Ben Hutchings09994d12011-10-03 04:42:46 +00002807 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002808#ifdef CONFIG_RFS_ACCEL
2809 struct netdev_rx_queue *rxqueue;
2810 struct rps_dev_flow_table *flow_table;
2811 struct rps_dev_flow *old_rflow;
2812 u32 flow_id;
2813 u16 rxq_index;
2814 int rc;
2815
2816 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002817 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2818 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002819 goto out;
2820 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2821 if (rxq_index == skb_get_rx_queue(skb))
2822 goto out;
2823
2824 rxqueue = dev->_rx + rxq_index;
2825 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2826 if (!flow_table)
2827 goto out;
2828 flow_id = skb->rxhash & flow_table->mask;
2829 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2830 rxq_index, flow_id);
2831 if (rc < 0)
2832 goto out;
2833 old_rflow = rflow;
2834 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002835 rflow->filter = rc;
2836 if (old_rflow->filter == rflow->filter)
2837 old_rflow->filter = RPS_NO_FILTER;
2838 out:
2839#endif
2840 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002841 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002842 }
2843
Ben Hutchings09994d12011-10-03 04:42:46 +00002844 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002845 return rflow;
2846}
2847
Tom Herbert0a9627f2010-03-16 08:03:29 +00002848/*
2849 * get_rps_cpu is called from netif_receive_skb and returns the target
2850 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002851 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002852 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002853static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2854 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002855{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002856 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002857 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002858 struct rps_dev_flow_table *flow_table;
2859 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002860 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002861 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002862
Tom Herbert0a9627f2010-03-16 08:03:29 +00002863 if (skb_rx_queue_recorded(skb)) {
2864 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002865 if (unlikely(index >= dev->real_num_rx_queues)) {
2866 WARN_ONCE(dev->real_num_rx_queues > 1,
2867 "%s received packet on queue %u, but number "
2868 "of RX queues is %u\n",
2869 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002870 goto done;
2871 }
2872 rxqueue = dev->_rx + index;
2873 } else
2874 rxqueue = dev->_rx;
2875
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002876 map = rcu_dereference(rxqueue->rps_map);
2877 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002878 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002879 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002880 tcpu = map->cpus[0];
2881 if (cpu_online(tcpu))
2882 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002883 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002884 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002885 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002886 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002887 }
2888
Changli Gao2d47b452010-08-17 19:00:56 +00002889 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002890 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002891 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002892
Tom Herbertfec5e652010-04-16 16:01:27 -07002893 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2894 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2895 if (flow_table && sock_flow_table) {
2896 u16 next_cpu;
2897 struct rps_dev_flow *rflow;
2898
2899 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2900 tcpu = rflow->cpu;
2901
2902 next_cpu = sock_flow_table->ents[skb->rxhash &
2903 sock_flow_table->mask];
2904
2905 /*
2906 * If the desired CPU (where last recvmsg was done) is
2907 * different from current CPU (one in the rx-queue flow
2908 * table entry), switch if one of the following holds:
2909 * - Current CPU is unset (equal to RPS_NO_CPU).
2910 * - Current CPU is offline.
2911 * - The current CPU's queue tail has advanced beyond the
2912 * last packet that was enqueued using this table entry.
2913 * This guarantees that all previous packets for the flow
2914 * have been dequeued, thus preserving in order delivery.
2915 */
2916 if (unlikely(tcpu != next_cpu) &&
2917 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2918 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00002919 rflow->last_qtail)) >= 0)) {
2920 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002921 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00002922 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00002923
Tom Herbertfec5e652010-04-16 16:01:27 -07002924 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2925 *rflowp = rflow;
2926 cpu = tcpu;
2927 goto done;
2928 }
2929 }
2930
Tom Herbert0a9627f2010-03-16 08:03:29 +00002931 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002932 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002933
2934 if (cpu_online(tcpu)) {
2935 cpu = tcpu;
2936 goto done;
2937 }
2938 }
2939
2940done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002941 return cpu;
2942}
2943
Ben Hutchingsc4454772011-01-19 11:03:53 +00002944#ifdef CONFIG_RFS_ACCEL
2945
2946/**
2947 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2948 * @dev: Device on which the filter was set
2949 * @rxq_index: RX queue index
2950 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2951 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2952 *
2953 * Drivers that implement ndo_rx_flow_steer() should periodically call
2954 * this function for each installed filter and remove the filters for
2955 * which it returns %true.
2956 */
2957bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2958 u32 flow_id, u16 filter_id)
2959{
2960 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2961 struct rps_dev_flow_table *flow_table;
2962 struct rps_dev_flow *rflow;
2963 bool expire = true;
2964 int cpu;
2965
2966 rcu_read_lock();
2967 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2968 if (flow_table && flow_id <= flow_table->mask) {
2969 rflow = &flow_table->flows[flow_id];
2970 cpu = ACCESS_ONCE(rflow->cpu);
2971 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2972 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2973 rflow->last_qtail) <
2974 (int)(10 * flow_table->mask)))
2975 expire = false;
2976 }
2977 rcu_read_unlock();
2978 return expire;
2979}
2980EXPORT_SYMBOL(rps_may_expire_flow);
2981
2982#endif /* CONFIG_RFS_ACCEL */
2983
Tom Herbert0a9627f2010-03-16 08:03:29 +00002984/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002985static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002986{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002987 struct softnet_data *sd = data;
2988
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002989 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002990 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002991}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002992
Tom Herbertfec5e652010-04-16 16:01:27 -07002993#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002994
2995/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002996 * Check if this softnet_data structure is another cpu one
2997 * If yes, queue it to our IPI list and return 1
2998 * If no, return 0
2999 */
3000static int rps_ipi_queued(struct softnet_data *sd)
3001{
3002#ifdef CONFIG_RPS
3003 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
3004
3005 if (sd != mysd) {
3006 sd->rps_ipi_next = mysd->rps_ipi_list;
3007 mysd->rps_ipi_list = sd;
3008
3009 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3010 return 1;
3011 }
3012#endif /* CONFIG_RPS */
3013 return 0;
3014}
3015
3016/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00003017 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3018 * queue (may be a remote CPU queue).
3019 */
Tom Herbertfec5e652010-04-16 16:01:27 -07003020static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3021 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00003022{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003023 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003024 unsigned long flags;
3025
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003026 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003027
3028 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003029
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003030 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003031 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
3032 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00003033enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003034 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003035 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003036 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00003037 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003038 return NET_RX_SUCCESS;
3039 }
3040
Eric Dumazetebda37c22010-05-06 23:51:21 +00003041 /* Schedule NAPI for backlog device
3042 * We can use non atomic operation since we own the queue lock
3043 */
3044 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003045 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003046 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003047 }
3048 goto enqueue;
3049 }
3050
Changli Gaodee42872010-05-02 05:42:16 +00003051 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003052 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003053
Tom Herbert0a9627f2010-03-16 08:03:29 +00003054 local_irq_restore(flags);
3055
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003056 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003057 kfree_skb(skb);
3058 return NET_RX_DROP;
3059}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003060
Linus Torvalds1da177e2005-04-16 15:20:36 -07003061/**
3062 * netif_rx - post buffer to the network code
3063 * @skb: buffer to post
3064 *
3065 * This function receives a packet from a device driver and queues it for
3066 * the upper (protocol) levels to process. It always succeeds. The buffer
3067 * may be dropped during processing for congestion control or by the
3068 * protocol layers.
3069 *
3070 * return values:
3071 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003072 * NET_RX_DROP (packet was dropped)
3073 *
3074 */
3075
3076int netif_rx(struct sk_buff *skb)
3077{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003078 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003079
3080 /* if netpoll wants it, pretend we never saw it */
3081 if (netpoll_rx(skb))
3082 return NET_RX_DROP;
3083
Eric Dumazet588f0332011-11-15 04:12:55 +00003084 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085
Koki Sanagicf66ba52010-08-23 18:45:02 +09003086 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003087#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003088 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07003089 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003090 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003091
Changli Gaocece1942010-08-07 20:35:43 -07003092 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003093 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003094
3095 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003096 if (cpu < 0)
3097 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003098
3099 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3100
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003101 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003102 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00003103 } else
3104#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07003105 {
3106 unsigned int qtail;
3107 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3108 put_cpu();
3109 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003110 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003111}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003112EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003113
3114int netif_rx_ni(struct sk_buff *skb)
3115{
3116 int err;
3117
3118 preempt_disable();
3119 err = netif_rx(skb);
3120 if (local_softirq_pending())
3121 do_softirq();
3122 preempt_enable();
3123
3124 return err;
3125}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003126EXPORT_SYMBOL(netif_rx_ni);
3127
Linus Torvalds1da177e2005-04-16 15:20:36 -07003128static void net_tx_action(struct softirq_action *h)
3129{
3130 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3131
3132 if (sd->completion_queue) {
3133 struct sk_buff *clist;
3134
3135 local_irq_disable();
3136 clist = sd->completion_queue;
3137 sd->completion_queue = NULL;
3138 local_irq_enable();
3139
3140 while (clist) {
3141 struct sk_buff *skb = clist;
3142 clist = clist->next;
3143
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003144 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09003145 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003146 __kfree_skb(skb);
3147 }
3148 }
3149
3150 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003151 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003152
3153 local_irq_disable();
3154 head = sd->output_queue;
3155 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003156 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003157 local_irq_enable();
3158
3159 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003160 struct Qdisc *q = head;
3161 spinlock_t *root_lock;
3162
Linus Torvalds1da177e2005-04-16 15:20:36 -07003163 head = head->next_sched;
3164
David S. Miller5fb66222008-08-02 20:02:43 -07003165 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003166 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003167 smp_mb__before_clear_bit();
3168 clear_bit(__QDISC_STATE_SCHED,
3169 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003170 qdisc_run(q);
3171 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003173 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003174 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003175 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003176 } else {
3177 smp_mb__before_clear_bit();
3178 clear_bit(__QDISC_STATE_SCHED,
3179 &q->state);
3180 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003181 }
3182 }
3183 }
3184}
3185
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003186#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3187 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003188/* This hook is defined here for ATM LANE */
3189int (*br_fdb_test_addr_hook)(struct net_device *dev,
3190 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003191EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003192#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003193
Linus Torvalds1da177e2005-04-16 15:20:36 -07003194#ifdef CONFIG_NET_CLS_ACT
3195/* TODO: Maybe we should just force sch_ingress to be compiled in
3196 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3197 * a compare and 2 stores extra right now if we dont have it on
3198 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003199 * NOTE: This doesn't stop any functionality; if you dont have
3200 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003201 *
3202 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003203static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003204{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003205 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003206 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003207 int result = TC_ACT_OK;
3208 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003209
Stephen Hemmingerde384832010-08-01 00:33:23 -07003210 if (unlikely(MAX_RED_LOOP < ttl++)) {
Joe Perchese87cc472012-05-13 21:56:26 +00003211 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3212 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003213 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003214 }
3215
Herbert Xuf697c3e2007-10-14 00:38:47 -07003216 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3217 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3218
David S. Miller83874002008-07-17 00:53:03 -07003219 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003220 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003221 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003222 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3223 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003224 spin_unlock(qdisc_lock(q));
3225 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003226
Linus Torvalds1da177e2005-04-16 15:20:36 -07003227 return result;
3228}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003229
3230static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3231 struct packet_type **pt_prev,
3232 int *ret, struct net_device *orig_dev)
3233{
Eric Dumazet24824a02010-10-02 06:11:55 +00003234 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3235
3236 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003237 goto out;
3238
3239 if (*pt_prev) {
3240 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3241 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003242 }
3243
Eric Dumazet24824a02010-10-02 06:11:55 +00003244 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003245 case TC_ACT_SHOT:
3246 case TC_ACT_STOLEN:
3247 kfree_skb(skb);
3248 return NULL;
3249 }
3250
3251out:
3252 skb->tc_verd = 0;
3253 return skb;
3254}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003255#endif
3256
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003257/**
3258 * netdev_rx_handler_register - register receive handler
3259 * @dev: device to register a handler for
3260 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003261 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003262 *
3263 * Register a receive hander for a device. This handler will then be
3264 * called from __netif_receive_skb. A negative errno code is returned
3265 * on a failure.
3266 *
3267 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003268 *
3269 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003270 */
3271int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003272 rx_handler_func_t *rx_handler,
3273 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003274{
3275 ASSERT_RTNL();
3276
3277 if (dev->rx_handler)
3278 return -EBUSY;
3279
Jiri Pirko93e2c322010-06-10 03:34:59 +00003280 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003281 rcu_assign_pointer(dev->rx_handler, rx_handler);
3282
3283 return 0;
3284}
3285EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3286
3287/**
3288 * netdev_rx_handler_unregister - unregister receive handler
3289 * @dev: device to unregister a handler from
3290 *
3291 * Unregister a receive hander from a device.
3292 *
3293 * The caller must hold the rtnl_mutex.
3294 */
3295void netdev_rx_handler_unregister(struct net_device *dev)
3296{
3297
3298 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003299 RCU_INIT_POINTER(dev->rx_handler, NULL);
3300 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003301}
3302EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3303
Mel Gormanb4b9e352012-07-31 16:44:26 -07003304/*
3305 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3306 * the special handling of PFMEMALLOC skbs.
3307 */
3308static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3309{
3310 switch (skb->protocol) {
3311 case __constant_htons(ETH_P_ARP):
3312 case __constant_htons(ETH_P_IP):
3313 case __constant_htons(ETH_P_IPV6):
3314 case __constant_htons(ETH_P_8021Q):
3315 return true;
3316 default:
3317 return false;
3318 }
3319}
3320
Eric Dumazet10f744d2010-03-28 23:07:20 -07003321static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003322{
3323 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003324 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003325 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003326 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003327 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003328 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003329 __be16 type;
Mel Gormanb4b9e352012-07-31 16:44:26 -07003330 unsigned long pflags = current->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003331
Eric Dumazet588f0332011-11-15 04:12:55 +00003332 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003333
Koki Sanagicf66ba52010-08-23 18:45:02 +09003334 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003335
Mel Gormanb4b9e352012-07-31 16:44:26 -07003336 /*
3337 * PFMEMALLOC skbs are special, they should
3338 * - be delivered to SOCK_MEMALLOC sockets only
3339 * - stay away from userspace
3340 * - have bounded memory usage
3341 *
3342 * Use PF_MEMALLOC as this saves us from propagating the allocation
3343 * context down to all allocation sites.
3344 */
3345 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
3346 current->flags |= PF_MEMALLOC;
3347
Linus Torvalds1da177e2005-04-16 15:20:36 -07003348 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003349 if (netpoll_receive_skb(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003350 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003351
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003352 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003353
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003354 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003355 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003356 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003357
3358 pt_prev = NULL;
3359
3360 rcu_read_lock();
3361
David S. Miller63d8ea72011-02-28 10:48:59 -08003362another_round:
David S. Millerb6858172012-07-23 16:27:54 -07003363 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08003364
3365 __this_cpu_inc(softnet_data.processed);
3366
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003367 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3368 skb = vlan_untag(skb);
3369 if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003370 goto unlock;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003371 }
3372
Linus Torvalds1da177e2005-04-16 15:20:36 -07003373#ifdef CONFIG_NET_CLS_ACT
3374 if (skb->tc_verd & TC_NCLS) {
3375 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3376 goto ncls;
3377 }
3378#endif
3379
Mel Gormanb4b9e352012-07-31 16:44:26 -07003380 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
3381 goto skip_taps;
3382
Linus Torvalds1da177e2005-04-16 15:20:36 -07003383 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003384 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003385 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003386 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003387 pt_prev = ptype;
3388 }
3389 }
3390
Mel Gormanb4b9e352012-07-31 16:44:26 -07003391skip_taps:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003392#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003393 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3394 if (!skb)
Mel Gormanb4b9e352012-07-31 16:44:26 -07003395 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003396ncls:
3397#endif
3398
Mel Gormanb4b9e352012-07-31 16:44:26 -07003399 if (sk_memalloc_socks() && skb_pfmemalloc(skb)
3400 && !skb_pfmemalloc_protocol(skb))
3401 goto drop;
3402
John Fastabend24257172011-10-10 09:16:41 +00003403 if (vlan_tx_tag_present(skb)) {
3404 if (pt_prev) {
3405 ret = deliver_skb(skb, pt_prev, orig_dev);
3406 pt_prev = NULL;
3407 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003408 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00003409 goto another_round;
3410 else if (unlikely(!skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07003411 goto unlock;
John Fastabend24257172011-10-10 09:16:41 +00003412 }
3413
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003414 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003415 if (rx_handler) {
3416 if (pt_prev) {
3417 ret = deliver_skb(skb, pt_prev, orig_dev);
3418 pt_prev = NULL;
3419 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003420 switch (rx_handler(&skb)) {
3421 case RX_HANDLER_CONSUMED:
Mel Gormanb4b9e352012-07-31 16:44:26 -07003422 goto unlock;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003423 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003424 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003425 case RX_HANDLER_EXACT:
3426 deliver_exact = true;
3427 case RX_HANDLER_PASS:
3428 break;
3429 default:
3430 BUG();
3431 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003432 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003433
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00003434 if (vlan_tx_nonzero_tag_present(skb))
3435 skb->pkt_type = PACKET_OTHERHOST;
3436
David S. Miller63d8ea72011-02-28 10:48:59 -08003437 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003438 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003439
Linus Torvalds1da177e2005-04-16 15:20:36 -07003440 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003441 list_for_each_entry_rcu(ptype,
3442 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003443 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003444 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3445 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003446 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003447 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003448 pt_prev = ptype;
3449 }
3450 }
3451
3452 if (pt_prev) {
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003453 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00003454 goto drop;
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00003455 else
3456 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003457 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07003458drop:
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003459 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003460 kfree_skb(skb);
3461 /* Jamal, now you will not able to escape explaining
3462 * me how you were going to use this. :-)
3463 */
3464 ret = NET_RX_DROP;
3465 }
3466
Mel Gormanb4b9e352012-07-31 16:44:26 -07003467unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003468 rcu_read_unlock();
Mel Gormanb4b9e352012-07-31 16:44:26 -07003469out:
3470 tsk_restore_flags(current, pflags, PF_MEMALLOC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003471 return ret;
3472}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003473
3474/**
3475 * netif_receive_skb - process receive buffer from network
3476 * @skb: buffer to process
3477 *
3478 * netif_receive_skb() is the main receive data processing function.
3479 * It always succeeds. The buffer may be dropped during processing
3480 * for congestion control or by the protocol layers.
3481 *
3482 * This function may only be called from softirq context and interrupts
3483 * should be enabled.
3484 *
3485 * Return values (usually ignored):
3486 * NET_RX_SUCCESS: no congestion
3487 * NET_RX_DROP: packet was dropped
3488 */
3489int netif_receive_skb(struct sk_buff *skb)
3490{
Eric Dumazet588f0332011-11-15 04:12:55 +00003491 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003492
Richard Cochranc1f19b52010-07-17 08:49:36 +00003493 if (skb_defer_rx_timestamp(skb))
3494 return NET_RX_SUCCESS;
3495
Eric Dumazetdf334542010-03-24 19:13:54 +00003496#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01003497 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07003498 struct rps_dev_flow voidflow, *rflow = &voidflow;
3499 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003500
Eric Dumazet3b098e22010-05-15 23:57:10 -07003501 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003502
Eric Dumazet3b098e22010-05-15 23:57:10 -07003503 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003504
Eric Dumazet3b098e22010-05-15 23:57:10 -07003505 if (cpu >= 0) {
3506 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3507 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00003508 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07003509 }
Eric Dumazetadc93002011-11-17 03:13:26 +00003510 rcu_read_unlock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003511 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003512#endif
Eric Dumazetadc93002011-11-17 03:13:26 +00003513 return __netif_receive_skb(skb);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003514}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003515EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003516
Eric Dumazet88751272010-04-19 05:07:33 +00003517/* Network device is going away, flush any packets still pending
3518 * Called with irqs disabled.
3519 */
Changli Gao152102c2010-03-30 20:16:22 +00003520static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003521{
Changli Gao152102c2010-03-30 20:16:22 +00003522 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003523 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003524 struct sk_buff *skb, *tmp;
3525
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003526 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003527 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003528 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003529 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003530 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003531 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003532 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003533 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003534 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003535
3536 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3537 if (skb->dev == dev) {
3538 __skb_unlink(skb, &sd->process_queue);
3539 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003540 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003541 }
3542 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003543}
3544
Herbert Xud565b0a2008-12-15 23:38:52 -08003545static int napi_gro_complete(struct sk_buff *skb)
3546{
Vlad Yasevich22061d82012-11-15 08:49:11 +00003547 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003548 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003549 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08003550 int err = -ENOENT;
3551
Herbert Xufc59f9a2009-04-14 15:11:06 -07003552 if (NAPI_GRO_CB(skb)->count == 1) {
3553 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003554 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003555 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003556
3557 rcu_read_lock();
3558 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003559 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08003560 continue;
3561
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003562 err = ptype->callbacks.gro_complete(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003563 break;
3564 }
3565 rcu_read_unlock();
3566
3567 if (err) {
3568 WARN_ON(&ptype->list == head);
3569 kfree_skb(skb);
3570 return NET_RX_SUCCESS;
3571 }
3572
3573out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003574 return netif_receive_skb(skb);
3575}
3576
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003577/* napi->gro_list contains packets ordered by age.
3578 * youngest packets at the head of it.
3579 * Complete skbs in reverse order to reduce latencies.
3580 */
3581void napi_gro_flush(struct napi_struct *napi, bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08003582{
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003583 struct sk_buff *skb, *prev = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003584
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003585 /* scan list and build reverse chain */
3586 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3587 skb->prev = prev;
3588 prev = skb;
Herbert Xud565b0a2008-12-15 23:38:52 -08003589 }
3590
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003591 for (skb = prev; skb; skb = prev) {
3592 skb->next = NULL;
3593
3594 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3595 return;
3596
3597 prev = skb->prev;
3598 napi_gro_complete(skb);
3599 napi->gro_count--;
3600 }
3601
Herbert Xud565b0a2008-12-15 23:38:52 -08003602 napi->gro_list = NULL;
3603}
Eric Dumazet86cac582010-08-31 18:25:32 +00003604EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003605
Rami Rosenbb728822012-11-28 21:55:25 +00003606static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003607{
3608 struct sk_buff **pp = NULL;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003609 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08003610 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00003611 struct list_head *head = &offload_base;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003612 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003613 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003614 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003615
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003616 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003617 goto normal;
3618
David S. Miller21dc3302010-08-23 00:13:46 -07003619 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003620 goto normal;
3621
Herbert Xud565b0a2008-12-15 23:38:52 -08003622 rcu_read_lock();
3623 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003624 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08003625 continue;
3626
Herbert Xu86911732009-01-29 14:19:50 +00003627 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003628 mac_len = skb->network_header - skb->mac_header;
3629 skb->mac_len = mac_len;
3630 NAPI_GRO_CB(skb)->same_flow = 0;
3631 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003632 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003633
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00003634 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003635 break;
3636 }
3637 rcu_read_unlock();
3638
3639 if (&ptype->list == head)
3640 goto normal;
3641
Herbert Xu0da2afd52008-12-26 14:57:42 -08003642 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003643 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003644
Herbert Xud565b0a2008-12-15 23:38:52 -08003645 if (pp) {
3646 struct sk_buff *nskb = *pp;
3647
3648 *pp = nskb->next;
3649 nskb->next = NULL;
3650 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003651 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003652 }
3653
Herbert Xu0da2afd52008-12-26 14:57:42 -08003654 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003655 goto ok;
3656
Herbert Xu4ae55442009-02-08 18:00:36 +00003657 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003658 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003659
Herbert Xu4ae55442009-02-08 18:00:36 +00003660 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003661 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003662 NAPI_GRO_CB(skb)->age = jiffies;
Herbert Xu86911732009-01-29 14:19:50 +00003663 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003664 skb->next = napi->gro_list;
3665 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003666 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003667
Herbert Xuad0f9902009-02-01 01:24:55 -08003668pull:
Herbert Xucb189782009-05-26 18:50:31 +00003669 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3670 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3671
3672 BUG_ON(skb->end - skb->tail < grow);
3673
3674 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3675
3676 skb->tail += grow;
3677 skb->data_len -= grow;
3678
3679 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003680 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003681
Eric Dumazet9e903e02011-10-18 21:00:24 +00003682 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003683 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003684 memmove(skb_shinfo(skb)->frags,
3685 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003686 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003687 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003688 }
3689
Herbert Xud565b0a2008-12-15 23:38:52 -08003690ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003691 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003692
3693normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003694 ret = GRO_NORMAL;
3695 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003696}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003697
Eric Dumazet40d08022010-08-26 22:03:08 -07003698static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003699__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003700{
3701 struct sk_buff *p;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003702 unsigned int maclen = skb->dev->hard_header_len;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003703
3704 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003705 unsigned long diffs;
3706
3707 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003708 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet5ca3b72c2012-02-08 08:51:50 +00003709 if (maclen == ETH_HLEN)
3710 diffs |= compare_ether_header(skb_mac_header(p),
3711 skb_gro_mac_header(skb));
3712 else if (!diffs)
3713 diffs = memcmp(skb_mac_header(p),
3714 skb_gro_mac_header(skb),
3715 maclen);
Eric Dumazet40d08022010-08-26 22:03:08 -07003716 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003717 NAPI_GRO_CB(p)->flush = 0;
3718 }
3719
3720 return dev_gro_receive(napi, skb);
3721}
Herbert Xu5d38a072009-01-04 16:13:40 -08003722
Rami Rosenbb728822012-11-28 21:55:25 +00003723static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003724{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003725 switch (ret) {
3726 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003727 if (netif_receive_skb(skb))
3728 ret = GRO_DROP;
3729 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003730
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003731 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08003732 kfree_skb(skb);
3733 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003734
Eric Dumazetdaa86542012-04-19 07:07:40 +00003735 case GRO_MERGED_FREE:
Eric Dumazetd7e88832012-04-30 08:10:34 +00003736 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
3737 kmem_cache_free(skbuff_head_cache, skb);
3738 else
3739 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00003740 break;
3741
Ben Hutchings5b252f02009-10-29 07:17:09 +00003742 case GRO_HELD:
3743 case GRO_MERGED:
3744 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003745 }
3746
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003747 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003748}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003749
Eric Dumazetca07e432012-10-06 22:28:06 +00003750static void skb_gro_reset_offset(struct sk_buff *skb)
Herbert Xu78a478d2009-05-26 18:50:21 +00003751{
Eric Dumazetca07e432012-10-06 22:28:06 +00003752 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3753 const skb_frag_t *frag0 = &pinfo->frags[0];
3754
Herbert Xu78a478d2009-05-26 18:50:21 +00003755 NAPI_GRO_CB(skb)->data_offset = 0;
3756 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003757 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003758
Herbert Xu78d3fd02009-05-26 18:50:23 +00003759 if (skb->mac_header == skb->tail &&
Eric Dumazetca07e432012-10-06 22:28:06 +00003760 pinfo->nr_frags &&
3761 !PageHighMem(skb_frag_page(frag0))) {
3762 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3763 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
Herbert Xu74895942009-05-26 18:50:27 +00003764 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003765}
Herbert Xu78a478d2009-05-26 18:50:21 +00003766
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003767gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003768{
Herbert Xu86911732009-01-29 14:19:50 +00003769 skb_gro_reset_offset(skb);
3770
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003771 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003772}
3773EXPORT_SYMBOL(napi_gro_receive);
3774
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003775static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003776{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003777 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00003778 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
3779 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003780 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003781 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003782 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003783
3784 napi->skb = skb;
3785}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003786
Herbert Xu76620aa2009-04-16 02:02:07 -07003787struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003788{
Herbert Xu5d38a072009-01-04 16:13:40 -08003789 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003790
3791 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003792 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3793 if (skb)
3794 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003795 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003796 return skb;
3797}
Herbert Xu76620aa2009-04-16 02:02:07 -07003798EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003799
Rami Rosenbb728822012-11-28 21:55:25 +00003800static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003801 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003802{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003803 switch (ret) {
3804 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003805 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003806 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003807
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003808 if (ret == GRO_HELD)
3809 skb_gro_pull(skb, -ETH_HLEN);
3810 else if (netif_receive_skb(skb))
3811 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003812 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003813
3814 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003815 case GRO_MERGED_FREE:
3816 napi_reuse_skb(napi, skb);
3817 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003818
3819 case GRO_MERGED:
3820 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003821 }
3822
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003823 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003824}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003825
Eric Dumazet4adb9c42012-05-18 20:49:06 +00003826static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003827{
Herbert Xu76620aa2009-04-16 02:02:07 -07003828 struct sk_buff *skb = napi->skb;
3829 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003830 unsigned int hlen;
3831 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003832
3833 napi->skb = NULL;
3834
3835 skb_reset_mac_header(skb);
3836 skb_gro_reset_offset(skb);
3837
Herbert Xua5b1cf22009-05-26 18:50:28 +00003838 off = skb_gro_offset(skb);
3839 hlen = off + sizeof(*eth);
3840 eth = skb_gro_header_fast(skb, off);
3841 if (skb_gro_header_hard(skb, hlen)) {
3842 eth = skb_gro_header_slow(skb, hlen, off);
3843 if (unlikely(!eth)) {
3844 napi_reuse_skb(napi, skb);
3845 skb = NULL;
3846 goto out;
3847 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003848 }
3849
3850 skb_gro_pull(skb, sizeof(*eth));
3851
3852 /*
3853 * This works because the only protocols we care about don't require
3854 * special handling. We'll fix it up properly at the end.
3855 */
3856 skb->protocol = eth->h_proto;
3857
3858out:
3859 return skb;
3860}
Herbert Xu76620aa2009-04-16 02:02:07 -07003861
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003862gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003863{
3864 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003865
3866 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003867 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003868
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003869 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003870}
3871EXPORT_SYMBOL(napi_gro_frags);
3872
Eric Dumazete326bed2010-04-22 00:22:45 -07003873/*
3874 * net_rps_action sends any pending IPI's for rps.
3875 * Note: called with local irq disabled, but exits with local irq enabled.
3876 */
3877static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3878{
3879#ifdef CONFIG_RPS
3880 struct softnet_data *remsd = sd->rps_ipi_list;
3881
3882 if (remsd) {
3883 sd->rps_ipi_list = NULL;
3884
3885 local_irq_enable();
3886
3887 /* Send pending IPI's to kick RPS processing on remote cpus. */
3888 while (remsd) {
3889 struct softnet_data *next = remsd->rps_ipi_next;
3890
3891 if (cpu_online(remsd->cpu))
3892 __smp_call_function_single(remsd->cpu,
3893 &remsd->csd, 0);
3894 remsd = next;
3895 }
3896 } else
3897#endif
3898 local_irq_enable();
3899}
3900
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003901static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003902{
3903 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003904 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003905
Eric Dumazete326bed2010-04-22 00:22:45 -07003906#ifdef CONFIG_RPS
3907 /* Check if we have pending ipi, its better to send them now,
3908 * not waiting net_rx_action() end.
3909 */
3910 if (sd->rps_ipi_list) {
3911 local_irq_disable();
3912 net_rps_action_and_irq_enable(sd);
3913 }
3914#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003915 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003916 local_irq_disable();
3917 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003918 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003919 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003920
Changli Gao6e7676c2010-04-27 15:07:33 -07003921 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003922 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003923 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003924 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003925 input_queue_head_incr(sd);
3926 if (++work >= quota) {
3927 local_irq_enable();
3928 return work;
3929 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003930 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003931
Changli Gao6e7676c2010-04-27 15:07:33 -07003932 rps_lock(sd);
3933 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003934 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003935 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3936 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003937
Changli Gao6e7676c2010-04-27 15:07:33 -07003938 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003939 /*
3940 * Inline a custom version of __napi_complete().
3941 * only current cpu owns and manipulates this napi,
3942 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3943 * we can use a plain write instead of clear_bit(),
3944 * and we dont need an smp_mb() memory barrier.
3945 */
3946 list_del(&napi->poll_list);
3947 napi->state = 0;
3948
Changli Gao6e7676c2010-04-27 15:07:33 -07003949 quota = work + qlen;
3950 }
3951 rps_unlock(sd);
3952 }
3953 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003954
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003955 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003956}
3957
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003958/**
3959 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003960 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003961 *
3962 * The entry's receive function will be scheduled to run
3963 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003964void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003965{
3966 unsigned long flags;
3967
3968 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003969 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003970 local_irq_restore(flags);
3971}
3972EXPORT_SYMBOL(__napi_schedule);
3973
Herbert Xud565b0a2008-12-15 23:38:52 -08003974void __napi_complete(struct napi_struct *n)
3975{
3976 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3977 BUG_ON(n->gro_list);
3978
3979 list_del(&n->poll_list);
3980 smp_mb__before_clear_bit();
3981 clear_bit(NAPI_STATE_SCHED, &n->state);
3982}
3983EXPORT_SYMBOL(__napi_complete);
3984
3985void napi_complete(struct napi_struct *n)
3986{
3987 unsigned long flags;
3988
3989 /*
3990 * don't let napi dequeue from the cpu poll list
3991 * just in case its running on a different cpu
3992 */
3993 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3994 return;
3995
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00003996 napi_gro_flush(n, false);
Herbert Xud565b0a2008-12-15 23:38:52 -08003997 local_irq_save(flags);
3998 __napi_complete(n);
3999 local_irq_restore(flags);
4000}
4001EXPORT_SYMBOL(napi_complete);
4002
4003void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4004 int (*poll)(struct napi_struct *, int), int weight)
4005{
4006 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00004007 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004008 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08004009 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08004010 napi->poll = poll;
4011 napi->weight = weight;
4012 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08004013 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08004014#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08004015 spin_lock_init(&napi->poll_lock);
4016 napi->poll_owner = -1;
4017#endif
4018 set_bit(NAPI_STATE_SCHED, &napi->state);
4019}
4020EXPORT_SYMBOL(netif_napi_add);
4021
4022void netif_napi_del(struct napi_struct *napi)
4023{
4024 struct sk_buff *skb, *next;
4025
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08004026 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07004027 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08004028
4029 for (skb = napi->gro_list; skb; skb = next) {
4030 next = skb->next;
4031 skb->next = NULL;
4032 kfree_skb(skb);
4033 }
4034
4035 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00004036 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08004037}
4038EXPORT_SYMBOL(netif_napi_del);
4039
Linus Torvalds1da177e2005-04-16 15:20:36 -07004040static void net_rx_action(struct softirq_action *h)
4041{
Eric Dumazete326bed2010-04-22 00:22:45 -07004042 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004043 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07004044 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07004045 void *have;
4046
Linus Torvalds1da177e2005-04-16 15:20:36 -07004047 local_irq_disable();
4048
Eric Dumazete326bed2010-04-22 00:22:45 -07004049 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004050 struct napi_struct *n;
4051 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004052
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004053 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004054 * Allow this to run for 2 jiffies since which will allow
4055 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004056 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08004057 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004058 goto softnet_break;
4059
4060 local_irq_enable();
4061
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004062 /* Even though interrupts have been re-enabled, this
4063 * access is safe because interrupts can only add new
4064 * entries to the tail of this list, and only ->poll()
4065 * calls can remove this head entry from the list.
4066 */
Eric Dumazete326bed2010-04-22 00:22:45 -07004067 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004068
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004069 have = netpoll_poll_lock(n);
4070
4071 weight = n->weight;
4072
David S. Miller0a7606c2007-10-29 21:28:47 -07004073 /* This NAPI_STATE_SCHED test is for avoiding a race
4074 * with netpoll's poll_napi(). Only the entity which
4075 * obtains the lock and sees NAPI_STATE_SCHED set will
4076 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004077 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07004078 */
4079 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00004080 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07004081 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00004082 trace_napi_poll(n);
4083 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004084
4085 WARN_ON_ONCE(work > weight);
4086
4087 budget -= work;
4088
4089 local_irq_disable();
4090
4091 /* Drivers must not modify the NAPI state if they
4092 * consume the entire weight. In such cases this code
4093 * still "owns" the NAPI instance and therefore can
4094 * move the instance around on the list at-will.
4095 */
David S. Millerfed17f32008-01-07 21:00:40 -08004096 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07004097 if (unlikely(napi_disable_pending(n))) {
4098 local_irq_enable();
4099 napi_complete(n);
4100 local_irq_disable();
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004101 } else {
4102 if (n->gro_list) {
4103 /* flush too old packets
4104 * If HZ < 1000, flush all packets.
4105 */
4106 local_irq_enable();
4107 napi_gro_flush(n, HZ >= 1000);
4108 local_irq_disable();
4109 }
Eric Dumazete326bed2010-04-22 00:22:45 -07004110 list_move_tail(&n->poll_list, &sd->poll_list);
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00004111 }
David S. Millerfed17f32008-01-07 21:00:40 -08004112 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004113
4114 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004115 }
4116out:
Eric Dumazete326bed2010-04-22 00:22:45 -07004117 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004118
Chris Leechdb217332006-06-17 21:24:58 -07004119#ifdef CONFIG_NET_DMA
4120 /*
4121 * There may not be any more sk_buffs coming right now, so push
4122 * any pending DMA copies to hardware
4123 */
Dan Williams2ba05622009-01-06 11:38:14 -07004124 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07004125#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07004126
Linus Torvalds1da177e2005-04-16 15:20:36 -07004127 return;
4128
4129softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00004130 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004131 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4132 goto out;
4133}
4134
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004135static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004136
4137/**
4138 * register_gifconf - register a SIOCGIF handler
4139 * @family: Address family
4140 * @gifconf: Function handler
4141 *
4142 * Register protocol dependent address dumping routines. The handler
4143 * that is passed must not be freed or reused until it has been replaced
4144 * by another handler.
4145 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004146int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004147{
4148 if (family >= NPROTO)
4149 return -EINVAL;
4150 gifconf_list[family] = gifconf;
4151 return 0;
4152}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004153EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004154
4155
4156/*
4157 * Map an interface index to its name (SIOCGIFNAME)
4158 */
4159
4160/*
4161 * We need this ioctl for efficient implementation of the
4162 * if_indextoname() function required by the IPv6 API. Without
4163 * it, we would have to search all the interfaces to find a
4164 * match. --pb
4165 */
4166
Eric W. Biederman881d9662007-09-17 11:56:21 -07004167static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004168{
4169 struct net_device *dev;
4170 struct ifreq ifr;
Brian Haleyc91f6df2012-11-26 05:21:08 +00004171 unsigned seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004172
4173 /*
4174 * Fetch the caller's info block.
4175 */
4176
4177 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4178 return -EFAULT;
4179
Brian Haleyc91f6df2012-11-26 05:21:08 +00004180retry:
4181 seq = read_seqbegin(&devnet_rename_seq);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004182 rcu_read_lock();
4183 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004184 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004185 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004186 return -ENODEV;
4187 }
4188
4189 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004190 rcu_read_unlock();
Brian Haleyc91f6df2012-11-26 05:21:08 +00004191 if (read_seqretry(&devnet_rename_seq, seq))
4192 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004193
4194 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
4195 return -EFAULT;
4196 return 0;
4197}
4198
4199/*
4200 * Perform a SIOCGIFCONF call. This structure will change
4201 * size eventually, and there is nothing I can do about it.
4202 * Thus we will need a 'compatibility mode'.
4203 */
4204
Eric W. Biederman881d9662007-09-17 11:56:21 -07004205static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004206{
4207 struct ifconf ifc;
4208 struct net_device *dev;
4209 char __user *pos;
4210 int len;
4211 int total;
4212 int i;
4213
4214 /*
4215 * Fetch the caller's info block.
4216 */
4217
4218 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
4219 return -EFAULT;
4220
4221 pos = ifc.ifc_buf;
4222 len = ifc.ifc_len;
4223
4224 /*
4225 * Loop over the interfaces, and write an info block for each.
4226 */
4227
4228 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004229 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004230 for (i = 0; i < NPROTO; i++) {
4231 if (gifconf_list[i]) {
4232 int done;
4233 if (!pos)
4234 done = gifconf_list[i](dev, NULL, 0);
4235 else
4236 done = gifconf_list[i](dev, pos + total,
4237 len - total);
4238 if (done < 0)
4239 return -EFAULT;
4240 total += done;
4241 }
4242 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004243 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004244
4245 /*
4246 * All done. Write the updated control block back to the caller.
4247 */
4248 ifc.ifc_len = total;
4249
4250 /*
4251 * Both BSD and Solaris return 0 here, so we do too.
4252 */
4253 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
4254}
4255
4256#ifdef CONFIG_PROC_FS
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004257
Eric Dumazet2def16a2012-04-02 22:33:02 +00004258#define BUCKET_SPACE (32 - NETDEV_HASHBITS - 1)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004259
4260#define get_bucket(x) ((x) >> BUCKET_SPACE)
4261#define get_offset(x) ((x) & ((1 << BUCKET_SPACE) - 1))
4262#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))
4263
Eric Dumazet2def16a2012-04-02 22:33:02 +00004264static inline struct net_device *dev_from_same_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004265{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004266 struct net *net = seq_file_net(seq);
4267 struct net_device *dev;
4268 struct hlist_node *p;
4269 struct hlist_head *h;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004270 unsigned int count = 0, offset = get_offset(*pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004271
Eric Dumazet2def16a2012-04-02 22:33:02 +00004272 h = &net->dev_name_head[get_bucket(*pos)];
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004273 hlist_for_each_entry_rcu(dev, p, h, name_hlist) {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004274 if (++count == offset)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004275 return dev;
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004276 }
4277
4278 return NULL;
4279}
4280
Eric Dumazet2def16a2012-04-02 22:33:02 +00004281static inline struct net_device *dev_from_bucket(struct seq_file *seq, loff_t *pos)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004282{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004283 struct net_device *dev;
4284 unsigned int bucket;
4285
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004286 do {
Eric Dumazet2def16a2012-04-02 22:33:02 +00004287 dev = dev_from_same_bucket(seq, pos);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004288 if (dev)
4289 return dev;
4290
Eric Dumazet2def16a2012-04-02 22:33:02 +00004291 bucket = get_bucket(*pos) + 1;
4292 *pos = set_bucket_offset(bucket, 1);
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004293 } while (bucket < NETDEV_HASHENTRIES);
4294
4295 return NULL;
4296}
4297
Linus Torvalds1da177e2005-04-16 15:20:36 -07004298/*
4299 * This is invoked by the /proc filesystem handler to display a device
4300 * in detail.
4301 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004302void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004303 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004304{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004305 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07004306 if (!*pos)
4307 return SEQ_START_TOKEN;
4308
Eric Dumazet2def16a2012-04-02 22:33:02 +00004309 if (get_bucket(*pos) >= NETDEV_HASHENTRIES)
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004310 return NULL;
Pavel Emelianov7562f872007-05-03 15:13:45 -07004311
Eric Dumazet2def16a2012-04-02 22:33:02 +00004312 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004313}
4314
4315void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4316{
4317 ++*pos;
Eric Dumazet2def16a2012-04-02 22:33:02 +00004318 return dev_from_bucket(seq, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004319}
4320
4321void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004322 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004323{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004324 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004325}
4326
4327static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4328{
Eric Dumazet28172732010-07-07 14:58:56 -07004329 struct rtnl_link_stats64 temp;
4330 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004331
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004332 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4333 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004334 dev->name, stats->rx_bytes, stats->rx_packets,
4335 stats->rx_errors,
4336 stats->rx_dropped + stats->rx_missed_errors,
4337 stats->rx_fifo_errors,
4338 stats->rx_length_errors + stats->rx_over_errors +
4339 stats->rx_crc_errors + stats->rx_frame_errors,
4340 stats->rx_compressed, stats->multicast,
4341 stats->tx_bytes, stats->tx_packets,
4342 stats->tx_errors, stats->tx_dropped,
4343 stats->tx_fifo_errors, stats->collisions,
4344 stats->tx_carrier_errors +
4345 stats->tx_aborted_errors +
4346 stats->tx_window_errors +
4347 stats->tx_heartbeat_errors,
4348 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004349}
4350
4351/*
4352 * Called from the PROCfs module. This now uses the new arbitrary sized
4353 * /proc/net interface to create /proc/net/dev
4354 */
4355static int dev_seq_show(struct seq_file *seq, void *v)
4356{
4357 if (v == SEQ_START_TOKEN)
4358 seq_puts(seq, "Inter-| Receive "
4359 " | Transmit\n"
4360 " face |bytes packets errs drop fifo frame "
4361 "compressed multicast|bytes packets errs "
4362 "drop fifo colls carrier compressed\n");
4363 else
4364 dev_seq_printf_stats(seq, v);
4365 return 0;
4366}
4367
Changli Gaodee42872010-05-02 05:42:16 +00004368static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004369{
Changli Gaodee42872010-05-02 05:42:16 +00004370 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004371
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004372 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004373 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004374 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004375 break;
4376 } else
4377 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004378 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004379}
4380
4381static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4382{
4383 return softnet_get_online(pos);
4384}
4385
4386static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4387{
4388 ++*pos;
4389 return softnet_get_online(pos);
4390}
4391
4392static void softnet_seq_stop(struct seq_file *seq, void *v)
4393{
4394}
4395
4396static int softnet_seq_show(struct seq_file *seq, void *v)
4397{
Changli Gaodee42872010-05-02 05:42:16 +00004398 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004399
Tom Herbert0a9627f2010-03-16 08:03:29 +00004400 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004401 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004402 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004403 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004404 return 0;
4405}
4406
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004407static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004408 .start = dev_seq_start,
4409 .next = dev_seq_next,
4410 .stop = dev_seq_stop,
4411 .show = dev_seq_show,
4412};
4413
4414static int dev_seq_open(struct inode *inode, struct file *file)
4415{
Denis V. Luneve372c412007-11-19 22:31:54 -08004416 return seq_open_net(inode, file, &dev_seq_ops,
Eric Dumazet2def16a2012-04-02 22:33:02 +00004417 sizeof(struct seq_net_private));
Anton Blanchard5cac98d2011-11-27 21:14:46 +00004418}
4419
Arjan van de Ven9a321442007-02-12 00:55:35 -08004420static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421 .owner = THIS_MODULE,
4422 .open = dev_seq_open,
4423 .read = seq_read,
4424 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004425 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004426};
4427
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004428static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004429 .start = softnet_seq_start,
4430 .next = softnet_seq_next,
4431 .stop = softnet_seq_stop,
4432 .show = softnet_seq_show,
4433};
4434
4435static int softnet_seq_open(struct inode *inode, struct file *file)
4436{
4437 return seq_open(file, &softnet_seq_ops);
4438}
4439
Arjan van de Ven9a321442007-02-12 00:55:35 -08004440static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004441 .owner = THIS_MODULE,
4442 .open = softnet_seq_open,
4443 .read = seq_read,
4444 .llseek = seq_lseek,
4445 .release = seq_release,
4446};
4447
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004448static void *ptype_get_idx(loff_t pos)
4449{
4450 struct packet_type *pt = NULL;
4451 loff_t i = 0;
4452 int t;
4453
4454 list_for_each_entry_rcu(pt, &ptype_all, list) {
4455 if (i == pos)
4456 return pt;
4457 ++i;
4458 }
4459
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004460 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004461 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4462 if (i == pos)
4463 return pt;
4464 ++i;
4465 }
4466 }
4467 return NULL;
4468}
4469
4470static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004471 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004472{
4473 rcu_read_lock();
4474 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4475}
4476
4477static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4478{
4479 struct packet_type *pt;
4480 struct list_head *nxt;
4481 int hash;
4482
4483 ++*pos;
4484 if (v == SEQ_START_TOKEN)
4485 return ptype_get_idx(0);
4486
4487 pt = v;
4488 nxt = pt->list.next;
4489 if (pt->type == htons(ETH_P_ALL)) {
4490 if (nxt != &ptype_all)
4491 goto found;
4492 hash = 0;
4493 nxt = ptype_base[0].next;
4494 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004495 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004496
4497 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004498 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004499 return NULL;
4500 nxt = ptype_base[hash].next;
4501 }
4502found:
4503 return list_entry(nxt, struct packet_type, list);
4504}
4505
4506static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004507 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004508{
4509 rcu_read_unlock();
4510}
4511
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004512static int ptype_seq_show(struct seq_file *seq, void *v)
4513{
4514 struct packet_type *pt = v;
4515
4516 if (v == SEQ_START_TOKEN)
4517 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004518 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004519 if (pt->type == htons(ETH_P_ALL))
4520 seq_puts(seq, "ALL ");
4521 else
4522 seq_printf(seq, "%04x", ntohs(pt->type));
4523
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004524 seq_printf(seq, " %-8s %pF\n",
4525 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004526 }
4527
4528 return 0;
4529}
4530
4531static const struct seq_operations ptype_seq_ops = {
4532 .start = ptype_seq_start,
4533 .next = ptype_seq_next,
4534 .stop = ptype_seq_stop,
4535 .show = ptype_seq_show,
4536};
4537
4538static int ptype_seq_open(struct inode *inode, struct file *file)
4539{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004540 return seq_open_net(inode, file, &ptype_seq_ops,
4541 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004542}
4543
4544static const struct file_operations ptype_seq_fops = {
4545 .owner = THIS_MODULE,
4546 .open = ptype_seq_open,
4547 .read = seq_read,
4548 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004549 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004550};
4551
4552
Pavel Emelyanov46650792007-10-08 20:38:39 -07004553static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004554{
4555 int rc = -ENOMEM;
4556
Eric W. Biederman881d9662007-09-17 11:56:21 -07004557 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004558 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004559 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004560 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004561 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004562 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004563
Eric W. Biederman881d9662007-09-17 11:56:21 -07004564 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004565 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004566 rc = 0;
4567out:
4568 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004569out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004570 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004571out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004572 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004573out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004574 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004575 goto out;
4576}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004577
Pavel Emelyanov46650792007-10-08 20:38:39 -07004578static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004579{
4580 wext_proc_exit(net);
4581
4582 proc_net_remove(net, "ptype");
4583 proc_net_remove(net, "softnet_stat");
4584 proc_net_remove(net, "dev");
4585}
4586
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004587static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004588 .init = dev_proc_net_init,
4589 .exit = dev_proc_net_exit,
4590};
4591
4592static int __init dev_proc_init(void)
4593{
4594 return register_pernet_subsys(&dev_proc_ops);
4595}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004596#else
4597#define dev_proc_init() 0
4598#endif /* CONFIG_PROC_FS */
4599
4600
4601/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004602 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004603 * @slave: slave device
4604 * @master: new master device
4605 *
4606 * Changes the master device of the slave. Pass %NULL to break the
4607 * bonding. The caller must hold the RTNL semaphore. On a failure
4608 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004609 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004610 */
4611int netdev_set_master(struct net_device *slave, struct net_device *master)
4612{
4613 struct net_device *old = slave->master;
4614
4615 ASSERT_RTNL();
4616
4617 if (master) {
4618 if (old)
4619 return -EBUSY;
4620 dev_hold(master);
4621 }
4622
4623 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004624
Eric Dumazet6df427f2011-05-19 19:37:40 +00004625 if (old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004626 dev_put(old);
Jiri Pirko1765a572011-02-12 06:48:36 +00004627 return 0;
4628}
4629EXPORT_SYMBOL(netdev_set_master);
4630
4631/**
4632 * netdev_set_bond_master - set up bonding master/slave pair
4633 * @slave: slave device
4634 * @master: new master device
4635 *
4636 * Changes the master device of the slave. Pass %NULL to break the
4637 * bonding. The caller must hold the RTNL semaphore. On a failure
4638 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4639 * to the routing socket and the function returns zero.
4640 */
4641int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4642{
4643 int err;
4644
4645 ASSERT_RTNL();
4646
4647 err = netdev_set_master(slave, master);
4648 if (err)
4649 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004650 if (master)
4651 slave->flags |= IFF_SLAVE;
4652 else
4653 slave->flags &= ~IFF_SLAVE;
4654
4655 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4656 return 0;
4657}
Jiri Pirko1765a572011-02-12 06:48:36 +00004658EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004659
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004660static void dev_change_rx_flags(struct net_device *dev, int flags)
4661{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004662 const struct net_device_ops *ops = dev->netdev_ops;
4663
4664 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4665 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004666}
4667
Wang Chendad9b332008-06-18 01:48:28 -07004668static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004669{
Eric Dumazetb536db92011-11-30 21:42:26 +00004670 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004671 kuid_t uid;
4672 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004673
Patrick McHardy24023452007-07-14 18:51:31 -07004674 ASSERT_RTNL();
4675
Wang Chendad9b332008-06-18 01:48:28 -07004676 dev->flags |= IFF_PROMISC;
4677 dev->promiscuity += inc;
4678 if (dev->promiscuity == 0) {
4679 /*
4680 * Avoid overflow.
4681 * If inc causes overflow, untouch promisc and return error.
4682 */
4683 if (inc < 0)
4684 dev->flags &= ~IFF_PROMISC;
4685 else {
4686 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004687 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
4688 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004689 return -EOVERFLOW;
4690 }
4691 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004692 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004693 pr_info("device %s %s promiscuous mode\n",
4694 dev->name,
4695 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11004696 if (audit_enabled) {
4697 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004698 audit_log(current->audit_context, GFP_ATOMIC,
4699 AUDIT_ANOM_PROMISCUOUS,
4700 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4701 dev->name, (dev->flags & IFF_PROMISC),
4702 (old_flags & IFF_PROMISC),
Eric W. Biedermane1760bd2012-09-10 22:39:43 -07004703 from_kuid(&init_user_ns, audit_get_loginuid(current)),
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06004704 from_kuid(&init_user_ns, uid),
4705 from_kgid(&init_user_ns, gid),
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004706 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004707 }
Patrick McHardy24023452007-07-14 18:51:31 -07004708
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004709 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004710 }
Wang Chendad9b332008-06-18 01:48:28 -07004711 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004712}
4713
Linus Torvalds1da177e2005-04-16 15:20:36 -07004714/**
4715 * dev_set_promiscuity - update promiscuity count on a device
4716 * @dev: device
4717 * @inc: modifier
4718 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004719 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004720 * remains above zero the interface remains promiscuous. Once it hits zero
4721 * the device reverts back to normal filtering operation. A negative inc
4722 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004723 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004724 */
Wang Chendad9b332008-06-18 01:48:28 -07004725int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004726{
Eric Dumazetb536db92011-11-30 21:42:26 +00004727 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004728 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004729
Wang Chendad9b332008-06-18 01:48:28 -07004730 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004731 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004732 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004733 if (dev->flags != old_flags)
4734 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004735 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004736}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004737EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004738
4739/**
4740 * dev_set_allmulti - update allmulti count on a device
4741 * @dev: device
4742 * @inc: modifier
4743 *
4744 * Add or remove reception of all multicast frames to a device. While the
4745 * count in the device remains above zero the interface remains listening
4746 * to all interfaces. Once it hits zero the device reverts back to normal
4747 * filtering operation. A negative @inc value is used to drop the counter
4748 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004749 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004750 */
4751
Wang Chendad9b332008-06-18 01:48:28 -07004752int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004753{
Eric Dumazetb536db92011-11-30 21:42:26 +00004754 unsigned int old_flags = dev->flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004755
Patrick McHardy24023452007-07-14 18:51:31 -07004756 ASSERT_RTNL();
4757
Linus Torvalds1da177e2005-04-16 15:20:36 -07004758 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004759 dev->allmulti += inc;
4760 if (dev->allmulti == 0) {
4761 /*
4762 * Avoid overflow.
4763 * If inc causes overflow, untouch allmulti and return error.
4764 */
4765 if (inc < 0)
4766 dev->flags &= ~IFF_ALLMULTI;
4767 else {
4768 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00004769 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
4770 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07004771 return -EOVERFLOW;
4772 }
4773 }
Patrick McHardy24023452007-07-14 18:51:31 -07004774 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004775 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004776 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004777 }
Wang Chendad9b332008-06-18 01:48:28 -07004778 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004779}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004780EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004781
4782/*
4783 * Upload unicast and multicast address lists to device and
4784 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004785 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004786 * are present.
4787 */
4788void __dev_set_rx_mode(struct net_device *dev)
4789{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004790 const struct net_device_ops *ops = dev->netdev_ops;
4791
Patrick McHardy4417da62007-06-27 01:28:10 -07004792 /* dev_open will call this function so the list will stay sane. */
4793 if (!(dev->flags&IFF_UP))
4794 return;
4795
4796 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004797 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004798
Jiri Pirko01789342011-08-16 06:29:00 +00004799 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004800 /* Unicast addresses changes may only happen under the rtnl,
4801 * therefore calling __dev_set_promiscuity here is safe.
4802 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004803 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004804 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004805 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004806 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004807 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004808 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004809 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004810 }
Jiri Pirko01789342011-08-16 06:29:00 +00004811
4812 if (ops->ndo_set_rx_mode)
4813 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004814}
4815
4816void dev_set_rx_mode(struct net_device *dev)
4817{
David S. Millerb9e40852008-07-15 00:15:08 -07004818 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004819 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004820 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004821}
4822
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004823/**
4824 * dev_get_flags - get flags reported to userspace
4825 * @dev: device
4826 *
4827 * Get the combination of flag bits exported through APIs to userspace.
4828 */
Eric Dumazet95c96172012-04-15 05:58:06 +00004829unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004830{
Eric Dumazet95c96172012-04-15 05:58:06 +00004831 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004832
4833 flags = (dev->flags & ~(IFF_PROMISC |
4834 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004835 IFF_RUNNING |
4836 IFF_LOWER_UP |
4837 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004838 (dev->gflags & (IFF_PROMISC |
4839 IFF_ALLMULTI));
4840
Stefan Rompfb00055a2006-03-20 17:09:11 -08004841 if (netif_running(dev)) {
4842 if (netif_oper_up(dev))
4843 flags |= IFF_RUNNING;
4844 if (netif_carrier_ok(dev))
4845 flags |= IFF_LOWER_UP;
4846 if (netif_dormant(dev))
4847 flags |= IFF_DORMANT;
4848 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004849
4850 return flags;
4851}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004852EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004853
Patrick McHardybd380812010-02-26 06:34:53 +00004854int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004855{
Eric Dumazetb536db92011-11-30 21:42:26 +00004856 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004857 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004858
Patrick McHardy24023452007-07-14 18:51:31 -07004859 ASSERT_RTNL();
4860
Linus Torvalds1da177e2005-04-16 15:20:36 -07004861 /*
4862 * Set the flags on our device.
4863 */
4864
4865 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4866 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4867 IFF_AUTOMEDIA)) |
4868 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4869 IFF_ALLMULTI));
4870
4871 /*
4872 * Load in the correct multicast list now the flags have changed.
4873 */
4874
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004875 if ((old_flags ^ flags) & IFF_MULTICAST)
4876 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004877
Patrick McHardy4417da62007-06-27 01:28:10 -07004878 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004879
4880 /*
4881 * Have we downed the interface. We handle IFF_UP ourselves
4882 * according to user attempts to set it, rather than blindly
4883 * setting it.
4884 */
4885
4886 ret = 0;
4887 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004888 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004889
4890 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004891 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004892 }
4893
Linus Torvalds1da177e2005-04-16 15:20:36 -07004894 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004895 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4896
Linus Torvalds1da177e2005-04-16 15:20:36 -07004897 dev->gflags ^= IFF_PROMISC;
4898 dev_set_promiscuity(dev, inc);
4899 }
4900
4901 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4902 is important. Some (broken) drivers set IFF_PROMISC, when
4903 IFF_ALLMULTI is requested not asking us and not reporting.
4904 */
4905 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004906 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4907
Linus Torvalds1da177e2005-04-16 15:20:36 -07004908 dev->gflags ^= IFF_ALLMULTI;
4909 dev_set_allmulti(dev, inc);
4910 }
4911
Patrick McHardybd380812010-02-26 06:34:53 +00004912 return ret;
4913}
4914
4915void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4916{
4917 unsigned int changes = dev->flags ^ old_flags;
4918
4919 if (changes & IFF_UP) {
4920 if (dev->flags & IFF_UP)
4921 call_netdevice_notifiers(NETDEV_UP, dev);
4922 else
4923 call_netdevice_notifiers(NETDEV_DOWN, dev);
4924 }
4925
4926 if (dev->flags & IFF_UP &&
4927 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4928 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4929}
4930
4931/**
4932 * dev_change_flags - change device settings
4933 * @dev: device
4934 * @flags: device state flags
4935 *
4936 * Change settings on device based state flags. The flags are
4937 * in the userspace exported format.
4938 */
Eric Dumazetb536db92011-11-30 21:42:26 +00004939int dev_change_flags(struct net_device *dev, unsigned int flags)
Patrick McHardybd380812010-02-26 06:34:53 +00004940{
Eric Dumazetb536db92011-11-30 21:42:26 +00004941 int ret;
4942 unsigned int changes, old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004943
4944 ret = __dev_change_flags(dev, flags);
4945 if (ret < 0)
4946 return ret;
4947
4948 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004949 if (changes)
4950 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004951
Patrick McHardybd380812010-02-26 06:34:53 +00004952 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004953 return ret;
4954}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004955EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004956
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004957/**
4958 * dev_set_mtu - Change maximum transfer unit
4959 * @dev: device
4960 * @new_mtu: new transfer unit
4961 *
4962 * Change the maximum transfer size of the network device.
4963 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004964int dev_set_mtu(struct net_device *dev, int new_mtu)
4965{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004966 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004967 int err;
4968
4969 if (new_mtu == dev->mtu)
4970 return 0;
4971
4972 /* MTU must be positive. */
4973 if (new_mtu < 0)
4974 return -EINVAL;
4975
4976 if (!netif_device_present(dev))
4977 return -ENODEV;
4978
4979 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004980 if (ops->ndo_change_mtu)
4981 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004982 else
4983 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004984
Jiri Pirkoe3d8fab2012-12-03 01:16:32 +00004985 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004986 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004987 return err;
4988}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004989EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004990
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004991/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004992 * dev_set_group - Change group this device belongs to
4993 * @dev: device
4994 * @new_group: group this device should belong to
4995 */
4996void dev_set_group(struct net_device *dev, int new_group)
4997{
4998 dev->group = new_group;
4999}
5000EXPORT_SYMBOL(dev_set_group);
5001
5002/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005003 * dev_set_mac_address - Change Media Access Control Address
5004 * @dev: device
5005 * @sa: new address
5006 *
5007 * Change the hardware (MAC) address of the device
5008 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005009int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
5010{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005011 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005012 int err;
5013
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005014 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005015 return -EOPNOTSUPP;
5016 if (sa->sa_family != dev->type)
5017 return -EINVAL;
5018 if (!netif_device_present(dev))
5019 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005020 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005021 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005022 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005023 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005024 return err;
5025}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005026EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005027
5028/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00005029 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07005030 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07005031static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005032{
5033 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00005034 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005035
5036 if (!dev)
5037 return -ENODEV;
5038
5039 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005040 case SIOCGIFFLAGS: /* Get interface flags */
5041 ifr->ifr_flags = (short) dev_get_flags(dev);
5042 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005043
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005044 case SIOCGIFMETRIC: /* Get the metric on the interface
5045 (currently unused) */
5046 ifr->ifr_metric = 0;
5047 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005048
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005049 case SIOCGIFMTU: /* Get the MTU of a device */
5050 ifr->ifr_mtu = dev->mtu;
5051 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005052
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005053 case SIOCGIFHWADDR:
5054 if (!dev->addr_len)
5055 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
5056 else
5057 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
5058 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
5059 ifr->ifr_hwaddr.sa_family = dev->type;
5060 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005061
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005062 case SIOCGIFSLAVE:
5063 err = -EINVAL;
5064 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005065
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005066 case SIOCGIFMAP:
5067 ifr->ifr_map.mem_start = dev->mem_start;
5068 ifr->ifr_map.mem_end = dev->mem_end;
5069 ifr->ifr_map.base_addr = dev->base_addr;
5070 ifr->ifr_map.irq = dev->irq;
5071 ifr->ifr_map.dma = dev->dma;
5072 ifr->ifr_map.port = dev->if_port;
5073 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005074
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005075 case SIOCGIFINDEX:
5076 ifr->ifr_ifindex = dev->ifindex;
5077 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005078
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005079 case SIOCGIFTXQLEN:
5080 ifr->ifr_qlen = dev->tx_queue_len;
5081 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005082
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005083 default:
5084 /* dev_ioctl() should ensure this case
5085 * is never reached
5086 */
5087 WARN_ON(1);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005088 err = -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005089 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005090
5091 }
5092 return err;
5093}
5094
5095/*
5096 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
5097 */
5098static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
5099{
5100 int err;
5101 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08005102 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005103
5104 if (!dev)
5105 return -ENODEV;
5106
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08005107 ops = dev->netdev_ops;
5108
Jeff Garzik14e3e072007-10-08 00:06:32 -07005109 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005110 case SIOCSIFFLAGS: /* Set interface flags */
5111 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07005112
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005113 case SIOCSIFMETRIC: /* Set the metric on the interface
5114 (currently unused) */
5115 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07005116
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005117 case SIOCSIFMTU: /* Set the MTU of a device */
5118 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07005119
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005120 case SIOCSIFHWADDR:
5121 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005122
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005123 case SIOCSIFHWBROADCAST:
5124 if (ifr->ifr_hwaddr.sa_family != dev->type)
5125 return -EINVAL;
5126 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
5127 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
5128 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
5129 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005130
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005131 case SIOCSIFMAP:
5132 if (ops->ndo_set_config) {
5133 if (!netif_device_present(dev))
5134 return -ENODEV;
5135 return ops->ndo_set_config(dev, &ifr->ifr_map);
5136 }
5137 return -EOPNOTSUPP;
5138
5139 case SIOCADDMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00005140 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005141 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
5142 return -EINVAL;
5143 if (!netif_device_present(dev))
5144 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00005145 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005146
5147 case SIOCDELMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00005148 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005149 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
5150 return -EINVAL;
5151 if (!netif_device_present(dev))
5152 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00005153 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005154
5155 case SIOCSIFTXQLEN:
5156 if (ifr->ifr_qlen < 0)
5157 return -EINVAL;
5158 dev->tx_queue_len = ifr->ifr_qlen;
5159 return 0;
5160
5161 case SIOCSIFNAME:
5162 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
5163 return dev_change_name(dev, ifr->ifr_newname);
5164
Richard Cochran4dc360c2011-10-19 17:00:35 -04005165 case SIOCSHWTSTAMP:
5166 err = net_hwtstamp_validate(ifr);
5167 if (err)
5168 return err;
5169 /* fall through */
5170
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005171 /*
5172 * Unknown or private ioctl
5173 */
5174 default:
5175 if ((cmd >= SIOCDEVPRIVATE &&
5176 cmd <= SIOCDEVPRIVATE + 15) ||
5177 cmd == SIOCBONDENSLAVE ||
5178 cmd == SIOCBONDRELEASE ||
5179 cmd == SIOCBONDSETHWADDR ||
5180 cmd == SIOCBONDSLAVEINFOQUERY ||
5181 cmd == SIOCBONDINFOQUERY ||
5182 cmd == SIOCBONDCHANGEACTIVE ||
5183 cmd == SIOCGMIIPHY ||
5184 cmd == SIOCGMIIREG ||
5185 cmd == SIOCSMIIREG ||
5186 cmd == SIOCBRADDIF ||
5187 cmd == SIOCBRDELIF ||
5188 cmd == SIOCSHWTSTAMP ||
5189 cmd == SIOCWANDEV) {
5190 err = -EOPNOTSUPP;
5191 if (ops->ndo_do_ioctl) {
5192 if (netif_device_present(dev))
5193 err = ops->ndo_do_ioctl(dev, ifr, cmd);
5194 else
5195 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005196 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005197 } else
5198 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005199
5200 }
5201 return err;
5202}
5203
5204/*
5205 * This function handles all "interface"-type I/O control requests. The actual
5206 * 'doing' part of this is dev_ifsioc above.
5207 */
5208
5209/**
5210 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005211 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005212 * @cmd: command to issue
5213 * @arg: pointer to a struct ifreq in user space
5214 *
5215 * Issue ioctl functions to devices. This is normally called by the
5216 * user space syscall interfaces but can sometimes be useful for
5217 * other purposes. The return value is the return from the syscall if
5218 * positive or a negative errno code on error.
5219 */
5220
Eric W. Biederman881d9662007-09-17 11:56:21 -07005221int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005222{
5223 struct ifreq ifr;
5224 int ret;
5225 char *colon;
5226
5227 /* One special case: SIOCGIFCONF takes ifconf argument
5228 and requires shared lock, because it sleeps writing
5229 to user space.
5230 */
5231
5232 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005233 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005234 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005235 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005236 return ret;
5237 }
5238 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005239 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005240
5241 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
5242 return -EFAULT;
5243
5244 ifr.ifr_name[IFNAMSIZ-1] = 0;
5245
5246 colon = strchr(ifr.ifr_name, ':');
5247 if (colon)
5248 *colon = 0;
5249
5250 /*
5251 * See which interface the caller is talking about.
5252 */
5253
5254 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005255 /*
5256 * These ioctl calls:
5257 * - can be done by all.
5258 * - atomic and do not require locking.
5259 * - return a value
5260 */
5261 case SIOCGIFFLAGS:
5262 case SIOCGIFMETRIC:
5263 case SIOCGIFMTU:
5264 case SIOCGIFHWADDR:
5265 case SIOCGIFSLAVE:
5266 case SIOCGIFMAP:
5267 case SIOCGIFINDEX:
5268 case SIOCGIFTXQLEN:
5269 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005270 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005271 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005272 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005273 if (!ret) {
5274 if (colon)
5275 *colon = ':';
5276 if (copy_to_user(arg, &ifr,
5277 sizeof(struct ifreq)))
5278 ret = -EFAULT;
5279 }
5280 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005281
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005282 case SIOCETHTOOL:
5283 dev_load(net, ifr.ifr_name);
5284 rtnl_lock();
5285 ret = dev_ethtool(net, &ifr);
5286 rtnl_unlock();
5287 if (!ret) {
5288 if (colon)
5289 *colon = ':';
5290 if (copy_to_user(arg, &ifr,
5291 sizeof(struct ifreq)))
5292 ret = -EFAULT;
5293 }
5294 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005295
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005296 /*
5297 * These ioctl calls:
5298 * - require superuser power.
5299 * - require strict serialization.
5300 * - return a value
5301 */
5302 case SIOCGMIIPHY:
5303 case SIOCGMIIREG:
5304 case SIOCSIFNAME:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +00005305 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005306 return -EPERM;
5307 dev_load(net, ifr.ifr_name);
5308 rtnl_lock();
5309 ret = dev_ifsioc(net, &ifr, cmd);
5310 rtnl_unlock();
5311 if (!ret) {
5312 if (colon)
5313 *colon = ':';
5314 if (copy_to_user(arg, &ifr,
5315 sizeof(struct ifreq)))
5316 ret = -EFAULT;
5317 }
5318 return ret;
5319
5320 /*
5321 * These ioctl calls:
5322 * - require superuser power.
5323 * - require strict serialization.
5324 * - do not return a value
5325 */
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +00005326 case SIOCSIFMAP:
5327 case SIOCSIFTXQLEN:
5328 if (!capable(CAP_NET_ADMIN))
5329 return -EPERM;
5330 /* fall through */
5331 /*
5332 * These ioctl calls:
5333 * - require local superuser power.
5334 * - require strict serialization.
5335 * - do not return a value
5336 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005337 case SIOCSIFFLAGS:
5338 case SIOCSIFMETRIC:
5339 case SIOCSIFMTU:
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005340 case SIOCSIFHWADDR:
5341 case SIOCSIFSLAVE:
5342 case SIOCADDMULTI:
5343 case SIOCDELMULTI:
5344 case SIOCSIFHWBROADCAST:
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005345 case SIOCSMIIREG:
5346 case SIOCBONDENSLAVE:
5347 case SIOCBONDRELEASE:
5348 case SIOCBONDSETHWADDR:
5349 case SIOCBONDCHANGEACTIVE:
5350 case SIOCBRADDIF:
5351 case SIOCBRDELIF:
5352 case SIOCSHWTSTAMP:
Eric W. Biederman5e1fccc2012-11-16 03:03:04 +00005353 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005354 return -EPERM;
5355 /* fall through */
5356 case SIOCBONDSLAVEINFOQUERY:
5357 case SIOCBONDINFOQUERY:
5358 dev_load(net, ifr.ifr_name);
5359 rtnl_lock();
5360 ret = dev_ifsioc(net, &ifr, cmd);
5361 rtnl_unlock();
5362 return ret;
5363
5364 case SIOCGIFMEM:
5365 /* Get the per device memory space. We can add this but
5366 * currently do not support it */
5367 case SIOCSIFMEM:
5368 /* Set the per device memory buffer space.
5369 * Not applicable in our case */
5370 case SIOCSIFLINK:
Lifeng Sun41c31f32011-04-27 22:04:51 +00005371 return -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005372
5373 /*
5374 * Unknown or private ioctl.
5375 */
5376 default:
5377 if (cmd == SIOCWANDEV ||
5378 (cmd >= SIOCDEVPRIVATE &&
5379 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005380 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005381 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005382 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005383 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005384 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005385 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005386 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005387 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005388 }
5389 /* Take care of Wireless Extensions */
5390 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5391 return wext_handle_ioctl(net, &ifr, cmd, arg);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005392 return -ENOTTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005393 }
5394}
5395
5396
5397/**
5398 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005399 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005400 *
5401 * Returns a suitable unique value for a new device interface
5402 * number. The caller must hold the rtnl semaphore or the
5403 * dev_base_lock to be sure it remains unique.
5404 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005405static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005406{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005407 int ifindex = net->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005408 for (;;) {
5409 if (++ifindex <= 0)
5410 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005411 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00005412 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005413 }
5414}
5415
Linus Torvalds1da177e2005-04-16 15:20:36 -07005416/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005417static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005418
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005419static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005420{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005421 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005422}
5423
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005424static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005425{
Krishna Kumare93737b2009-12-08 22:26:02 +00005426 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005427
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005428 BUG_ON(dev_boot_phase);
5429 ASSERT_RTNL();
5430
Krishna Kumare93737b2009-12-08 22:26:02 +00005431 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005432 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005433 * for initialization unwind. Remove those
5434 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005435 */
5436 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005437 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
5438 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005439
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005440 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005441 list_del(&dev->unreg_list);
5442 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005443 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005444 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005445 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005446 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005447
Octavian Purdila44345722010-12-13 12:44:07 +00005448 /* If device is running, close it first. */
5449 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005450
Octavian Purdila44345722010-12-13 12:44:07 +00005451 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005452 /* And unlink it from device chain. */
5453 unlist_netdevice(dev);
5454
5455 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005456 }
5457
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005458 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005459
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005460 list_for_each_entry(dev, head, unreg_list) {
5461 /* Shutdown queueing discipline. */
5462 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005463
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005464
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005465 /* Notify protocols, that we are about to destroy
5466 this device. They should clean all the things.
5467 */
5468 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5469
Patrick McHardya2835762010-02-26 06:34:51 +00005470 if (!dev->rtnl_link_ops ||
5471 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5472 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5473
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005474 /*
5475 * Flush the unicast and multicast chains
5476 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005477 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00005478 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005479
5480 if (dev->netdev_ops->ndo_uninit)
5481 dev->netdev_ops->ndo_uninit(dev);
5482
5483 /* Notifier chain MUST detach us from master device. */
5484 WARN_ON(dev->master);
5485
5486 /* Remove entries from kobject tree */
5487 netdev_unregister_kobject(dev);
5488 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005489
Eric W. Biederman850a5452011-10-13 22:25:23 +00005490 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005491
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005492 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005493 dev_put(dev);
5494}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005495
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005496static void rollback_registered(struct net_device *dev)
5497{
5498 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005499
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005500 list_add(&dev->unreg_list, &single);
5501 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005502 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005503}
5504
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005505static netdev_features_t netdev_fix_features(struct net_device *dev,
5506 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07005507{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005508 /* Fix illegal checksum combinations */
5509 if ((features & NETIF_F_HW_CSUM) &&
5510 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005511 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005512 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5513 }
5514
Herbert Xub63365a2008-10-23 01:11:29 -07005515 /* Fix illegal SG+CSUM combinations. */
5516 if ((features & NETIF_F_SG) &&
5517 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005518 netdev_dbg(dev,
5519 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005520 features &= ~NETIF_F_SG;
5521 }
5522
5523 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005524 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005525 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005526 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005527 }
5528
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005529 /* TSO ECN requires that TSO is present as well. */
5530 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5531 features &= ~NETIF_F_TSO_ECN;
5532
Michał Mirosław212b5732011-02-15 16:59:16 +00005533 /* Software GSO depends on SG. */
5534 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005535 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005536 features &= ~NETIF_F_GSO;
5537 }
5538
Michał Mirosławacd11302011-01-24 15:45:15 -08005539 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005540 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005541 /* maybe split UFO into V4 and V6? */
5542 if (!((features & NETIF_F_GEN_CSUM) ||
5543 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5544 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005545 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005546 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005547 features &= ~NETIF_F_UFO;
5548 }
5549
5550 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005551 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005552 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005553 features &= ~NETIF_F_UFO;
5554 }
5555 }
5556
5557 return features;
5558}
Herbert Xub63365a2008-10-23 01:11:29 -07005559
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005560int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005561{
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005562 netdev_features_t features;
Michał Mirosław5455c692011-02-15 16:59:17 +00005563 int err = 0;
5564
Michał Mirosław87267482011-04-12 09:56:38 +00005565 ASSERT_RTNL();
5566
Michał Mirosław5455c692011-02-15 16:59:17 +00005567 features = netdev_get_wanted_features(dev);
5568
5569 if (dev->netdev_ops->ndo_fix_features)
5570 features = dev->netdev_ops->ndo_fix_features(dev, features);
5571
5572 /* driver might be less strict about feature dependencies */
5573 features = netdev_fix_features(dev, features);
5574
5575 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005576 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005577
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005578 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5579 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00005580
5581 if (dev->netdev_ops->ndo_set_features)
5582 err = dev->netdev_ops->ndo_set_features(dev, features);
5583
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005584 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005585 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00005586 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5587 err, &features, &dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005588 return -1;
5589 }
5590
5591 if (!err)
5592 dev->features = features;
5593
5594 return 1;
5595}
5596
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005597/**
5598 * netdev_update_features - recalculate device features
5599 * @dev: the device to check
5600 *
5601 * Recalculate dev->features set and send notifications if it
5602 * has changed. Should be called after driver or hardware dependent
5603 * conditions might have changed that influence the features.
5604 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005605void netdev_update_features(struct net_device *dev)
5606{
5607 if (__netdev_update_features(dev))
5608 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005609}
5610EXPORT_SYMBOL(netdev_update_features);
5611
Linus Torvalds1da177e2005-04-16 15:20:36 -07005612/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005613 * netdev_change_features - recalculate device features
5614 * @dev: the device to check
5615 *
5616 * Recalculate dev->features set and send notifications even
5617 * if they have not changed. Should be called instead of
5618 * netdev_update_features() if also dev->vlan_features might
5619 * have changed to allow the changes to be propagated to stacked
5620 * VLAN devices.
5621 */
5622void netdev_change_features(struct net_device *dev)
5623{
5624 __netdev_update_features(dev);
5625 netdev_features_change(dev);
5626}
5627EXPORT_SYMBOL(netdev_change_features);
5628
5629/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005630 * netif_stacked_transfer_operstate - transfer operstate
5631 * @rootdev: the root or lower level device to transfer state from
5632 * @dev: the device to transfer operstate to
5633 *
5634 * Transfer operational state from root to device. This is normally
5635 * called when a stacking relationship exists between the root
5636 * device and the device(a leaf device).
5637 */
5638void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5639 struct net_device *dev)
5640{
5641 if (rootdev->operstate == IF_OPER_DORMANT)
5642 netif_dormant_on(dev);
5643 else
5644 netif_dormant_off(dev);
5645
5646 if (netif_carrier_ok(rootdev)) {
5647 if (!netif_carrier_ok(dev))
5648 netif_carrier_on(dev);
5649 } else {
5650 if (netif_carrier_ok(dev))
5651 netif_carrier_off(dev);
5652 }
5653}
5654EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5655
Tom Herbertbf264142010-11-26 08:36:09 +00005656#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005657static int netif_alloc_rx_queues(struct net_device *dev)
5658{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005659 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005660 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005661
Tom Herbertbd25fa72010-10-18 18:00:16 +00005662 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005663
Tom Herbertbd25fa72010-10-18 18:00:16 +00005664 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5665 if (!rx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005666 pr_err("netdev: Unable to allocate %u rx queues\n", count);
Tom Herbertbd25fa72010-10-18 18:00:16 +00005667 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005668 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005669 dev->_rx = rx;
5670
Tom Herbertbd25fa72010-10-18 18:00:16 +00005671 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005672 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005673 return 0;
5674}
Tom Herbertbf264142010-11-26 08:36:09 +00005675#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005676
Changli Gaoaa942102010-12-04 02:31:41 +00005677static void netdev_init_one_queue(struct net_device *dev,
5678 struct netdev_queue *queue, void *_unused)
5679{
5680 /* Initialize queue lock */
5681 spin_lock_init(&queue->_xmit_lock);
5682 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5683 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005684 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005685 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00005686#ifdef CONFIG_BQL
5687 dql_init(&queue->dql, HZ);
5688#endif
Changli Gaoaa942102010-12-04 02:31:41 +00005689}
5690
Tom Herberte6484932010-10-18 18:04:39 +00005691static int netif_alloc_netdev_queues(struct net_device *dev)
5692{
5693 unsigned int count = dev->num_tx_queues;
5694 struct netdev_queue *tx;
5695
5696 BUG_ON(count < 1);
5697
5698 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5699 if (!tx) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005700 pr_err("netdev: Unable to allocate %u tx queues\n", count);
Tom Herberte6484932010-10-18 18:04:39 +00005701 return -ENOMEM;
5702 }
5703 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005704
Tom Herberte6484932010-10-18 18:04:39 +00005705 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5706 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005707
5708 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005709}
5710
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005711/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005712 * register_netdevice - register a network device
5713 * @dev: device to register
5714 *
5715 * Take a completed network device structure and add it to the kernel
5716 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5717 * chain. 0 is returned on success. A negative errno code is returned
5718 * on a failure to set up the device, or if the name is a duplicate.
5719 *
5720 * Callers must hold the rtnl semaphore. You may want
5721 * register_netdev() instead of this.
5722 *
5723 * BUGS:
5724 * The locking appears insufficient to guarantee two parallel registers
5725 * will not get the same name.
5726 */
5727
5728int register_netdevice(struct net_device *dev)
5729{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005730 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005731 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005732
5733 BUG_ON(dev_boot_phase);
5734 ASSERT_RTNL();
5735
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005736 might_sleep();
5737
Linus Torvalds1da177e2005-04-16 15:20:36 -07005738 /* When net_device's are persistent, this will be fatal. */
5739 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005740 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005741
David S. Millerf1f28aa2008-07-15 00:08:33 -07005742 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005743 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005744
Linus Torvalds1da177e2005-04-16 15:20:36 -07005745 dev->iflink = -1;
5746
Gao feng828de4f2012-09-13 20:58:27 +00005747 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005748 if (ret < 0)
5749 goto out;
5750
Linus Torvalds1da177e2005-04-16 15:20:36 -07005751 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005752 if (dev->netdev_ops->ndo_init) {
5753 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005754 if (ret) {
5755 if (ret > 0)
5756 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005757 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005758 }
5759 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005760
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00005761 ret = -EBUSY;
5762 if (!dev->ifindex)
5763 dev->ifindex = dev_new_index(net);
5764 else if (__dev_get_by_index(net, dev->ifindex))
5765 goto err_uninit;
5766
Linus Torvalds1da177e2005-04-16 15:20:36 -07005767 if (dev->iflink == -1)
5768 dev->iflink = dev->ifindex;
5769
Michał Mirosław5455c692011-02-15 16:59:17 +00005770 /* Transfer changeable features to wanted_features and enable
5771 * software offloads (GSO and GRO).
5772 */
5773 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005774 dev->features |= NETIF_F_SOFT_FEATURES;
5775 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005776
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005777 /* Turn on no cache copy if HW is doing checksum */
Michał Mirosław34324dc2011-11-15 15:29:55 +00005778 if (!(dev->flags & IFF_LOOPBACK)) {
5779 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5780 if (dev->features & NETIF_F_ALL_CSUM) {
5781 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5782 dev->features |= NETIF_F_NOCACHE_COPY;
5783 }
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005784 }
5785
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005786 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005787 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005788 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005789
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005790 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5791 ret = notifier_to_errno(ret);
5792 if (ret)
5793 goto err_uninit;
5794
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005795 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005796 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005797 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005798 dev->reg_state = NETREG_REGISTERED;
5799
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005800 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005801
Linus Torvalds1da177e2005-04-16 15:20:36 -07005802 /*
5803 * Default initial state at registry is that the
5804 * device is present.
5805 */
5806
5807 set_bit(__LINK_STATE_PRESENT, &dev->state);
5808
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01005809 linkwatch_init_dev(dev);
5810
Linus Torvalds1da177e2005-04-16 15:20:36 -07005811 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005812 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005813 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04005814 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005815
5816 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005817 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005818 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005819 if (ret) {
5820 rollback_registered(dev);
5821 dev->reg_state = NETREG_UNREGISTERED;
5822 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005823 /*
5824 * Prevent userspace races by waiting until the network
5825 * device is fully setup before sending notifications.
5826 */
Patrick McHardya2835762010-02-26 06:34:51 +00005827 if (!dev->rtnl_link_ops ||
5828 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5829 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830
5831out:
5832 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005833
5834err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005835 if (dev->netdev_ops->ndo_uninit)
5836 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005837 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005838}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005839EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005840
5841/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005842 * init_dummy_netdev - init a dummy network device for NAPI
5843 * @dev: device to init
5844 *
5845 * This takes a network device structure and initialize the minimum
5846 * amount of fields so it can be used to schedule NAPI polls without
5847 * registering a full blown interface. This is to be used by drivers
5848 * that need to tie several hardware interfaces to a single NAPI
5849 * poll scheduler due to HW limitations.
5850 */
5851int init_dummy_netdev(struct net_device *dev)
5852{
5853 /* Clear everything. Note we don't initialize spinlocks
5854 * are they aren't supposed to be taken by any of the
5855 * NAPI code and this dummy netdev is supposed to be
5856 * only ever used for NAPI polls
5857 */
5858 memset(dev, 0, sizeof(struct net_device));
5859
5860 /* make sure we BUG if trying to hit standard
5861 * register/unregister code path
5862 */
5863 dev->reg_state = NETREG_DUMMY;
5864
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005865 /* NAPI wants this */
5866 INIT_LIST_HEAD(&dev->napi_list);
5867
5868 /* a dummy interface is started by default */
5869 set_bit(__LINK_STATE_PRESENT, &dev->state);
5870 set_bit(__LINK_STATE_START, &dev->state);
5871
Eric Dumazet29b44332010-10-11 10:22:12 +00005872 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5873 * because users of this 'device' dont need to change
5874 * its refcount.
5875 */
5876
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005877 return 0;
5878}
5879EXPORT_SYMBOL_GPL(init_dummy_netdev);
5880
5881
5882/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005883 * register_netdev - register a network device
5884 * @dev: device to register
5885 *
5886 * Take a completed network device structure and add it to the kernel
5887 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5888 * chain. 0 is returned on success. A negative errno code is returned
5889 * on a failure to set up the device, or if the name is a duplicate.
5890 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005891 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005892 * and expands the device name if you passed a format string to
5893 * alloc_netdev.
5894 */
5895int register_netdev(struct net_device *dev)
5896{
5897 int err;
5898
5899 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005900 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005901 rtnl_unlock();
5902 return err;
5903}
5904EXPORT_SYMBOL(register_netdev);
5905
Eric Dumazet29b44332010-10-11 10:22:12 +00005906int netdev_refcnt_read(const struct net_device *dev)
5907{
5908 int i, refcnt = 0;
5909
5910 for_each_possible_cpu(i)
5911 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5912 return refcnt;
5913}
5914EXPORT_SYMBOL(netdev_refcnt_read);
5915
Ben Hutchings2c530402012-07-10 10:55:09 +00005916/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005917 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00005918 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005919 *
5920 * This is called when unregistering network devices.
5921 *
5922 * Any protocol or device that holds a reference should register
5923 * for netdevice notification, and cleanup and put back the
5924 * reference if they receive an UNREGISTER event.
5925 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005926 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005927 */
5928static void netdev_wait_allrefs(struct net_device *dev)
5929{
5930 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005931 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005932
Eric Dumazete014deb2009-11-17 05:59:21 +00005933 linkwatch_forget_dev(dev);
5934
Linus Torvalds1da177e2005-04-16 15:20:36 -07005935 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005936 refcnt = netdev_refcnt_read(dev);
5937
5938 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005939 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005940 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005941
5942 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005943 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005944
Eric Dumazet748e2d92012-08-22 21:50:59 +00005945 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005946 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00005947 rtnl_lock();
5948
Eric Dumazet0115e8e2012-08-22 17:19:46 +00005949 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005950 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5951 &dev->state)) {
5952 /* We must not have linkwatch events
5953 * pending on unregister. If this
5954 * happens, we simply run the queue
5955 * unscheduled, resulting in a noop
5956 * for this device.
5957 */
5958 linkwatch_run_queue();
5959 }
5960
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005961 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005962
5963 rebroadcast_time = jiffies;
5964 }
5965
5966 msleep(250);
5967
Eric Dumazet29b44332010-10-11 10:22:12 +00005968 refcnt = netdev_refcnt_read(dev);
5969
Linus Torvalds1da177e2005-04-16 15:20:36 -07005970 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00005971 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
5972 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005973 warning_time = jiffies;
5974 }
5975 }
5976}
5977
5978/* The sequence is:
5979 *
5980 * rtnl_lock();
5981 * ...
5982 * register_netdevice(x1);
5983 * register_netdevice(x2);
5984 * ...
5985 * unregister_netdevice(y1);
5986 * unregister_netdevice(y2);
5987 * ...
5988 * rtnl_unlock();
5989 * free_netdev(y1);
5990 * free_netdev(y2);
5991 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005992 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005993 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005994 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005995 * without deadlocking with linkwatch via keventd.
5996 * 2) Since we run with the RTNL semaphore not held, we can sleep
5997 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005998 *
5999 * We must not return until all unregister events added during
6000 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006001 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006002void netdev_run_todo(void)
6003{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006004 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006005
Linus Torvalds1da177e2005-04-16 15:20:36 -07006006 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006007 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07006008
6009 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07006010
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006011
6012 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00006013 if (!list_empty(&list))
6014 rcu_barrier();
6015
Linus Torvalds1da177e2005-04-16 15:20:36 -07006016 while (!list_empty(&list)) {
6017 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00006018 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006019 list_del(&dev->todo_list);
6020
Eric Dumazet748e2d92012-08-22 21:50:59 +00006021 rtnl_lock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006022 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric Dumazet748e2d92012-08-22 21:50:59 +00006023 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00006024
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006025 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006026 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07006027 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006028 dump_stack();
6029 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006030 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006031
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006032 dev->reg_state = NETREG_UNREGISTERED;
6033
Changli Gao152102c2010-03-30 20:16:22 +00006034 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07006035
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006036 netdev_wait_allrefs(dev);
6037
6038 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00006039 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00006040 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6041 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07006042 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006043
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07006044 if (dev->destructor)
6045 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07006046
6047 /* Free network device */
6048 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006049 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006050}
6051
Ben Hutchings3cfde792010-07-09 09:11:52 +00006052/* Convert net_device_stats to rtnl_link_stats64. They have the same
6053 * fields in the same order, with only the type differing.
6054 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006055void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6056 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00006057{
6058#if BITS_PER_LONG == 64
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006059 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6060 memcpy(stats64, netdev_stats, sizeof(*stats64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00006061#else
6062 size_t i, n = sizeof(*stats64) / sizeof(u64);
6063 const unsigned long *src = (const unsigned long *)netdev_stats;
6064 u64 *dst = (u64 *)stats64;
6065
6066 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6067 sizeof(*stats64) / sizeof(u64));
6068 for (i = 0; i < n; i++)
6069 dst[i] = src[i];
6070#endif
6071}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00006072EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00006073
Eric Dumazetd83345a2009-11-16 03:36:51 +00006074/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006075 * dev_get_stats - get network device statistics
6076 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07006077 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006078 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00006079 * Get network statistics from device. Return @storage.
6080 * The device driver may provide its own method by setting
6081 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6082 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006083 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00006084struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6085 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00006086{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006087 const struct net_device_ops *ops = dev->netdev_ops;
6088
Eric Dumazet28172732010-07-07 14:58:56 -07006089 if (ops->ndo_get_stats64) {
6090 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006091 ops->ndo_get_stats64(dev, storage);
6092 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00006093 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006094 } else {
6095 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07006096 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00006097 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07006098 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07006099}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08006100EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07006101
Eric Dumazet24824a02010-10-02 06:11:55 +00006102struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07006103{
Eric Dumazet24824a02010-10-02 06:11:55 +00006104 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07006105
Eric Dumazet24824a02010-10-02 06:11:55 +00006106#ifdef CONFIG_NET_CLS_ACT
6107 if (queue)
6108 return queue;
6109 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
6110 if (!queue)
6111 return NULL;
6112 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00006113 queue->qdisc = &noop_qdisc;
6114 queue->qdisc_sleeping = &noop_qdisc;
6115 rcu_assign_pointer(dev->ingress_queue, queue);
6116#endif
6117 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07006118}
6119
Eric Dumazet2c60db02012-09-16 09:17:26 +00006120static const struct ethtool_ops default_ethtool_ops;
6121
Linus Torvalds1da177e2005-04-16 15:20:36 -07006122/**
Tom Herbert36909ea2011-01-09 19:36:31 +00006123 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07006124 * @sizeof_priv: size of private data to allocate space for
6125 * @name: device name format string
6126 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00006127 * @txqs: the number of TX subqueues to allocate
6128 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07006129 *
6130 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07006131 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00006132 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006133 */
Tom Herbert36909ea2011-01-09 19:36:31 +00006134struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
6135 void (*setup)(struct net_device *),
6136 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006137{
Linus Torvalds1da177e2005-04-16 15:20:36 -07006138 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07006139 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006140 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006141
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07006142 BUG_ON(strlen(name) >= sizeof(dev->name));
6143
Tom Herbert36909ea2011-01-09 19:36:31 +00006144 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006145 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00006146 return NULL;
6147 }
6148
Tom Herbert36909ea2011-01-09 19:36:31 +00006149#ifdef CONFIG_RPS
6150 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006151 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00006152 return NULL;
6153 }
6154#endif
6155
David S. Millerfd2ea0a2008-07-17 01:56:23 -07006156 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006157 if (sizeof_priv) {
6158 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006159 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006160 alloc_size += sizeof_priv;
6161 }
6162 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006163 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006164
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07006165 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006166 if (!p) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006167 pr_err("alloc_netdev: Unable to allocate device\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07006168 return NULL;
6169 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006170
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006171 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006172 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006173
Eric Dumazet29b44332010-10-11 10:22:12 +00006174 dev->pcpu_refcnt = alloc_percpu(int);
6175 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00006176 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006177
Linus Torvalds1da177e2005-04-16 15:20:36 -07006178 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006179 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006180
Jiri Pirko22bedad2010-04-01 21:22:57 +00006181 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006182 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006183
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006184 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006185
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006186 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00006187 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006188
Herbert Xud565b0a2008-12-15 23:38:52 -08006189 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006190 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006191 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07006192 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006193 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006194
6195 dev->num_tx_queues = txqs;
6196 dev->real_num_tx_queues = txqs;
6197 if (netif_alloc_netdev_queues(dev))
6198 goto free_all;
6199
6200#ifdef CONFIG_RPS
6201 dev->num_rx_queues = rxqs;
6202 dev->real_num_rx_queues = rxqs;
6203 if (netif_alloc_rx_queues(dev))
6204 goto free_all;
6205#endif
6206
Linus Torvalds1da177e2005-04-16 15:20:36 -07006207 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006208 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00006209 if (!dev->ethtool_ops)
6210 dev->ethtool_ops = &default_ethtool_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006211 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006212
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006213free_all:
6214 free_netdev(dev);
6215 return NULL;
6216
Eric Dumazet29b44332010-10-11 10:22:12 +00006217free_pcpu:
6218 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00006219 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006220#ifdef CONFIG_RPS
6221 kfree(dev->_rx);
6222#endif
6223
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006224free_p:
6225 kfree(p);
6226 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006227}
Tom Herbert36909ea2011-01-09 19:36:31 +00006228EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006229
6230/**
6231 * free_netdev - free network device
6232 * @dev: device
6233 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006234 * This function does the last stage of destroying an allocated device
6235 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006236 * If this is the last reference then it will be freed.
6237 */
6238void free_netdev(struct net_device *dev)
6239{
Herbert Xud565b0a2008-12-15 23:38:52 -08006240 struct napi_struct *p, *n;
6241
Denis V. Lunevf3005d72008-04-16 02:02:18 -07006242 release_net(dev_net(dev));
6243
David S. Millere8a04642008-07-17 00:34:19 -07006244 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006245#ifdef CONFIG_RPS
6246 kfree(dev->_rx);
6247#endif
David S. Millere8a04642008-07-17 00:34:19 -07006248
Eric Dumazet33d480c2011-08-11 19:30:52 +00006249 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006250
Jiri Pirkof001fde2009-05-05 02:48:28 +00006251 /* Flush device addresses */
6252 dev_addr_flush(dev);
6253
Herbert Xud565b0a2008-12-15 23:38:52 -08006254 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6255 netif_napi_del(p);
6256
Eric Dumazet29b44332010-10-11 10:22:12 +00006257 free_percpu(dev->pcpu_refcnt);
6258 dev->pcpu_refcnt = NULL;
6259
Stephen Hemminger3041a062006-05-26 13:25:24 -07006260 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006261 if (dev->reg_state == NETREG_UNINITIALIZED) {
6262 kfree((char *)dev - dev->padded);
6263 return;
6264 }
6265
6266 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6267 dev->reg_state = NETREG_RELEASED;
6268
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006269 /* will free via device release */
6270 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006271}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006272EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006273
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006274/**
6275 * synchronize_net - Synchronize with packet receive processing
6276 *
6277 * Wait for packets currently being received to be done.
6278 * Does not block later packets from starting.
6279 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006280void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006281{
6282 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006283 if (rtnl_is_locked())
6284 synchronize_rcu_expedited();
6285 else
6286 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006287}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006288EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006289
6290/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006291 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006292 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006293 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006294 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006295 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006296 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006297 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006298 *
6299 * Callers must hold the rtnl semaphore. You may want
6300 * unregister_netdev() instead of this.
6301 */
6302
Eric Dumazet44a08732009-10-27 07:03:04 +00006303void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006304{
Herbert Xua6620712007-12-12 19:21:56 -08006305 ASSERT_RTNL();
6306
Eric Dumazet44a08732009-10-27 07:03:04 +00006307 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006308 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006309 } else {
6310 rollback_registered(dev);
6311 /* Finish processing unregister after unlock */
6312 net_set_todo(dev);
6313 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006314}
Eric Dumazet44a08732009-10-27 07:03:04 +00006315EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006316
6317/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006318 * unregister_netdevice_many - unregister many devices
6319 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006320 */
6321void unregister_netdevice_many(struct list_head *head)
6322{
6323 struct net_device *dev;
6324
6325 if (!list_empty(head)) {
6326 rollback_registered_many(head);
6327 list_for_each_entry(dev, head, unreg_list)
6328 net_set_todo(dev);
6329 }
6330}
Eric Dumazet63c80992009-10-27 07:06:49 +00006331EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006332
6333/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006334 * unregister_netdev - remove device from the kernel
6335 * @dev: device
6336 *
6337 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006338 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006339 *
6340 * This is just a wrapper for unregister_netdevice that takes
6341 * the rtnl semaphore. In general you want to use this and not
6342 * unregister_netdevice.
6343 */
6344void unregister_netdev(struct net_device *dev)
6345{
6346 rtnl_lock();
6347 unregister_netdevice(dev);
6348 rtnl_unlock();
6349}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006350EXPORT_SYMBOL(unregister_netdev);
6351
Eric W. Biedermance286d32007-09-12 13:53:49 +02006352/**
6353 * dev_change_net_namespace - move device to different nethost namespace
6354 * @dev: device
6355 * @net: network namespace
6356 * @pat: If not NULL name pattern to try if the current device name
6357 * is already taken in the destination network namespace.
6358 *
6359 * This function shuts down a device interface and moves it
6360 * to a new network namespace. On success 0 is returned, on
6361 * a failure a netagive errno code is returned.
6362 *
6363 * Callers must hold the rtnl semaphore.
6364 */
6365
6366int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6367{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006368 int err;
6369
6370 ASSERT_RTNL();
6371
6372 /* Don't allow namespace local devices to be moved. */
6373 err = -EINVAL;
6374 if (dev->features & NETIF_F_NETNS_LOCAL)
6375 goto out;
6376
6377 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02006378 if (dev->reg_state != NETREG_REGISTERED)
6379 goto out;
6380
6381 /* Get out if there is nothing todo */
6382 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006383 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006384 goto out;
6385
6386 /* Pick the destination device name, and ensure
6387 * we can use it in the destination network namespace.
6388 */
6389 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006390 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006391 /* We get here if we can't use the current device name */
6392 if (!pat)
6393 goto out;
Gao feng828de4f2012-09-13 20:58:27 +00006394 if (dev_get_valid_name(net, dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006395 goto out;
6396 }
6397
6398 /*
6399 * And now a mini version of register_netdevice unregister_netdevice.
6400 */
6401
6402 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006403 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006404
6405 /* And unlink it from device chain */
6406 err = -ENODEV;
6407 unlist_netdevice(dev);
6408
6409 synchronize_net();
6410
6411 /* Shutdown queueing discipline. */
6412 dev_shutdown(dev);
6413
6414 /* Notify protocols, that we are about to destroy
6415 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006416
6417 Note that dev->reg_state stays at NETREG_REGISTERED.
6418 This is wanted because this way 8021q and macvlan know
6419 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006420 */
6421 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00006422 rcu_barrier();
6423 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00006424 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006425
6426 /*
6427 * Flush the unicast and multicast chains
6428 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006429 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00006430 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006431
Serge Hallyn4e66ae22012-12-03 16:17:12 +00006432 /* Send a netdev-removed uevent to the old namespace */
6433 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
6434
Eric W. Biedermance286d32007-09-12 13:53:49 +02006435 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006436 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006437
Eric W. Biedermance286d32007-09-12 13:53:49 +02006438 /* If there is an ifindex conflict assign a new one */
6439 if (__dev_get_by_index(net, dev->ifindex)) {
6440 int iflink = (dev->iflink == dev->ifindex);
6441 dev->ifindex = dev_new_index(net);
6442 if (iflink)
6443 dev->iflink = dev->ifindex;
6444 }
6445
Serge Hallyn4e66ae22012-12-03 16:17:12 +00006446 /* Send a netdev-add uevent to the new namespace */
6447 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
6448
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006449 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006450 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006451 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006452
6453 /* Add the device back in the hashes */
6454 list_netdevice(dev);
6455
6456 /* Notify protocols, that a new device appeared. */
6457 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6458
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006459 /*
6460 * Prevent userspace races by waiting until the network
6461 * device is fully setup before sending notifications.
6462 */
6463 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6464
Eric W. Biedermance286d32007-09-12 13:53:49 +02006465 synchronize_net();
6466 err = 0;
6467out:
6468 return err;
6469}
Johannes Berg463d0182009-07-14 00:33:35 +02006470EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006471
Linus Torvalds1da177e2005-04-16 15:20:36 -07006472static int dev_cpu_callback(struct notifier_block *nfb,
6473 unsigned long action,
6474 void *ocpu)
6475{
6476 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006477 struct sk_buff *skb;
6478 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6479 struct softnet_data *sd, *oldsd;
6480
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006481 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006482 return NOTIFY_OK;
6483
6484 local_irq_disable();
6485 cpu = smp_processor_id();
6486 sd = &per_cpu(softnet_data, cpu);
6487 oldsd = &per_cpu(softnet_data, oldcpu);
6488
6489 /* Find end of our completion_queue. */
6490 list_skb = &sd->completion_queue;
6491 while (*list_skb)
6492 list_skb = &(*list_skb)->next;
6493 /* Append completion queue from offline CPU. */
6494 *list_skb = oldsd->completion_queue;
6495 oldsd->completion_queue = NULL;
6496
Linus Torvalds1da177e2005-04-16 15:20:36 -07006497 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006498 if (oldsd->output_queue) {
6499 *sd->output_queue_tailp = oldsd->output_queue;
6500 sd->output_queue_tailp = oldsd->output_queue_tailp;
6501 oldsd->output_queue = NULL;
6502 oldsd->output_queue_tailp = &oldsd->output_queue;
6503 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006504 /* Append NAPI poll list from offline CPU. */
6505 if (!list_empty(&oldsd->poll_list)) {
6506 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6507 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6508 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006509
6510 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6511 local_irq_enable();
6512
6513 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006514 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6515 netif_rx(skb);
6516 input_queue_head_incr(oldsd);
6517 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006518 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006519 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006520 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006521 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006522
6523 return NOTIFY_OK;
6524}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006525
6526
Herbert Xu7f353bf2007-08-10 15:47:58 -07006527/**
Herbert Xub63365a2008-10-23 01:11:29 -07006528 * netdev_increment_features - increment feature set by one
6529 * @all: current feature set
6530 * @one: new feature set
6531 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006532 *
6533 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006534 * @one to the master device with current feature set @all. Will not
6535 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006536 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00006537netdev_features_t netdev_increment_features(netdev_features_t all,
6538 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006539{
Michał Mirosław1742f182011-04-22 06:31:16 +00006540 if (mask & NETIF_F_GEN_CSUM)
6541 mask |= NETIF_F_ALL_CSUM;
6542 mask |= NETIF_F_VLAN_CHALLENGED;
6543
6544 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6545 all &= one | ~NETIF_F_ALL_FOR_ALL;
6546
Michał Mirosław1742f182011-04-22 06:31:16 +00006547 /* If one device supports hw checksumming, set for all. */
6548 if (all & NETIF_F_GEN_CSUM)
6549 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006550
6551 return all;
6552}
Herbert Xub63365a2008-10-23 01:11:29 -07006553EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006554
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006555static struct hlist_head *netdev_create_hash(void)
6556{
6557 int i;
6558 struct hlist_head *hash;
6559
6560 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6561 if (hash != NULL)
6562 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6563 INIT_HLIST_HEAD(&hash[i]);
6564
6565 return hash;
6566}
6567
Eric W. Biederman881d9662007-09-17 11:56:21 -07006568/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006569static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006570{
Rustad, Mark D734b6542012-07-18 09:06:07 +00006571 if (net != &init_net)
6572 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006573
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006574 net->dev_name_head = netdev_create_hash();
6575 if (net->dev_name_head == NULL)
6576 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006577
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006578 net->dev_index_head = netdev_create_hash();
6579 if (net->dev_index_head == NULL)
6580 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006581
6582 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006583
6584err_idx:
6585 kfree(net->dev_name_head);
6586err_name:
6587 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006588}
6589
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006590/**
6591 * netdev_drivername - network driver for the device
6592 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006593 *
6594 * Determine network driver for device.
6595 */
David S. Miller3019de12011-06-06 16:41:33 -07006596const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006597{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006598 const struct device_driver *driver;
6599 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006600 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006601
6602 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006603 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006604 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006605
6606 driver = parent->driver;
6607 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006608 return driver->name;
6609 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006610}
6611
Joe Perchesb004ff42012-09-12 20:12:19 -07006612static int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006613 struct va_format *vaf)
6614{
6615 int r;
6616
Joe Perchesb004ff42012-09-12 20:12:19 -07006617 if (dev && dev->dev.parent) {
Joe Perches666f3552012-09-12 20:14:11 -07006618 r = dev_printk_emit(level[1] - '0',
6619 dev->dev.parent,
6620 "%s %s %s: %pV",
6621 dev_driver_string(dev->dev.parent),
6622 dev_name(dev->dev.parent),
6623 netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006624 } else if (dev) {
Joe Perches256df2f2010-06-27 01:02:35 +00006625 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006626 } else {
Joe Perches256df2f2010-06-27 01:02:35 +00006627 r = printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006628 }
Joe Perches256df2f2010-06-27 01:02:35 +00006629
6630 return r;
6631}
6632
6633int netdev_printk(const char *level, const struct net_device *dev,
6634 const char *format, ...)
6635{
6636 struct va_format vaf;
6637 va_list args;
6638 int r;
6639
6640 va_start(args, format);
6641
6642 vaf.fmt = format;
6643 vaf.va = &args;
6644
6645 r = __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07006646
Joe Perches256df2f2010-06-27 01:02:35 +00006647 va_end(args);
6648
6649 return r;
6650}
6651EXPORT_SYMBOL(netdev_printk);
6652
6653#define define_netdev_printk_level(func, level) \
6654int func(const struct net_device *dev, const char *fmt, ...) \
6655{ \
6656 int r; \
6657 struct va_format vaf; \
6658 va_list args; \
6659 \
6660 va_start(args, fmt); \
6661 \
6662 vaf.fmt = fmt; \
6663 vaf.va = &args; \
6664 \
6665 r = __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07006666 \
Joe Perches256df2f2010-06-27 01:02:35 +00006667 va_end(args); \
6668 \
6669 return r; \
6670} \
6671EXPORT_SYMBOL(func);
6672
6673define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6674define_netdev_printk_level(netdev_alert, KERN_ALERT);
6675define_netdev_printk_level(netdev_crit, KERN_CRIT);
6676define_netdev_printk_level(netdev_err, KERN_ERR);
6677define_netdev_printk_level(netdev_warn, KERN_WARNING);
6678define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6679define_netdev_printk_level(netdev_info, KERN_INFO);
6680
Pavel Emelyanov46650792007-10-08 20:38:39 -07006681static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006682{
6683 kfree(net->dev_name_head);
6684 kfree(net->dev_index_head);
6685}
6686
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006687static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006688 .init = netdev_init,
6689 .exit = netdev_exit,
6690};
6691
Pavel Emelyanov46650792007-10-08 20:38:39 -07006692static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006693{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006694 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006695 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006696 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006697 * initial network namespace
6698 */
6699 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006700 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006701 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006702 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006703
6704 /* Ignore unmoveable devices (i.e. loopback) */
6705 if (dev->features & NETIF_F_NETNS_LOCAL)
6706 continue;
6707
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006708 /* Leave virtual devices for the generic cleanup */
6709 if (dev->rtnl_link_ops)
6710 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006711
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006712 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006713 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6714 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006715 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00006716 pr_emerg("%s: failed to move %s to init_net: %d\n",
6717 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006718 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006719 }
6720 }
6721 rtnl_unlock();
6722}
6723
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006724static void __net_exit default_device_exit_batch(struct list_head *net_list)
6725{
6726 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006727 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006728 * Do this across as many network namespaces as possible to
6729 * improve batching efficiency.
6730 */
6731 struct net_device *dev;
6732 struct net *net;
6733 LIST_HEAD(dev_kill_list);
6734
6735 rtnl_lock();
6736 list_for_each_entry(net, net_list, exit_list) {
6737 for_each_netdev_reverse(net, dev) {
6738 if (dev->rtnl_link_ops)
6739 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6740 else
6741 unregister_netdevice_queue(dev, &dev_kill_list);
6742 }
6743 }
6744 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006745 list_del(&dev_kill_list);
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006746 rtnl_unlock();
6747}
6748
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006749static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006750 .exit = default_device_exit,
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006751 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006752};
6753
Linus Torvalds1da177e2005-04-16 15:20:36 -07006754/*
6755 * Initialize the DEV module. At boot time this walks the device list and
6756 * unhooks any devices that fail to initialise (normally hardware not
6757 * present) and leaves us with a valid list of present and active devices.
6758 *
6759 */
6760
6761/*
6762 * This is called single threaded during boot, so no need
6763 * to take the rtnl semaphore.
6764 */
6765static int __init net_dev_init(void)
6766{
6767 int i, rc = -ENOMEM;
6768
6769 BUG_ON(!dev_boot_phase);
6770
Linus Torvalds1da177e2005-04-16 15:20:36 -07006771 if (dev_proc_init())
6772 goto out;
6773
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006774 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006775 goto out;
6776
6777 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006778 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006779 INIT_LIST_HEAD(&ptype_base[i]);
6780
Vlad Yasevich62532da2012-11-15 08:49:10 +00006781 INIT_LIST_HEAD(&offload_base);
6782
Eric W. Biederman881d9662007-09-17 11:56:21 -07006783 if (register_pernet_subsys(&netdev_net_ops))
6784 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006785
6786 /*
6787 * Initialise the packet receive queues.
6788 */
6789
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006790 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006791 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006792
Changli Gaodee42872010-05-02 05:42:16 +00006793 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006794 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006795 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006796 sd->completion_queue = NULL;
6797 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006798 sd->output_queue = NULL;
6799 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006800#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006801 sd->csd.func = rps_trigger_softirq;
6802 sd->csd.info = sd;
6803 sd->csd.flags = 0;
6804 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006805#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006806
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006807 sd->backlog.poll = process_backlog;
6808 sd->backlog.weight = weight_p;
6809 sd->backlog.gro_list = NULL;
6810 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006811 }
6812
Linus Torvalds1da177e2005-04-16 15:20:36 -07006813 dev_boot_phase = 0;
6814
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006815 /* The loopback device is special if any other network devices
6816 * is present in a network namespace the loopback device must
6817 * be present. Since we now dynamically allocate and free the
6818 * loopback device ensure this invariant is maintained by
6819 * keeping the loopback device as the first device on the
6820 * list of network devices. Ensuring the loopback devices
6821 * is the first device that appears and the last network device
6822 * that disappears.
6823 */
6824 if (register_pernet_device(&loopback_net_ops))
6825 goto out;
6826
6827 if (register_pernet_device(&default_device_ops))
6828 goto out;
6829
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006830 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6831 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006832
6833 hotcpu_notifier(dev_cpu_callback, 0);
6834 dst_init();
6835 dev_mcast_init();
6836 rc = 0;
6837out:
6838 return rc;
6839}
6840
6841subsys_initcall(net_dev_init);
6842
Krishna Kumare88721f2009-02-18 17:55:02 -08006843static int __init initialize_hashrnd(void)
6844{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006845 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006846 return 0;
6847}
6848
6849late_initcall_sync(initialize_hashrnd);
6850