blob: 51f89cd0a3f4c9a33b889c0d62c5f1b2abf7c27f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080078#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000082#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090083#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/notifier.h>
97#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020098#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099#include <net/sock.h>
100#include <linux/rtnetlink.h>
101#include <linux/proc_fs.h>
102#include <linux/seq_file.h>
103#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/dst.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000107#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <linux/highmem.h>
109#include <linux/init.h>
110#include <linux/kmod.h>
111#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/netpoll.h>
113#include <linux/rcupdate.h>
114#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700115#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700126#include <linux/ipv6.h>
127#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700128#include <linux/jhash.h>
129#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700130#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900131#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900132#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000133#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700134#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000135#include <linux/cpu_rmap.h>
Tom Herberte971b722011-08-14 19:46:12 +0000136#include <linux/if_tunnel.h>
Changli Gaoae1511b2011-08-18 23:23:47 -0700137#include <linux/if_pppox.h>
Changli Gao5dd17e02011-09-20 22:36:07 +0000138#include <linux/ppp_defs.h>
Richard Cochran4dc360c2011-10-19 17:00:35 -0400139#include <linux/net_tstamp.h>
Eric Dumazet588f0332011-11-15 04:12:55 +0000140#include <linux/jump_label.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700142#include "net-sysfs.h"
143
Herbert Xud565b0a2008-12-15 23:38:52 -0800144/* Instead of increasing this, you should create a hash table. */
145#define MAX_GRO_SKBS 8
146
Herbert Xu5d38a072009-01-04 16:13:40 -0800147/* This should be increased if a protocol with a bigger head is added. */
148#define GRO_MAX_HEAD (MAX_HEADER + 128)
149
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150/*
151 * The list of packet types we will receive (as opposed to discard)
152 * and the routines to invoke.
153 *
154 * Why 16. Because with 16 the only overlap we get on a hash of the
155 * low nibble of the protocol value is RARP/SNAP/X.25.
156 *
157 * NOTE: That is no longer true with the addition of VLAN tags. Not
158 * sure which should go first, but I bet it won't make much
159 * difference if we are running VLANs. The good news is that
160 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700161 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 * --BLG
163 *
164 * 0800 IP
165 * 8100 802.1Q VLAN
166 * 0001 802.3
167 * 0002 AX.25
168 * 0004 802.2
169 * 8035 RARP
170 * 0005 SNAP
171 * 0805 X.25
172 * 0806 ARP
173 * 8137 IPX
174 * 0009 Localtalk
175 * 86DD IPv6
176 */
177
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800178#define PTYPE_HASH_SIZE (16)
179#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
180
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800182static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700183static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700186 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 * semaphore.
188 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800189 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 *
191 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700192 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 * actual updates. This allows pure readers to access the list even
194 * while a writer is preparing to update it.
195 *
196 * To put it another way, dev_base_lock is held for writing only to
197 * protect against pure readers; the rtnl semaphore provides the
198 * protection against other writers.
199 *
200 * See, for example usages, register_netdevice() and
201 * unregister_netdevice(), which must be called with the rtnl
202 * semaphore held.
203 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205EXPORT_SYMBOL(dev_base_lock);
206
Thomas Graf4e985ad2011-06-21 03:11:20 +0000207static inline void dev_base_seq_inc(struct net *net)
208{
209 while (++net->dev_base_seq == 0);
210}
211
Eric W. Biederman881d9662007-09-17 11:56:21 -0700212static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213{
214 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
stephen hemminger08e98972009-11-10 07:20:34 +0000215 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216}
217
Eric W. Biederman881d9662007-09-17 11:56:21 -0700218static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700220 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221}
222
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000223static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000224{
225#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000226 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000227#endif
228}
229
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000230static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000231{
232#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000233 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000234#endif
235}
236
Eric W. Biedermance286d32007-09-12 13:53:49 +0200237/* Device list insertion */
238static int list_netdevice(struct net_device *dev)
239{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900240 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200241
242 ASSERT_RTNL();
243
244 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800245 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000246 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000247 hlist_add_head_rcu(&dev->index_hlist,
248 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200249 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000250
251 dev_base_seq_inc(net);
252
Eric W. Biedermance286d32007-09-12 13:53:49 +0200253 return 0;
254}
255
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000256/* Device list removal
257 * caller must respect a RCU grace period before freeing/reusing dev
258 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200259static void unlist_netdevice(struct net_device *dev)
260{
261 ASSERT_RTNL();
262
263 /* Unlink dev from the device chain */
264 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800265 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000266 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000267 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200268 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000269
270 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200271}
272
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273/*
274 * Our notifier list
275 */
276
Alan Sternf07d5b92006-05-09 15:23:03 -0700277static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278
279/*
280 * Device drivers call our routines to queue packets here. We empty the
281 * queue in the local softnet handler.
282 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700283
Eric Dumazet9958da02010-04-17 04:17:02 +0000284DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700285EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286
David S. Millercf508b12008-07-22 14:16:42 -0700287#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700288/*
David S. Millerc773e842008-07-08 23:13:53 -0700289 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700290 * according to dev->type
291 */
292static const unsigned short netdev_lock_type[] =
293 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
294 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
295 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
296 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
297 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
298 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
299 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
300 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
301 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
302 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
303 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
304 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
305 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800306 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
Dmitry Eremin-Solenikov929122c2009-08-14 20:00:20 +0400307 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000308 ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700309
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700310static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700311 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
312 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
313 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
314 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
315 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
316 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
317 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
318 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
319 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
320 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
321 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
322 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
323 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800324 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
Dmitry Eremin-Solenikov929122c2009-08-14 20:00:20 +0400325 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000326 "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700327
328static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700329static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700330
331static inline unsigned short netdev_lock_pos(unsigned short dev_type)
332{
333 int i;
334
335 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
336 if (netdev_lock_type[i] == dev_type)
337 return i;
338 /* the last key is used by default */
339 return ARRAY_SIZE(netdev_lock_type) - 1;
340}
341
David S. Millercf508b12008-07-22 14:16:42 -0700342static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
343 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700344{
345 int i;
346
347 i = netdev_lock_pos(dev_type);
348 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
349 netdev_lock_name[i]);
350}
David S. Millercf508b12008-07-22 14:16:42 -0700351
352static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
353{
354 int i;
355
356 i = netdev_lock_pos(dev->type);
357 lockdep_set_class_and_name(&dev->addr_list_lock,
358 &netdev_addr_lock_key[i],
359 netdev_lock_name[i]);
360}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700361#else
David S. Millercf508b12008-07-22 14:16:42 -0700362static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
363 unsigned short dev_type)
364{
365}
366static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700367{
368}
369#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
371/*******************************************************************************
372
373 Protocol management and registration routines
374
375*******************************************************************************/
376
377/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 * Add a protocol ID to the list. Now that the input handler is
379 * smarter we can dispense with all the messy stuff that used to be
380 * here.
381 *
382 * BEWARE!!! Protocol handlers, mangling input packets,
383 * MUST BE last in hash buckets and checking protocol handlers
384 * MUST start from promiscuous ptype_all chain in net_bh.
385 * It is true now, do not change it.
386 * Explanation follows: if protocol handler, mangling packet, will
387 * be the first on list, it is not able to sense, that packet
388 * is cloned and should be copied-on-write, so that it will
389 * change it and subsequent readers will get broken packet.
390 * --ANK (980803)
391 */
392
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000393static inline struct list_head *ptype_head(const struct packet_type *pt)
394{
395 if (pt->type == htons(ETH_P_ALL))
396 return &ptype_all;
397 else
398 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
399}
400
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401/**
402 * dev_add_pack - add packet handler
403 * @pt: packet type declaration
404 *
405 * Add a protocol handler to the networking stack. The passed &packet_type
406 * is linked into kernel lists and may not be freed until it has been
407 * removed from the kernel lists.
408 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900409 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 * guarantee all CPU's that are in middle of receiving packets
411 * will see the new packet type (until the next received packet).
412 */
413
414void dev_add_pack(struct packet_type *pt)
415{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000416 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000418 spin_lock(&ptype_lock);
419 list_add_rcu(&pt->list, head);
420 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700422EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424/**
425 * __dev_remove_pack - remove packet handler
426 * @pt: packet type declaration
427 *
428 * Remove a protocol handler that was previously added to the kernel
429 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
430 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900431 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 *
433 * The packet type might still be in use by receivers
434 * and must not be freed until after all the CPU's have gone
435 * through a quiescent state.
436 */
437void __dev_remove_pack(struct packet_type *pt)
438{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000439 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 struct packet_type *pt1;
441
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000442 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
444 list_for_each_entry(pt1, head, list) {
445 if (pt == pt1) {
446 list_del_rcu(&pt->list);
447 goto out;
448 }
449 }
450
451 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
452out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000453 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700455EXPORT_SYMBOL(__dev_remove_pack);
456
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457/**
458 * dev_remove_pack - remove packet handler
459 * @pt: packet type declaration
460 *
461 * Remove a protocol handler that was previously added to the kernel
462 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
463 * from the kernel lists and can be freed or reused once this function
464 * returns.
465 *
466 * This call sleeps to guarantee that no CPU is looking at the packet
467 * type after return.
468 */
469void dev_remove_pack(struct packet_type *pt)
470{
471 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900472
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 synchronize_net();
474}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700475EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
477/******************************************************************************
478
479 Device Boot-time Settings Routines
480
481*******************************************************************************/
482
483/* Boot time configuration table */
484static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
485
486/**
487 * netdev_boot_setup_add - add new setup entry
488 * @name: name of the device
489 * @map: configured settings for the device
490 *
491 * Adds new setup entry to the dev_boot_setup list. The function
492 * returns 0 on error and 1 on success. This is a generic routine to
493 * all netdevices.
494 */
495static int netdev_boot_setup_add(char *name, struct ifmap *map)
496{
497 struct netdev_boot_setup *s;
498 int i;
499
500 s = dev_boot_setup;
501 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
502 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
503 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700504 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 memcpy(&s[i].map, map, sizeof(s[i].map));
506 break;
507 }
508 }
509
510 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
511}
512
513/**
514 * netdev_boot_setup_check - check boot time settings
515 * @dev: the netdevice
516 *
517 * Check boot time settings for the device.
518 * The found settings are set for the device to be used
519 * later in the device probing.
520 * Returns 0 if no settings found, 1 if they are.
521 */
522int netdev_boot_setup_check(struct net_device *dev)
523{
524 struct netdev_boot_setup *s = dev_boot_setup;
525 int i;
526
527 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
528 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700529 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 dev->irq = s[i].map.irq;
531 dev->base_addr = s[i].map.base_addr;
532 dev->mem_start = s[i].map.mem_start;
533 dev->mem_end = s[i].map.mem_end;
534 return 1;
535 }
536 }
537 return 0;
538}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700539EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540
541
542/**
543 * netdev_boot_base - get address from boot time settings
544 * @prefix: prefix for network device
545 * @unit: id for network device
546 *
547 * Check boot time settings for the base address of device.
548 * The found settings are set for the device to be used
549 * later in the device probing.
550 * Returns 0 if no settings found.
551 */
552unsigned long netdev_boot_base(const char *prefix, int unit)
553{
554 const struct netdev_boot_setup *s = dev_boot_setup;
555 char name[IFNAMSIZ];
556 int i;
557
558 sprintf(name, "%s%d", prefix, unit);
559
560 /*
561 * If device already registered then return base of 1
562 * to indicate not to probe for this interface
563 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700564 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 return 1;
566
567 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
568 if (!strcmp(name, s[i].name))
569 return s[i].map.base_addr;
570 return 0;
571}
572
573/*
574 * Saves at boot time configured settings for any netdevice.
575 */
576int __init netdev_boot_setup(char *str)
577{
578 int ints[5];
579 struct ifmap map;
580
581 str = get_options(str, ARRAY_SIZE(ints), ints);
582 if (!str || !*str)
583 return 0;
584
585 /* Save settings */
586 memset(&map, 0, sizeof(map));
587 if (ints[0] > 0)
588 map.irq = ints[1];
589 if (ints[0] > 1)
590 map.base_addr = ints[2];
591 if (ints[0] > 2)
592 map.mem_start = ints[3];
593 if (ints[0] > 3)
594 map.mem_end = ints[4];
595
596 /* Add new entry to the list */
597 return netdev_boot_setup_add(str, &map);
598}
599
600__setup("netdev=", netdev_boot_setup);
601
602/*******************************************************************************
603
604 Device Interface Subroutines
605
606*******************************************************************************/
607
608/**
609 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700610 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 * @name: name to find
612 *
613 * Find an interface by name. Must be called under RTNL semaphore
614 * or @dev_base_lock. If the name is found a pointer to the device
615 * is returned. If the name is not found then %NULL is returned. The
616 * reference counters are not incremented so the caller must be
617 * careful with locks.
618 */
619
Eric W. Biederman881d9662007-09-17 11:56:21 -0700620struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621{
622 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700623 struct net_device *dev;
624 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700626 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 if (!strncmp(dev->name, name, IFNAMSIZ))
628 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700629
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 return NULL;
631}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700632EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633
634/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000635 * dev_get_by_name_rcu - find a device by its name
636 * @net: the applicable net namespace
637 * @name: name to find
638 *
639 * Find an interface by name.
640 * If the name is found a pointer to the device is returned.
641 * If the name is not found then %NULL is returned.
642 * The reference counters are not incremented so the caller must be
643 * careful with locks. The caller must hold RCU lock.
644 */
645
646struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
647{
648 struct hlist_node *p;
649 struct net_device *dev;
650 struct hlist_head *head = dev_name_hash(net, name);
651
652 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
653 if (!strncmp(dev->name, name, IFNAMSIZ))
654 return dev;
655
656 return NULL;
657}
658EXPORT_SYMBOL(dev_get_by_name_rcu);
659
660/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700662 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 * @name: name to find
664 *
665 * Find an interface by name. This can be called from any
666 * context and does its own locking. The returned handle has
667 * the usage count incremented and the caller must use dev_put() to
668 * release it when it is no longer needed. %NULL is returned if no
669 * matching device is found.
670 */
671
Eric W. Biederman881d9662007-09-17 11:56:21 -0700672struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673{
674 struct net_device *dev;
675
Eric Dumazet72c95282009-10-30 07:11:27 +0000676 rcu_read_lock();
677 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678 if (dev)
679 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000680 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 return dev;
682}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700683EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684
685/**
686 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700687 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 * @ifindex: index of device
689 *
690 * Search for an interface by index. Returns %NULL if the device
691 * is not found or a pointer to the device. The device has not
692 * had its reference counter increased so the caller must be careful
693 * about locking. The caller must hold either the RTNL semaphore
694 * or @dev_base_lock.
695 */
696
Eric W. Biederman881d9662007-09-17 11:56:21 -0700697struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698{
699 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700700 struct net_device *dev;
701 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700703 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 if (dev->ifindex == ifindex)
705 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700706
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 return NULL;
708}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700709EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000711/**
712 * dev_get_by_index_rcu - find a device by its ifindex
713 * @net: the applicable net namespace
714 * @ifindex: index of device
715 *
716 * Search for an interface by index. Returns %NULL if the device
717 * is not found or a pointer to the device. The device has not
718 * had its reference counter increased so the caller must be careful
719 * about locking. The caller must hold RCU lock.
720 */
721
722struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
723{
724 struct hlist_node *p;
725 struct net_device *dev;
726 struct hlist_head *head = dev_index_hash(net, ifindex);
727
728 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
729 if (dev->ifindex == ifindex)
730 return dev;
731
732 return NULL;
733}
734EXPORT_SYMBOL(dev_get_by_index_rcu);
735
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736
737/**
738 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700739 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 * @ifindex: index of device
741 *
742 * Search for an interface by index. Returns NULL if the device
743 * is not found or a pointer to the device. The device returned has
744 * had a reference added and the pointer is safe until the user calls
745 * dev_put to indicate they have finished with it.
746 */
747
Eric W. Biederman881d9662007-09-17 11:56:21 -0700748struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749{
750 struct net_device *dev;
751
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000752 rcu_read_lock();
753 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 if (dev)
755 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000756 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 return dev;
758}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700759EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
761/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000762 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700763 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 * @type: media type of device
765 * @ha: hardware address
766 *
767 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800768 * is not found or a pointer to the device.
769 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000770 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 * and the caller must therefore be careful about locking
772 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 */
774
Eric Dumazet941666c2010-12-05 01:23:53 +0000775struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
776 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777{
778 struct net_device *dev;
779
Eric Dumazet941666c2010-12-05 01:23:53 +0000780 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 if (dev->type == type &&
782 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700783 return dev;
784
785 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786}
Eric Dumazet941666c2010-12-05 01:23:53 +0000787EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300788
Eric W. Biederman881d9662007-09-17 11:56:21 -0700789struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700790{
791 struct net_device *dev;
792
793 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700794 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700795 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700796 return dev;
797
798 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700799}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700800EXPORT_SYMBOL(__dev_getfirstbyhwtype);
801
Eric W. Biederman881d9662007-09-17 11:56:21 -0700802struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000804 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000806 rcu_read_lock();
807 for_each_netdev_rcu(net, dev)
808 if (dev->type == type) {
809 dev_hold(dev);
810 ret = dev;
811 break;
812 }
813 rcu_read_unlock();
814 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816EXPORT_SYMBOL(dev_getfirstbyhwtype);
817
818/**
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000819 * dev_get_by_flags_rcu - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700820 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 * @if_flags: IFF_* values
822 * @mask: bitmask of bits in if_flags to check
823 *
824 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000825 * is not found or a pointer to the device. Must be called inside
826 * rcu_read_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 */
828
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000829struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700830 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700832 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833
Pavel Emelianov7562f872007-05-03 15:13:45 -0700834 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800835 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -0700837 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 break;
839 }
840 }
Pavel Emelianov7562f872007-05-03 15:13:45 -0700841 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842}
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000843EXPORT_SYMBOL(dev_get_by_flags_rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844
845/**
846 * dev_valid_name - check if name is okay for network device
847 * @name: name string
848 *
849 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700850 * to allow sysfs to work. We also disallow any kind of
851 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852 */
Mitch Williamsc2373ee2005-11-09 10:34:45 -0800853int dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700855 if (*name == '\0')
856 return 0;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700857 if (strlen(name) >= IFNAMSIZ)
858 return 0;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700859 if (!strcmp(name, ".") || !strcmp(name, ".."))
860 return 0;
861
862 while (*name) {
863 if (*name == '/' || isspace(*name))
864 return 0;
865 name++;
866 }
867 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700869EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870
871/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200872 * __dev_alloc_name - allocate a name for a device
873 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200875 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 *
877 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700878 * id. It scans list of devices to build up a free map, then chooses
879 * the first empty slot. The caller must hold the dev_base or rtnl lock
880 * while allocating the name and adding the device in order to avoid
881 * duplicates.
882 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
883 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 */
885
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200886static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887{
888 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 const char *p;
890 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700891 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 struct net_device *d;
893
894 p = strnchr(name, IFNAMSIZ-1, '%');
895 if (p) {
896 /*
897 * Verify the string as this thing may have come from
898 * the user. There must be either one "%d" and no other "%"
899 * characters.
900 */
901 if (p[1] != 'd' || strchr(p + 2, '%'))
902 return -EINVAL;
903
904 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700905 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 if (!inuse)
907 return -ENOMEM;
908
Eric W. Biederman881d9662007-09-17 11:56:21 -0700909 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910 if (!sscanf(d->name, name, &i))
911 continue;
912 if (i < 0 || i >= max_netdevices)
913 continue;
914
915 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200916 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 if (!strncmp(buf, d->name, IFNAMSIZ))
918 set_bit(i, inuse);
919 }
920
921 i = find_first_zero_bit(inuse, max_netdevices);
922 free_page((unsigned long) inuse);
923 }
924
Octavian Purdilad9031022009-11-18 02:36:59 +0000925 if (buf != name)
926 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200927 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930 /* It is possible to run out of possible slots
931 * when the name is long and there isn't enough space left
932 * for the digits, or if all bits are used.
933 */
934 return -ENFILE;
935}
936
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200937/**
938 * dev_alloc_name - allocate a name for a device
939 * @dev: device
940 * @name: name format string
941 *
942 * Passed a format string - eg "lt%d" it will try and find a suitable
943 * id. It scans list of devices to build up a free map, then chooses
944 * the first empty slot. The caller must hold the dev_base or rtnl lock
945 * while allocating the name and adding the device in order to avoid
946 * duplicates.
947 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
948 * Returns the number of the unit assigned or a negative errno code.
949 */
950
951int dev_alloc_name(struct net_device *dev, const char *name)
952{
953 char buf[IFNAMSIZ];
954 struct net *net;
955 int ret;
956
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900957 BUG_ON(!dev_net(dev));
958 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200959 ret = __dev_alloc_name(net, name, buf);
960 if (ret >= 0)
961 strlcpy(dev->name, buf, IFNAMSIZ);
962 return ret;
963}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700964EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200965
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000966static int dev_get_valid_name(struct net_device *dev, const char *name)
Octavian Purdilad9031022009-11-18 02:36:59 +0000967{
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000968 struct net *net;
969
970 BUG_ON(!dev_net(dev));
971 net = dev_net(dev);
972
Octavian Purdilad9031022009-11-18 02:36:59 +0000973 if (!dev_valid_name(name))
974 return -EINVAL;
975
Jiri Pirko1c5cae82011-04-30 01:21:32 +0000976 if (strchr(name, '%'))
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000977 return dev_alloc_name(dev, name);
Octavian Purdilad9031022009-11-18 02:36:59 +0000978 else if (__dev_get_by_name(net, name))
979 return -EEXIST;
Daniel Lezcano8ce6cebc2010-05-19 10:12:19 +0000980 else if (dev->name != name)
981 strlcpy(dev->name, name, IFNAMSIZ);
Octavian Purdilad9031022009-11-18 02:36:59 +0000982
983 return 0;
984}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985
986/**
987 * dev_change_name - change name of a device
988 * @dev: device
989 * @newname: name (or format string) must be at least IFNAMSIZ
990 *
991 * Change name of a device, can pass format strings "eth%d".
992 * for wildcarding.
993 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700994int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995{
Herbert Xufcc5a032007-07-30 17:03:38 -0700996 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700998 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700999 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000
1001 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001002 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001004 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 if (dev->flags & IFF_UP)
1006 return -EBUSY;
1007
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001008 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
1009 return 0;
1010
Herbert Xufcc5a032007-07-30 17:03:38 -07001011 memcpy(oldname, dev->name, IFNAMSIZ);
1012
Jiri Pirko1c5cae82011-04-30 01:21:32 +00001013 err = dev_get_valid_name(dev, newname);
Octavian Purdilad9031022009-11-18 02:36:59 +00001014 if (err < 0)
1015 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016
Herbert Xufcc5a032007-07-30 17:03:38 -07001017rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001018 ret = device_rename(&dev->dev, dev->name);
1019 if (ret) {
1020 memcpy(dev->name, oldname, IFNAMSIZ);
1021 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001022 }
Herbert Xu7f988ea2007-07-30 16:35:46 -07001023
1024 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001025 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001026 write_unlock_bh(&dev_base_lock);
1027
1028 synchronize_rcu();
1029
1030 write_lock_bh(&dev_base_lock);
1031 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001032 write_unlock_bh(&dev_base_lock);
1033
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001034 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001035 ret = notifier_to_errno(ret);
1036
1037 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001038 /* err >= 0 after dev_alloc_name() or stores the first errno */
1039 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001040 err = ret;
1041 memcpy(dev->name, oldname, IFNAMSIZ);
1042 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001043 } else {
1044 printk(KERN_ERR
1045 "%s: name change rollback failed: %d.\n",
1046 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001047 }
1048 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
1050 return err;
1051}
1052
1053/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001054 * dev_set_alias - change ifalias of a device
1055 * @dev: device
1056 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001057 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001058 *
1059 * Set ifalias for a device,
1060 */
1061int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1062{
1063 ASSERT_RTNL();
1064
1065 if (len >= IFALIASZ)
1066 return -EINVAL;
1067
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001068 if (!len) {
1069 if (dev->ifalias) {
1070 kfree(dev->ifalias);
1071 dev->ifalias = NULL;
1072 }
1073 return 0;
1074 }
1075
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001076 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001077 if (!dev->ifalias)
1078 return -ENOMEM;
1079
1080 strlcpy(dev->ifalias, alias, len+1);
1081 return len;
1082}
1083
1084
1085/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001086 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001087 * @dev: device to cause notification
1088 *
1089 * Called to indicate a device has changed features.
1090 */
1091void netdev_features_change(struct net_device *dev)
1092{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001093 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001094}
1095EXPORT_SYMBOL(netdev_features_change);
1096
1097/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098 * netdev_state_change - device changes state
1099 * @dev: device to cause notification
1100 *
1101 * Called to indicate a device has changed state. This function calls
1102 * the notifier chains for netdev_chain and sends a NEWLINK message
1103 * to the routing socket.
1104 */
1105void netdev_state_change(struct net_device *dev)
1106{
1107 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001108 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1110 }
1111}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001112EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001114int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001115{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001116 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001117}
1118EXPORT_SYMBOL(netdev_bonding_change);
1119
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120/**
1121 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001122 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 * @name: name of interface
1124 *
1125 * If a network interface is not present and the process has suitable
1126 * privileges this function loads the module. If module loading is not
1127 * available in this kernel then it becomes a nop.
1128 */
1129
Eric W. Biederman881d9662007-09-17 11:56:21 -07001130void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001132 struct net_device *dev;
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001133 int no_module;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134
Eric Dumazet72c95282009-10-30 07:11:27 +00001135 rcu_read_lock();
1136 dev = dev_get_by_name_rcu(net, name);
1137 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138
Vasiliy Kulikov8909c9a2011-03-02 00:33:13 +03001139 no_module = !dev;
1140 if (no_module && capable(CAP_NET_ADMIN))
1141 no_module = request_module("netdev-%s", name);
1142 if (no_module && capable(CAP_SYS_MODULE)) {
1143 if (!request_module("%s", name))
1144 pr_err("Loading kernel module for a network device "
1145"with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s "
1146"instead\n", name);
1147 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001149EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
Patrick McHardybd380812010-02-26 06:34:53 +00001151static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001153 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001154 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001156 ASSERT_RTNL();
1157
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158 if (!netif_device_present(dev))
1159 return -ENODEV;
1160
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001161 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1162 ret = notifier_to_errno(ret);
1163 if (ret)
1164 return ret;
1165
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001167
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001168 if (ops->ndo_validate_addr)
1169 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001170
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001171 if (!ret && ops->ndo_open)
1172 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173
Jeff Garzikbada3392007-10-23 20:19:37 -07001174 if (ret)
1175 clear_bit(__LINK_STATE_START, &dev->state);
1176 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001177 dev->flags |= IFF_UP;
David S. Millerb4bd07c2009-02-06 22:06:43 -08001178 net_dmaengine_get();
Patrick McHardy4417da62007-06-27 01:28:10 -07001179 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001180 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001182
Linus Torvalds1da177e2005-04-16 15:20:36 -07001183 return ret;
1184}
Patrick McHardybd380812010-02-26 06:34:53 +00001185
1186/**
1187 * dev_open - prepare an interface for use.
1188 * @dev: device to open
1189 *
1190 * Takes a device from down to up state. The device's private open
1191 * function is invoked and then the multicast lists are loaded. Finally
1192 * the device is moved into the up state and a %NETDEV_UP message is
1193 * sent to the netdev notifier chain.
1194 *
1195 * Calling this function on an active interface is a nop. On a failure
1196 * a negative errno code is returned.
1197 */
1198int dev_open(struct net_device *dev)
1199{
1200 int ret;
1201
Patrick McHardybd380812010-02-26 06:34:53 +00001202 if (dev->flags & IFF_UP)
1203 return 0;
1204
Patrick McHardybd380812010-02-26 06:34:53 +00001205 ret = __dev_open(dev);
1206 if (ret < 0)
1207 return ret;
1208
Patrick McHardybd380812010-02-26 06:34:53 +00001209 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1210 call_netdevice_notifiers(NETDEV_UP, dev);
1211
1212 return ret;
1213}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001214EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215
Octavian Purdila44345722010-12-13 12:44:07 +00001216static int __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217{
Octavian Purdila44345722010-12-13 12:44:07 +00001218 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001219
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001220 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001221 might_sleep();
1222
Octavian Purdila44345722010-12-13 12:44:07 +00001223 list_for_each_entry(dev, head, unreg_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001224 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225
Octavian Purdila44345722010-12-13 12:44:07 +00001226 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227
Octavian Purdila44345722010-12-13 12:44:07 +00001228 /* Synchronize to scheduled poll. We cannot touch poll list, it
1229 * can be even on different cpu. So just clear netif_running().
1230 *
1231 * dev->stop() will invoke napi_disable() on all of it's
1232 * napi_struct instances on this device.
1233 */
1234 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1235 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236
Octavian Purdila44345722010-12-13 12:44:07 +00001237 dev_deactivate_many(head);
1238
1239 list_for_each_entry(dev, head, unreg_list) {
1240 const struct net_device_ops *ops = dev->netdev_ops;
1241
1242 /*
1243 * Call the device specific close. This cannot fail.
1244 * Only if device is UP
1245 *
1246 * We allow it to be called even after a DETACH hot-plug
1247 * event.
1248 */
1249 if (ops->ndo_stop)
1250 ops->ndo_stop(dev);
1251
Octavian Purdila44345722010-12-13 12:44:07 +00001252 dev->flags &= ~IFF_UP;
Octavian Purdila44345722010-12-13 12:44:07 +00001253 net_dmaengine_put();
1254 }
1255
1256 return 0;
1257}
1258
1259static int __dev_close(struct net_device *dev)
1260{
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001261 int retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001262 LIST_HEAD(single);
1263
1264 list_add(&dev->unreg_list, &single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001265 retval = __dev_close_many(&single);
1266 list_del(&single);
1267 return retval;
Octavian Purdila44345722010-12-13 12:44:07 +00001268}
1269
Eric Dumazet3fbd8752011-01-19 21:23:22 +00001270static int dev_close_many(struct list_head *head)
Octavian Purdila44345722010-12-13 12:44:07 +00001271{
1272 struct net_device *dev, *tmp;
1273 LIST_HEAD(tmp_list);
1274
1275 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1276 if (!(dev->flags & IFF_UP))
1277 list_move(&dev->unreg_list, &tmp_list);
1278
1279 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001280
Octavian Purdila44345722010-12-13 12:44:07 +00001281 list_for_each_entry(dev, head, unreg_list) {
1282 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1283 call_netdevice_notifiers(NETDEV_DOWN, dev);
1284 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285
Octavian Purdila44345722010-12-13 12:44:07 +00001286 /* rollback_registered_many needs the complete original list */
1287 list_splice(&tmp_list, head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 return 0;
1289}
Patrick McHardybd380812010-02-26 06:34:53 +00001290
1291/**
1292 * dev_close - shutdown an interface.
1293 * @dev: device to shutdown
1294 *
1295 * This function moves an active device into down state. A
1296 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1297 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1298 * chain.
1299 */
1300int dev_close(struct net_device *dev)
1301{
Eric Dumazete14a5992011-05-10 12:26:06 -07001302 if (dev->flags & IFF_UP) {
1303 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001304
Eric Dumazete14a5992011-05-10 12:26:06 -07001305 list_add(&dev->unreg_list, &single);
1306 dev_close_many(&single);
1307 list_del(&single);
1308 }
Patrick McHardybd380812010-02-26 06:34:53 +00001309 return 0;
1310}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001311EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
1313
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001314/**
1315 * dev_disable_lro - disable Large Receive Offload on a device
1316 * @dev: device
1317 *
1318 * Disable Large Receive Offload (LRO) on a net device. Must be
1319 * called under RTNL. This is needed if received packets may be
1320 * forwarded to another interface.
1321 */
1322void dev_disable_lro(struct net_device *dev)
1323{
Michał Mirosław27660512011-03-18 16:56:34 +00001324 u32 flags;
1325
Neil Hormanf11970e2011-05-24 08:31:09 +00001326 /*
1327 * If we're trying to disable lro on a vlan device
1328 * use the underlying physical device instead
1329 */
1330 if (is_vlan_dev(dev))
1331 dev = vlan_dev_real_dev(dev);
1332
Michał Mirosław27660512011-03-18 16:56:34 +00001333 if (dev->ethtool_ops && dev->ethtool_ops->get_flags)
1334 flags = dev->ethtool_ops->get_flags(dev);
1335 else
1336 flags = ethtool_op_get_flags(dev);
1337
1338 if (!(flags & ETH_FLAG_LRO))
1339 return;
1340
1341 __ethtool_set_flags(dev, flags & ~ETH_FLAG_LRO);
Michał Mirosław22d59692011-04-21 12:42:15 +00001342 if (unlikely(dev->features & NETIF_F_LRO))
1343 netdev_WARN(dev, "failed to disable LRO!\n");
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001344}
1345EXPORT_SYMBOL(dev_disable_lro);
1346
1347
Eric W. Biederman881d9662007-09-17 11:56:21 -07001348static int dev_boot_phase = 1;
1349
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350/**
1351 * register_netdevice_notifier - register a network notifier block
1352 * @nb: notifier
1353 *
1354 * Register a notifier to be called when network device events occur.
1355 * The notifier passed is linked into the kernel structures and must
1356 * not be reused until it has been unregistered. A negative errno code
1357 * is returned on a failure.
1358 *
1359 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001360 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361 * view of the network device list.
1362 */
1363
1364int register_netdevice_notifier(struct notifier_block *nb)
1365{
1366 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001367 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001368 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 int err;
1370
1371 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001372 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001373 if (err)
1374 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001375 if (dev_boot_phase)
1376 goto unlock;
1377 for_each_net(net) {
1378 for_each_netdev(net, dev) {
1379 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1380 err = notifier_to_errno(err);
1381 if (err)
1382 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383
Eric W. Biederman881d9662007-09-17 11:56:21 -07001384 if (!(dev->flags & IFF_UP))
1385 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001386
Eric W. Biederman881d9662007-09-17 11:56:21 -07001387 nb->notifier_call(nb, NETDEV_UP, dev);
1388 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001390
1391unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392 rtnl_unlock();
1393 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001394
1395rollback:
1396 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001397 for_each_net(net) {
1398 for_each_netdev(net, dev) {
1399 if (dev == last)
1400 break;
Herbert Xufcc5a032007-07-30 17:03:38 -07001401
Eric W. Biederman881d9662007-09-17 11:56:21 -07001402 if (dev->flags & IFF_UP) {
1403 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1404 nb->notifier_call(nb, NETDEV_DOWN, dev);
1405 }
1406 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001407 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001408 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001409 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001410
1411 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001412 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001414EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415
1416/**
1417 * unregister_netdevice_notifier - unregister a network notifier block
1418 * @nb: notifier
1419 *
1420 * Unregister a notifier previously registered by
1421 * register_netdevice_notifier(). The notifier is unlinked into the
1422 * kernel structures and may then be reused. A negative errno code
1423 * is returned on a failure.
1424 */
1425
1426int unregister_netdevice_notifier(struct notifier_block *nb)
1427{
Herbert Xu9f514952006-03-25 01:24:25 -08001428 int err;
1429
1430 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001431 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xu9f514952006-03-25 01:24:25 -08001432 rtnl_unlock();
1433 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001435EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436
1437/**
1438 * call_netdevice_notifiers - call all network notifier blocks
1439 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001440 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441 *
1442 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001443 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001444 */
1445
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001446int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001447{
Jiri Pirkoab930472010-04-20 01:45:37 -07001448 ASSERT_RTNL();
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001449 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001451EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452
Eric Dumazet588f0332011-11-15 04:12:55 +00001453static struct jump_label_key netstamp_needed __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
1455void net_enable_timestamp(void)
1456{
Eric Dumazet588f0332011-11-15 04:12:55 +00001457 jump_label_inc(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001459EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460
1461void net_disable_timestamp(void)
1462{
Eric Dumazet588f0332011-11-15 04:12:55 +00001463 jump_label_dec(&netstamp_needed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001465EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466
Eric Dumazet3b098e22010-05-15 23:57:10 -07001467static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468{
Eric Dumazet588f0332011-11-15 04:12:55 +00001469 skb->tstamp.tv64 = 0;
1470 if (static_branch(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001471 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472}
1473
Eric Dumazet588f0332011-11-15 04:12:55 +00001474#define net_timestamp_check(COND, SKB) \
1475 if (static_branch(&netstamp_needed)) { \
1476 if ((COND) && !(SKB)->tstamp.tv64) \
1477 __net_timestamp(SKB); \
1478 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001479
Richard Cochran4dc360c2011-10-19 17:00:35 -04001480static int net_hwtstamp_validate(struct ifreq *ifr)
1481{
1482 struct hwtstamp_config cfg;
1483 enum hwtstamp_tx_types tx_type;
1484 enum hwtstamp_rx_filters rx_filter;
1485 int tx_type_valid = 0;
1486 int rx_filter_valid = 0;
1487
1488 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1489 return -EFAULT;
1490
1491 if (cfg.flags) /* reserved for future extensions */
1492 return -EINVAL;
1493
1494 tx_type = cfg.tx_type;
1495 rx_filter = cfg.rx_filter;
1496
1497 switch (tx_type) {
1498 case HWTSTAMP_TX_OFF:
1499 case HWTSTAMP_TX_ON:
1500 case HWTSTAMP_TX_ONESTEP_SYNC:
1501 tx_type_valid = 1;
1502 break;
1503 }
1504
1505 switch (rx_filter) {
1506 case HWTSTAMP_FILTER_NONE:
1507 case HWTSTAMP_FILTER_ALL:
1508 case HWTSTAMP_FILTER_SOME:
1509 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1510 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1511 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1512 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1513 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1514 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1515 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1516 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1517 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1518 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1519 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1520 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1521 rx_filter_valid = 1;
1522 break;
1523 }
1524
1525 if (!tx_type_valid || !rx_filter_valid)
1526 return -ERANGE;
1527
1528 return 0;
1529}
1530
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001531static inline bool is_skb_forwardable(struct net_device *dev,
1532 struct sk_buff *skb)
1533{
1534 unsigned int len;
1535
1536 if (!(dev->flags & IFF_UP))
1537 return false;
1538
1539 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1540 if (skb->len <= len)
1541 return true;
1542
1543 /* if TSO is enabled, we don't care about the length as the packet
1544 * could be forwarded without being segmented before
1545 */
1546 if (skb_is_gso(skb))
1547 return true;
1548
1549 return false;
1550}
1551
Arnd Bergmann44540962009-11-26 06:07:08 +00001552/**
1553 * dev_forward_skb - loopback an skb to another netif
1554 *
1555 * @dev: destination network device
1556 * @skb: buffer to forward
1557 *
1558 * return values:
1559 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001560 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001561 *
1562 * dev_forward_skb can be used for injecting an skb from the
1563 * start_xmit function of one device into the receive queue
1564 * of another device.
1565 *
1566 * The receiving device may be in another namespace, so
1567 * we have to clear all information in the skb that could
1568 * impact namespace isolation.
1569 */
1570int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1571{
Michael S. Tsirkin48c83012011-08-31 08:03:29 +00001572 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1573 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1574 atomic_long_inc(&dev->rx_dropped);
1575 kfree_skb(skb);
1576 return NET_RX_DROP;
1577 }
1578 }
1579
Arnd Bergmann44540962009-11-26 06:07:08 +00001580 skb_orphan(skb);
Ben Greearc736eef2010-07-22 09:54:47 +00001581 nf_reset(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001582
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001583 if (unlikely(!is_skb_forwardable(dev, skb))) {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00001584 atomic_long_inc(&dev->rx_dropped);
Eric Dumazet6ec82562010-05-06 00:53:53 -07001585 kfree_skb(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001586 return NET_RX_DROP;
Eric Dumazet6ec82562010-05-06 00:53:53 -07001587 }
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001588 skb_set_dev(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001589 skb->tstamp.tv64 = 0;
1590 skb->pkt_type = PACKET_HOST;
1591 skb->protocol = eth_type_trans(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001592 return netif_rx(skb);
1593}
1594EXPORT_SYMBOL_GPL(dev_forward_skb);
1595
Changli Gao71d9dec2010-12-15 19:57:25 +00001596static inline int deliver_skb(struct sk_buff *skb,
1597 struct packet_type *pt_prev,
1598 struct net_device *orig_dev)
1599{
1600 atomic_inc(&skb->users);
1601 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1602}
1603
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604/*
1605 * Support routine. Sends outgoing frames to any network
1606 * taps currently in use.
1607 */
1608
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001609static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610{
1611 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00001612 struct sk_buff *skb2 = NULL;
1613 struct packet_type *pt_prev = NULL;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001614
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615 rcu_read_lock();
1616 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1617 /* Never send packets back to the socket
1618 * they originated from - MvS (miquels@drinkel.ow.org)
1619 */
1620 if ((ptype->dev == dev || !ptype->dev) &&
1621 (ptype->af_packet_priv == NULL ||
1622 (struct sock *)ptype->af_packet_priv != skb->sk)) {
Changli Gao71d9dec2010-12-15 19:57:25 +00001623 if (pt_prev) {
1624 deliver_skb(skb2, pt_prev, skb->dev);
1625 pt_prev = ptype;
1626 continue;
1627 }
1628
1629 skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 if (!skb2)
1631 break;
1632
Eric Dumazet70978182010-12-20 21:22:51 +00001633 net_timestamp_set(skb2);
1634
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635 /* skb->nh should be correctly
1636 set by sender, so that the second statement is
1637 just protection against buggy protocols.
1638 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001639 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001641 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001642 skb2->network_header > skb2->tail) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643 if (net_ratelimit())
1644 printk(KERN_CRIT "protocol %04x is "
1645 "buggy, dev %s\n",
Sebastian Andrzej Siewior70777d02010-06-30 10:39:19 -07001646 ntohs(skb2->protocol),
1647 dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001648 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 }
1650
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001651 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 skb2->pkt_type = PACKET_OUTGOING;
Changli Gao71d9dec2010-12-15 19:57:25 +00001653 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 }
1655 }
Changli Gao71d9dec2010-12-15 19:57:25 +00001656 if (pt_prev)
1657 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001658 rcu_read_unlock();
1659}
1660
John Fastabend4f57c082011-01-17 08:06:04 +00001661/* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1662 * @dev: Network device
1663 * @txq: number of queues available
1664 *
1665 * If real_num_tx_queues is changed the tc mappings may no longer be
1666 * valid. To resolve this verify the tc mapping remains valid and if
1667 * not NULL the mapping. With no priorities mapping to this
1668 * offset/count pair it will no longer be used. In the worst case TC0
1669 * is invalid nothing can be done so disable priority mappings. If is
1670 * expected that drivers will fix this mapping if they can before
1671 * calling netif_set_real_num_tx_queues.
1672 */
Eric Dumazetbb134d22011-01-20 19:18:08 +00001673static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00001674{
1675 int i;
1676 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1677
1678 /* If TC0 is invalidated disable TC mapping */
1679 if (tc->offset + tc->count > txq) {
1680 pr_warning("Number of in use tx queues changed "
1681 "invalidating tc mappings. Priority "
1682 "traffic classification disabled!\n");
1683 dev->num_tc = 0;
1684 return;
1685 }
1686
1687 /* Invalidated prio to tc mappings set to TC0 */
1688 for (i = 1; i < TC_BITMASK + 1; i++) {
1689 int q = netdev_get_prio_tc_map(dev, i);
1690
1691 tc = &dev->tc_to_txq[q];
1692 if (tc->offset + tc->count > txq) {
1693 pr_warning("Number of in use tx queues "
1694 "changed. Priority %i to tc "
1695 "mapping %i is no longer valid "
1696 "setting map to 0\n",
1697 i, q);
1698 netdev_set_prio_tc_map(dev, i, 0);
1699 }
1700 }
1701}
1702
John Fastabendf0796d52010-07-01 13:21:57 +00001703/*
1704 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1705 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1706 */
Tom Herberte6484932010-10-18 18:04:39 +00001707int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00001708{
Tom Herbert1d24eb42010-11-21 13:17:27 +00001709 int rc;
1710
Tom Herberte6484932010-10-18 18:04:39 +00001711 if (txq < 1 || txq > dev->num_tx_queues)
1712 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00001713
Ben Hutchings5c565802011-02-15 19:39:21 +00001714 if (dev->reg_state == NETREG_REGISTERED ||
1715 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00001716 ASSERT_RTNL();
1717
Tom Herbert1d24eb42010-11-21 13:17:27 +00001718 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1719 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00001720 if (rc)
1721 return rc;
1722
John Fastabend4f57c082011-01-17 08:06:04 +00001723 if (dev->num_tc)
1724 netif_setup_tc(dev, txq);
1725
Tom Herberte6484932010-10-18 18:04:39 +00001726 if (txq < dev->real_num_tx_queues)
1727 qdisc_reset_all_tx_gt(dev, txq);
John Fastabendf0796d52010-07-01 13:21:57 +00001728 }
Tom Herberte6484932010-10-18 18:04:39 +00001729
1730 dev->real_num_tx_queues = txq;
1731 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00001732}
1733EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08001734
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001735#ifdef CONFIG_RPS
1736/**
1737 * netif_set_real_num_rx_queues - set actual number of RX queues used
1738 * @dev: Network device
1739 * @rxq: Actual number of RX queues
1740 *
1741 * This must be called either with the rtnl_lock held or before
1742 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07001743 * negative error code. If called before registration, it always
1744 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001745 */
1746int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1747{
1748 int rc;
1749
Tom Herbertbd25fa72010-10-18 18:00:16 +00001750 if (rxq < 1 || rxq > dev->num_rx_queues)
1751 return -EINVAL;
1752
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001753 if (dev->reg_state == NETREG_REGISTERED) {
1754 ASSERT_RTNL();
1755
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001756 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1757 rxq);
1758 if (rc)
1759 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00001760 }
1761
1762 dev->real_num_rx_queues = rxq;
1763 return 0;
1764}
1765EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1766#endif
1767
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001768static inline void __netif_reschedule(struct Qdisc *q)
1769{
1770 struct softnet_data *sd;
1771 unsigned long flags;
1772
1773 local_irq_save(flags);
1774 sd = &__get_cpu_var(softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00001775 q->next_sched = NULL;
1776 *sd->output_queue_tailp = q;
1777 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001778 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1779 local_irq_restore(flags);
1780}
1781
David S. Miller37437bb2008-07-16 02:15:04 -07001782void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001783{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001784 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1785 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001786}
1787EXPORT_SYMBOL(__netif_schedule);
1788
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001789void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001790{
David S. Miller3578b0c2010-08-03 00:24:04 -07001791 if (atomic_dec_and_test(&skb->users)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001792 struct softnet_data *sd;
1793 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001794
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001795 local_irq_save(flags);
1796 sd = &__get_cpu_var(softnet_data);
1797 skb->next = sd->completion_queue;
1798 sd->completion_queue = skb;
1799 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1800 local_irq_restore(flags);
1801 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001802}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001803EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001804
1805void dev_kfree_skb_any(struct sk_buff *skb)
1806{
1807 if (in_irq() || irqs_disabled())
1808 dev_kfree_skb_irq(skb);
1809 else
1810 dev_kfree_skb(skb);
1811}
1812EXPORT_SYMBOL(dev_kfree_skb_any);
1813
1814
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001815/**
1816 * netif_device_detach - mark device as removed
1817 * @dev: network device
1818 *
1819 * Mark device as removed from system and therefore no longer available.
1820 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001821void netif_device_detach(struct net_device *dev)
1822{
1823 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1824 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001825 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001826 }
1827}
1828EXPORT_SYMBOL(netif_device_detach);
1829
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001830/**
1831 * netif_device_attach - mark device as attached
1832 * @dev: network device
1833 *
1834 * Mark device as attached from system and restart if needed.
1835 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001836void netif_device_attach(struct net_device *dev)
1837{
1838 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1839 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001840 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001841 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001842 }
1843}
1844EXPORT_SYMBOL(netif_device_attach);
1845
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001846/**
1847 * skb_dev_set -- assign a new device to a buffer
1848 * @skb: buffer for the new device
1849 * @dev: network device
1850 *
1851 * If an skb is owned by a device already, we have to reset
1852 * all data private to the namespace a device belongs to
1853 * before assigning it a new device.
1854 */
1855#ifdef CONFIG_NET_NS
1856void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1857{
1858 skb_dst_drop(skb);
1859 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1860 secpath_reset(skb);
1861 nf_reset(skb);
1862 skb_init_secmark(skb);
1863 skb->mark = 0;
1864 skb->priority = 0;
1865 skb->nf_trace = 0;
1866 skb->ipvs_property = 0;
1867#ifdef CONFIG_NET_SCHED
1868 skb->tc_index = 0;
1869#endif
1870 }
1871 skb->dev = dev;
1872}
1873EXPORT_SYMBOL(skb_set_dev);
1874#endif /* CONFIG_NET_NS */
1875
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876/*
1877 * Invalidate hardware checksum when packet is to be mangled, and
1878 * complete checksum manually on outgoing path.
1879 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001880int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001881{
Al Virod3bc23e2006-11-14 21:24:49 -08001882 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001883 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884
Patrick McHardy84fa7932006-08-29 16:44:56 -07001885 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001886 goto out_set_summed;
1887
1888 if (unlikely(skb_shinfo(skb)->gso_size)) {
Herbert Xua430a432006-07-08 13:34:56 -07001889 /* Let GSO fix up the checksum. */
1890 goto out_set_summed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891 }
1892
Michał Mirosław55508d62010-12-14 15:24:08 +00001893 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07001894 BUG_ON(offset >= skb_headlen(skb));
1895 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1896
1897 offset += skb->csum_offset;
1898 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1899
1900 if (skb_cloned(skb) &&
1901 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1903 if (ret)
1904 goto out;
1905 }
1906
Herbert Xua0308472007-10-15 01:47:15 -07001907 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001908out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001909 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001910out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911 return ret;
1912}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001913EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001915/**
1916 * skb_gso_segment - Perform segmentation on skb.
1917 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001918 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001919 *
1920 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001921 *
1922 * It may return NULL if the skb requires no segmentation. This is
1923 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001924 */
Michał Mirosław04ed3e72011-01-24 15:32:47 -08001925struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001926{
1927 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1928 struct packet_type *ptype;
Al Viro252e3342006-11-14 20:48:11 -08001929 __be16 type = skb->protocol;
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001930 int vlan_depth = ETH_HLEN;
Herbert Xua430a432006-07-08 13:34:56 -07001931 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001932
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001933 while (type == htons(ETH_P_8021Q)) {
1934 struct vlan_hdr *vh;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001935
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001936 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
Jesse Gross7b9c6092010-10-20 13:56:04 +00001937 return ERR_PTR(-EINVAL);
1938
Jesse Grossc8d5bcd2010-10-29 12:14:54 +00001939 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1940 type = vh->h_vlan_encapsulated_proto;
1941 vlan_depth += VLAN_HLEN;
Jesse Gross7b9c6092010-10-20 13:56:04 +00001942 }
1943
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001944 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001945 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001946 __skb_pull(skb, skb->mac_len);
1947
Herbert Xu67fd1a72009-01-19 16:26:44 -08001948 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1949 struct net_device *dev = skb->dev;
1950 struct ethtool_drvinfo info = {};
1951
1952 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1953 dev->ethtool_ops->get_drvinfo(dev, &info);
1954
Joe Perchesb194a362010-10-30 11:08:52 +00001955 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
Herbert Xu67fd1a72009-01-19 16:26:44 -08001956 info.driver, dev ? dev->features : 0L,
1957 skb->sk ? skb->sk->sk_route_caps : 0L,
1958 skb->len, skb->data_len, skb->ip_summed);
1959
Herbert Xua430a432006-07-08 13:34:56 -07001960 if (skb_header_cloned(skb) &&
1961 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1962 return ERR_PTR(err);
1963 }
1964
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001965 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001966 list_for_each_entry_rcu(ptype,
1967 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001968 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001969 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001970 err = ptype->gso_send_check(skb);
1971 segs = ERR_PTR(err);
1972 if (err || skb_gso_ok(skb, features))
1973 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001974 __skb_push(skb, (skb->data -
1975 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001976 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001977 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001978 break;
1979 }
1980 }
1981 rcu_read_unlock();
1982
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001983 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07001984
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001985 return segs;
1986}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001987EXPORT_SYMBOL(skb_gso_segment);
1988
Herbert Xufb286bb2005-11-10 13:01:24 -08001989/* Take action when hardware reception checksum errors are detected. */
1990#ifdef CONFIG_BUG
1991void netdev_rx_csum_fault(struct net_device *dev)
1992{
1993 if (net_ratelimit()) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001994 printk(KERN_ERR "%s: hw csum failure.\n",
Stephen Hemminger246a4212005-12-08 15:21:39 -08001995 dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08001996 dump_stack();
1997 }
1998}
1999EXPORT_SYMBOL(netdev_rx_csum_fault);
2000#endif
2001
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002/* Actually, we should eliminate this check as soon as we know, that:
2003 * 1. IOMMU is present and allows to map all the memory.
2004 * 2. No high memory really exists on this machine.
2005 */
2006
Eric Dumazet9092c652010-04-02 13:34:49 -07002007static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002008{
Herbert Xu3d3a8532006-06-27 13:33:10 -07002009#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002010 int i;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002011 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002012 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2013 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2014 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002015 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00002016 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002017 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002018
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002019 if (PCI_DMA_BUS_IS_PHYS) {
2020 struct device *pdev = dev->dev.parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002021
Eric Dumazet9092c652010-04-02 13:34:49 -07002022 if (!pdev)
2023 return 0;
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002024 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
Ian Campbellea2ab692011-08-22 23:44:58 +00002025 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2026 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00002027 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2028 return 1;
2029 }
2030 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07002031#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002032 return 0;
2033}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002035struct dev_gso_cb {
2036 void (*destructor)(struct sk_buff *skb);
2037};
2038
2039#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2040
2041static void dev_gso_skb_destructor(struct sk_buff *skb)
2042{
2043 struct dev_gso_cb *cb;
2044
2045 do {
2046 struct sk_buff *nskb = skb->next;
2047
2048 skb->next = nskb->next;
2049 nskb->next = NULL;
2050 kfree_skb(nskb);
2051 } while (skb->next);
2052
2053 cb = DEV_GSO_CB(skb);
2054 if (cb->destructor)
2055 cb->destructor(skb);
2056}
2057
2058/**
2059 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2060 * @skb: buffer to segment
Jesse Gross91ecb632011-01-09 06:23:33 +00002061 * @features: device features as applicable to this skb
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002062 *
2063 * This function segments the given skb and stores the list of segments
2064 * in skb->next.
2065 */
Jesse Gross91ecb632011-01-09 06:23:33 +00002066static int dev_gso_segment(struct sk_buff *skb, int features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002067{
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002068 struct sk_buff *segs;
2069
Herbert Xu576a30e2006-06-27 13:22:38 -07002070 segs = skb_gso_segment(skb, features);
2071
2072 /* Verifying header integrity only. */
2073 if (!segs)
2074 return 0;
2075
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07002076 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002077 return PTR_ERR(segs);
2078
2079 skb->next = segs;
2080 DEV_GSO_CB(skb)->destructor = skb->destructor;
2081 skb->destructor = dev_gso_skb_destructor;
2082
2083 return 0;
2084}
2085
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002086/*
2087 * Try to orphan skb early, right before transmission by the device.
Oliver Hartkopp2244d072010-08-17 08:59:14 +00002088 * We cannot orphan skb if tx timestamp is requested or the sk-reference
2089 * is needed on driver level for other reasons, e.g. see net/can/raw.c
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002090 */
2091static inline void skb_orphan_try(struct sk_buff *skb)
2092{
Eric Dumazet87fd3082010-07-13 05:24:20 +00002093 struct sock *sk = skb->sk;
2094
Oliver Hartkopp2244d072010-08-17 08:59:14 +00002095 if (sk && !skb_shinfo(skb)->tx_flags) {
Eric Dumazet87fd3082010-07-13 05:24:20 +00002096 /* skb_tx_hash() wont be able to get sk.
2097 * We copy sk_hash into skb->rxhash
2098 */
2099 if (!skb->rxhash)
2100 skb->rxhash = sk->sk_hash;
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002101 skb_orphan(skb);
Eric Dumazet87fd3082010-07-13 05:24:20 +00002102 }
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002103}
2104
Jesse Gross03634662011-01-09 06:23:35 +00002105static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2106{
2107 return ((features & NETIF_F_GEN_CSUM) ||
2108 ((features & NETIF_F_V4_CSUM) &&
2109 protocol == htons(ETH_P_IP)) ||
2110 ((features & NETIF_F_V6_CSUM) &&
2111 protocol == htons(ETH_P_IPV6)) ||
2112 ((features & NETIF_F_FCOE_CRC) &&
2113 protocol == htons(ETH_P_FCOE)));
2114}
2115
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002116static u32 harmonize_features(struct sk_buff *skb, __be16 protocol, u32 features)
Jesse Grossf01a5232011-01-09 06:23:31 +00002117{
Eric Dumazetd4027862011-01-19 00:51:36 +00002118 if (!can_checksum_protocol(features, protocol)) {
Jesse Grossf01a5232011-01-09 06:23:31 +00002119 features &= ~NETIF_F_ALL_CSUM;
2120 features &= ~NETIF_F_SG;
2121 } else if (illegal_highdma(skb->dev, skb)) {
2122 features &= ~NETIF_F_SG;
2123 }
2124
2125 return features;
2126}
2127
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002128u32 netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00002129{
2130 __be16 protocol = skb->protocol;
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002131 u32 features = skb->dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00002132
2133 if (protocol == htons(ETH_P_8021Q)) {
2134 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2135 protocol = veh->h_vlan_encapsulated_proto;
Jesse Grossf01a5232011-01-09 06:23:31 +00002136 } else if (!vlan_tx_tag_present(skb)) {
2137 return harmonize_features(skb, protocol, features);
2138 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002139
Jesse Gross6ee400a2011-01-17 20:46:00 +00002140 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
Jesse Grossf01a5232011-01-09 06:23:31 +00002141
2142 if (protocol != htons(ETH_P_8021Q)) {
2143 return harmonize_features(skb, protocol, features);
2144 } else {
2145 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
Jesse Gross6ee400a2011-01-17 20:46:00 +00002146 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
Jesse Grossf01a5232011-01-09 06:23:31 +00002147 return harmonize_features(skb, protocol, features);
2148 }
Jesse Gross58e998c2010-10-29 12:14:55 +00002149}
Jesse Grossf01a5232011-01-09 06:23:31 +00002150EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00002151
John Fastabend6afff0c2010-06-16 14:18:12 +00002152/*
2153 * Returns true if either:
2154 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2155 * 2. skb is fragmented and the device does not support SG, or if
2156 * at least one of fragments is in highmem and device does not
2157 * support DMA from it.
2158 */
2159static inline int skb_needs_linearize(struct sk_buff *skb,
Jesse Gross02932ce2011-01-09 06:23:34 +00002160 int features)
John Fastabend6afff0c2010-06-16 14:18:12 +00002161{
Jesse Gross02932ce2011-01-09 06:23:34 +00002162 return skb_is_nonlinear(skb) &&
2163 ((skb_has_frag_list(skb) &&
2164 !(features & NETIF_F_FRAGLIST)) ||
Jesse Grosse1e78db2010-10-29 12:14:53 +00002165 (skb_shinfo(skb)->nr_frags &&
Jesse Gross02932ce2011-01-09 06:23:34 +00002166 !(features & NETIF_F_SG)));
John Fastabend6afff0c2010-06-16 14:18:12 +00002167}
2168
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002169int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2170 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002171{
Stephen Hemminger00829822008-11-20 20:14:53 -08002172 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00002173 int rc = NETDEV_TX_OK;
Koki Sanagiec764bf2011-05-30 21:48:34 +00002174 unsigned int skb_len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002175
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002176 if (likely(!skb->next)) {
Michał Mirosław04ed3e72011-01-24 15:32:47 -08002177 u32 features;
Jesse Grossfc741212011-01-09 06:23:32 +00002178
Eric Dumazet93f154b2009-05-18 22:19:19 -07002179 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002180 * If device doesn't need skb->dst, release it right now while
Eric Dumazet93f154b2009-05-18 22:19:19 -07002181 * its hot in this cpu cache
2182 */
Eric Dumazetadf30902009-06-02 05:19:30 +00002183 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2184 skb_dst_drop(skb);
2185
Eric Dumazet15c2d75f2010-12-07 00:30:37 +00002186 if (!list_empty(&ptype_all))
2187 dev_queue_xmit_nit(skb, dev);
2188
Eric Dumazetfc6055a2010-04-16 12:18:22 +00002189 skb_orphan_try(skb);
David S. Miller9ccb8972010-04-22 01:02:07 -07002190
Jesse Grossfc741212011-01-09 06:23:32 +00002191 features = netif_skb_features(skb);
2192
Jesse Gross7b9c6092010-10-20 13:56:04 +00002193 if (vlan_tx_tag_present(skb) &&
Jesse Grossfc741212011-01-09 06:23:32 +00002194 !(features & NETIF_F_HW_VLAN_TX)) {
Jesse Gross7b9c6092010-10-20 13:56:04 +00002195 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2196 if (unlikely(!skb))
2197 goto out;
2198
2199 skb->vlan_tci = 0;
2200 }
2201
Jesse Grossfc741212011-01-09 06:23:32 +00002202 if (netif_needs_gso(skb, features)) {
Jesse Gross91ecb632011-01-09 06:23:33 +00002203 if (unlikely(dev_gso_segment(skb, features)))
David S. Miller9ccb8972010-04-22 01:02:07 -07002204 goto out_kfree_skb;
2205 if (skb->next)
2206 goto gso;
John Fastabend6afff0c2010-06-16 14:18:12 +00002207 } else {
Jesse Gross02932ce2011-01-09 06:23:34 +00002208 if (skb_needs_linearize(skb, features) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002209 __skb_linearize(skb))
2210 goto out_kfree_skb;
2211
2212 /* If packet is not checksummed and device does not
2213 * support checksumming for this protocol, complete
2214 * checksumming here.
2215 */
2216 if (skb->ip_summed == CHECKSUM_PARTIAL) {
Michał Mirosław55508d62010-12-14 15:24:08 +00002217 skb_set_transport_header(skb,
2218 skb_checksum_start_offset(skb));
Jesse Gross03634662011-01-09 06:23:35 +00002219 if (!(features & NETIF_F_ALL_CSUM) &&
John Fastabend6afff0c2010-06-16 14:18:12 +00002220 skb_checksum_help(skb))
2221 goto out_kfree_skb;
2222 }
David S. Miller9ccb8972010-04-22 01:02:07 -07002223 }
2224
Koki Sanagiec764bf2011-05-30 21:48:34 +00002225 skb_len = skb->len;
Patrick Ohlyac45f602009-02-12 05:03:37 +00002226 rc = ops->ndo_start_xmit(skb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002227 trace_net_dev_xmit(skb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002228 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07002229 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00002230 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002231 }
2232
Herbert Xu576a30e2006-06-27 13:22:38 -07002233gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002234 do {
2235 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002236
2237 skb->next = nskb->next;
2238 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00002239
2240 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002241 * If device doesn't need nskb->dst, release it right now while
Krishna Kumar068a2de2009-12-09 20:59:58 +00002242 * its hot in this cpu cache
2243 */
2244 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2245 skb_dst_drop(nskb);
2246
Koki Sanagiec764bf2011-05-30 21:48:34 +00002247 skb_len = nskb->len;
Stephen Hemminger00829822008-11-20 20:14:53 -08002248 rc = ops->ndo_start_xmit(nskb, dev);
Koki Sanagiec764bf2011-05-30 21:48:34 +00002249 trace_net_dev_xmit(nskb, rc, dev, skb_len);
Patrick McHardyec634fe2009-07-05 19:23:38 -07002250 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002251 if (rc & ~NETDEV_TX_MASK)
2252 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07002253 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002254 skb->next = nskb;
2255 return rc;
2256 }
Eric Dumazet08baf562009-05-25 22:58:01 -07002257 txq_trans_update(txq);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002258 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07002259 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002260 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002261
Patrick McHardy572a9d72009-11-10 06:14:14 +00002262out_kfree_gso_skb:
2263 if (likely(skb->next == NULL))
2264 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002265out_kfree_skb:
2266 kfree_skb(skb);
Jesse Gross7b9c6092010-10-20 13:56:04 +00002267out:
Patrick McHardy572a9d72009-11-10 06:14:14 +00002268 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002269}
2270
Tom Herbert0a9627f2010-03-16 08:03:29 +00002271static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002272
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002273/*
2274 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2275 * to be used as a distribution range.
2276 */
2277u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2278 unsigned int num_tx_queues)
David S. Miller8f0f2222008-07-15 03:47:03 -07002279{
David S. Miller70192982009-01-27 16:34:47 -08002280 u32 hash;
John Fastabend4f57c082011-01-17 08:06:04 +00002281 u16 qoffset = 0;
2282 u16 qcount = num_tx_queues;
David S. Millerb6b2fed2008-07-21 09:48:06 -07002283
David S. Miller513de112009-05-03 14:43:10 -07002284 if (skb_rx_queue_recorded(skb)) {
2285 hash = skb_get_rx_queue(skb);
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002286 while (unlikely(hash >= num_tx_queues))
2287 hash -= num_tx_queues;
David S. Miller513de112009-05-03 14:43:10 -07002288 return hash;
2289 }
Eric Dumazetec581f62009-05-01 09:05:06 -07002290
John Fastabend4f57c082011-01-17 08:06:04 +00002291 if (dev->num_tc) {
2292 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2293 qoffset = dev->tc_to_txq[tc].offset;
2294 qcount = dev->tc_to_txq[tc].count;
2295 }
2296
Eric Dumazetec581f62009-05-01 09:05:06 -07002297 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08002298 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07002299 else
Eric Dumazet87fd3082010-07-13 05:24:20 +00002300 hash = (__force u16) skb->protocol ^ skb->rxhash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002301 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08002302
John Fastabend4f57c082011-01-17 08:06:04 +00002303 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
David S. Miller8f0f2222008-07-15 03:47:03 -07002304}
Vladislav Zolotarova3d22a62010-12-13 06:27:10 +00002305EXPORT_SYMBOL(__skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07002306
Eric Dumazeted046422009-11-13 21:54:04 +00002307static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2308{
2309 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2310 if (net_ratelimit()) {
Eric Dumazet7a161ea2010-04-08 21:26:13 +00002311 pr_warning("%s selects TX queue %d, but "
2312 "real number of TX queues is %d\n",
2313 dev->name, queue_index, dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00002314 }
2315 return 0;
2316 }
2317 return queue_index;
2318}
2319
Tom Herbert1d24eb42010-11-21 13:17:27 +00002320static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2321{
Tom Herbertbf264142010-11-26 08:36:09 +00002322#ifdef CONFIG_XPS
Tom Herbert1d24eb42010-11-21 13:17:27 +00002323 struct xps_dev_maps *dev_maps;
2324 struct xps_map *map;
2325 int queue_index = -1;
2326
2327 rcu_read_lock();
2328 dev_maps = rcu_dereference(dev->xps_maps);
2329 if (dev_maps) {
2330 map = rcu_dereference(
2331 dev_maps->cpu_map[raw_smp_processor_id()]);
2332 if (map) {
2333 if (map->len == 1)
2334 queue_index = map->queues[0];
2335 else {
2336 u32 hash;
2337 if (skb->sk && skb->sk->sk_hash)
2338 hash = skb->sk->sk_hash;
2339 else
2340 hash = (__force u16) skb->protocol ^
2341 skb->rxhash;
2342 hash = jhash_1word(hash, hashrnd);
2343 queue_index = map->queues[
2344 ((u64)hash * map->len) >> 32];
2345 }
2346 if (unlikely(queue_index >= dev->real_num_tx_queues))
2347 queue_index = -1;
2348 }
2349 }
2350 rcu_read_unlock();
2351
2352 return queue_index;
2353#else
2354 return -1;
2355#endif
2356}
2357
David S. Millere8a04642008-07-17 00:34:19 -07002358static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2359 struct sk_buff *skb)
2360{
Tom Herbertb0f77d02010-07-14 20:50:29 -07002361 int queue_index;
Helmut Schaadeabc772010-09-03 02:39:56 +00002362 const struct net_device_ops *ops = dev->netdev_ops;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002363
Tom Herbert3853b582010-11-21 13:17:29 +00002364 if (dev->real_num_tx_queues == 1)
2365 queue_index = 0;
2366 else if (ops->ndo_select_queue) {
Helmut Schaadeabc772010-09-03 02:39:56 +00002367 queue_index = ops->ndo_select_queue(dev, skb);
2368 queue_index = dev_cap_txqueue(dev, queue_index);
2369 } else {
2370 struct sock *sk = skb->sk;
2371 queue_index = sk_tx_queue_get(sk);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002372
Tom Herbert3853b582010-11-21 13:17:29 +00002373 if (queue_index < 0 || skb->ooo_okay ||
2374 queue_index >= dev->real_num_tx_queues) {
2375 int old_index = queue_index;
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002376
Tom Herbert1d24eb42010-11-21 13:17:27 +00002377 queue_index = get_xps_queue(dev, skb);
2378 if (queue_index < 0)
2379 queue_index = skb_tx_hash(dev, skb);
Tom Herbert3853b582010-11-21 13:17:29 +00002380
2381 if (queue_index != old_index && sk) {
2382 struct dst_entry *dst =
2383 rcu_dereference_check(sk->sk_dst_cache, 1);
Eric Dumazet8728c542010-04-11 21:18:17 +00002384
2385 if (dst && skb_dst(skb) == dst)
2386 sk_tx_queue_set(sk, queue_index);
2387 }
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00002388 }
2389 }
David S. Millereae792b2008-07-15 03:03:33 -07002390
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002391 skb_set_queue_mapping(skb, queue_index);
2392 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002393}
2394
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002395static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2396 struct net_device *dev,
2397 struct netdev_queue *txq)
2398{
2399 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002400 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002401 int rc;
2402
Eric Dumazeta2da5702011-01-20 03:48:19 +00002403 qdisc_skb_cb(skb)->pkt_len = skb->len;
2404 qdisc_calculate_pkt_len(skb, q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002405 /*
2406 * Heuristic to force contended enqueues to serialize on a
2407 * separate lock before trying to get qdisc main lock.
2408 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2409 * and dequeue packets faster.
2410 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00002411 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002412 if (unlikely(contended))
2413 spin_lock(&q->busylock);
2414
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002415 spin_lock(root_lock);
2416 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2417 kfree_skb(skb);
2418 rc = NET_XMIT_DROP;
2419 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07002420 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002421 /*
2422 * This is a work-conserving queue; there are no old skbs
2423 * waiting to be sent out; and the qdisc is not running -
2424 * xmit the skb directly.
2425 */
Eric Dumazet7fee2262010-05-11 23:19:48 +00002426 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2427 skb_dst_force(skb);
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002428
Eric Dumazetbfe0d022011-01-09 08:30:54 +00002429 qdisc_bstats_update(q, skb);
2430
Eric Dumazet79640a42010-06-02 05:09:29 -07002431 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2432 if (unlikely(contended)) {
2433 spin_unlock(&q->busylock);
2434 contended = false;
2435 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002436 __qdisc_run(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07002437 } else
Eric Dumazetbc135b22010-06-02 03:23:51 -07002438 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002439
2440 rc = NET_XMIT_SUCCESS;
2441 } else {
Eric Dumazet7fee2262010-05-11 23:19:48 +00002442 skb_dst_force(skb);
Eric Dumazeta2da5702011-01-20 03:48:19 +00002443 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07002444 if (qdisc_run_begin(q)) {
2445 if (unlikely(contended)) {
2446 spin_unlock(&q->busylock);
2447 contended = false;
2448 }
2449 __qdisc_run(q);
2450 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002451 }
2452 spin_unlock(root_lock);
Eric Dumazet79640a42010-06-02 05:09:29 -07002453 if (unlikely(contended))
2454 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002455 return rc;
2456}
2457
Eric Dumazet745e20f2010-09-29 13:23:09 -07002458static DEFINE_PER_CPU(int, xmit_recursion);
David S. Miller11a766c2010-10-25 12:51:55 -07002459#define RECURSION_LIMIT 10
Eric Dumazet745e20f2010-09-29 13:23:09 -07002460
Dave Jonesd29f7492008-07-22 14:09:06 -07002461/**
2462 * dev_queue_xmit - transmit a buffer
2463 * @skb: buffer to transmit
2464 *
2465 * Queue a buffer for transmission to a network device. The caller must
2466 * have set the device and priority and built the buffer before calling
2467 * this function. The function can be called from an interrupt.
2468 *
2469 * A negative errno code is returned on a failure. A success does not
2470 * guarantee the frame will be transmitted as it may be dropped due
2471 * to congestion or traffic shaping.
2472 *
2473 * -----------------------------------------------------------------------------------
2474 * I notice this method can also return errors from the queue disciplines,
2475 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2476 * be positive.
2477 *
2478 * Regardless of the return value, the skb is consumed, so it is currently
2479 * difficult to retry a send to this method. (You can bump the ref count
2480 * before sending to hold a reference for retry if you are careful.)
2481 *
2482 * When calling this method, interrupts MUST be enabled. This is because
2483 * the BH enable code must have IRQs enabled so that it will not deadlock.
2484 * --BLG
2485 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002486int dev_queue_xmit(struct sk_buff *skb)
2487{
2488 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002489 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490 struct Qdisc *q;
2491 int rc = -ENOMEM;
2492
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002493 /* Disable soft irqs for various locks below. Also
2494 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002495 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002496 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002497
David S. Millereae792b2008-07-15 03:03:33 -07002498 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002499 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002500
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002502 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002503#endif
Koki Sanagicf66ba52010-08-23 18:45:02 +09002504 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002506 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002507 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002508 }
2509
2510 /* The device has no queue. Common case for software devices:
2511 loopback, all the sorts of tunnels...
2512
Herbert Xu932ff272006-06-09 12:20:56 -07002513 Really, it is unlikely that netif_tx_lock protection is necessary
2514 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002515 counters.)
2516 However, it is possible, that they rely on protection
2517 made by us here.
2518
2519 Check this and shot the lock. It is not prone from deadlocks.
2520 Either shot noqueue qdisc, it is even simpler 8)
2521 */
2522 if (dev->flags & IFF_UP) {
2523 int cpu = smp_processor_id(); /* ok because BHs are off */
2524
David S. Millerc773e842008-07-08 23:13:53 -07002525 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002526
Eric Dumazet745e20f2010-09-29 13:23:09 -07002527 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2528 goto recursion_alert;
2529
David S. Millerc773e842008-07-08 23:13:53 -07002530 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002531
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002532 if (!netif_tx_queue_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07002533 __this_cpu_inc(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002534 rc = dev_hard_start_xmit(skb, dev, txq);
Eric Dumazet745e20f2010-09-29 13:23:09 -07002535 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00002536 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002537 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002538 goto out;
2539 }
2540 }
David S. Millerc773e842008-07-08 23:13:53 -07002541 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002542 if (net_ratelimit())
2543 printk(KERN_CRIT "Virtual device %s asks to "
2544 "queue packet!\n", dev->name);
2545 } else {
2546 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07002547 * unfortunately
2548 */
2549recursion_alert:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550 if (net_ratelimit())
2551 printk(KERN_CRIT "Dead loop on virtual device "
2552 "%s, fix it urgently!\n", dev->name);
2553 }
2554 }
2555
2556 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002557 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002558
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559 kfree_skb(skb);
2560 return rc;
2561out:
Herbert Xud4828d82006-06-22 02:28:18 -07002562 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002563 return rc;
2564}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002565EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002566
2567
2568/*=======================================================================
2569 Receiver routines
2570 =======================================================================*/
2571
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002572int netdev_max_backlog __read_mostly = 1000;
Eric Dumazet3b098e22010-05-15 23:57:10 -07002573int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002574int netdev_budget __read_mostly = 300;
2575int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002576
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002577/* Called with irq disabled */
2578static inline void ____napi_schedule(struct softnet_data *sd,
2579 struct napi_struct *napi)
2580{
2581 list_add_tail(&napi->poll_list, &sd->poll_list);
2582 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2583}
2584
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002585/*
2586 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
Tom Herbertbdeab992011-08-14 19:45:55 +00002587 * and src/dst port numbers. Sets rxhash in skb to non-zero hash value
2588 * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
2589 * if hash is a canonical 4-tuple hash over transport ports.
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002590 */
Tom Herbertbdeab992011-08-14 19:45:55 +00002591void __skb_get_rxhash(struct sk_buff *skb)
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002592{
Changli Gao12fcdef2010-08-17 19:04:32 +00002593 int nhoff, hash = 0, poff;
Eric Dumazetb71d1d42011-04-22 04:53:02 +00002594 const struct ipv6hdr *ip6;
2595 const struct iphdr *ip;
Changli Gao1ff19862011-08-18 22:07:54 -07002596 const struct vlan_hdr *vlan;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002597 u8 ip_proto;
Tom Herbert792df222011-08-14 19:45:04 +00002598 u32 addr1, addr2;
2599 u16 proto;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002600 union {
2601 u32 v32;
2602 u16 v16[2];
2603 } ports;
2604
2605 nhoff = skb_network_offset(skb);
Tom Herbert792df222011-08-14 19:45:04 +00002606 proto = skb->protocol;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002607
Tom Herberte971b722011-08-14 19:46:12 +00002608again:
Tom Herbert792df222011-08-14 19:45:04 +00002609 switch (proto) {
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002610 case __constant_htons(ETH_P_IP):
Changli Gao5dd17e02011-09-20 22:36:07 +00002611ip:
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002612 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2613 goto done;
2614
Eric Dumazetb71d1d42011-04-22 04:53:02 +00002615 ip = (const struct iphdr *) (skb->data + nhoff);
Paul Gortmaker56f8a752011-06-21 20:33:34 -07002616 if (ip_is_fragment(ip))
Changli Gaodbe57752010-08-17 19:01:38 +00002617 ip_proto = 0;
2618 else
2619 ip_proto = ip->protocol;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002620 addr1 = (__force u32) ip->saddr;
2621 addr2 = (__force u32) ip->daddr;
Tom Herbert792df222011-08-14 19:45:04 +00002622 nhoff += ip->ihl * 4;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002623 break;
2624 case __constant_htons(ETH_P_IPV6):
Changli Gao5dd17e02011-09-20 22:36:07 +00002625ipv6:
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002626 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2627 goto done;
2628
Eric Dumazetb71d1d42011-04-22 04:53:02 +00002629 ip6 = (const struct ipv6hdr *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002630 ip_proto = ip6->nexthdr;
2631 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2632 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
Tom Herbert792df222011-08-14 19:45:04 +00002633 nhoff += 40;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002634 break;
Changli Gao1ff19862011-08-18 22:07:54 -07002635 case __constant_htons(ETH_P_8021Q):
2636 if (!pskb_may_pull(skb, sizeof(*vlan) + nhoff))
2637 goto done;
2638 vlan = (const struct vlan_hdr *) (skb->data + nhoff);
2639 proto = vlan->h_vlan_encapsulated_proto;
2640 nhoff += sizeof(*vlan);
2641 goto again;
Changli Gaoae1511b2011-08-18 23:23:47 -07002642 case __constant_htons(ETH_P_PPP_SES):
2643 if (!pskb_may_pull(skb, PPPOE_SES_HLEN + nhoff))
2644 goto done;
2645 proto = *((__be16 *) (skb->data + nhoff +
2646 sizeof(struct pppoe_hdr)));
2647 nhoff += PPPOE_SES_HLEN;
Changli Gao5dd17e02011-09-20 22:36:07 +00002648 switch (proto) {
2649 case __constant_htons(PPP_IP):
2650 goto ip;
2651 case __constant_htons(PPP_IPV6):
2652 goto ipv6;
2653 default:
2654 goto done;
2655 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002656 default:
2657 goto done;
2658 }
2659
Tom Herberte971b722011-08-14 19:46:12 +00002660 switch (ip_proto) {
Tom Herbertc6865cb2011-08-14 19:46:29 +00002661 case IPPROTO_GRE:
2662 if (pskb_may_pull(skb, nhoff + 16)) {
2663 u8 *h = skb->data + nhoff;
2664 __be16 flags = *(__be16 *)h;
2665
2666 /*
2667 * Only look inside GRE if version zero and no
2668 * routing
2669 */
2670 if (!(flags & (GRE_VERSION|GRE_ROUTING))) {
2671 proto = *(__be16 *)(h + 2);
2672 nhoff += 4;
2673 if (flags & GRE_CSUM)
2674 nhoff += 4;
2675 if (flags & GRE_KEY)
2676 nhoff += 4;
2677 if (flags & GRE_SEQ)
2678 nhoff += 4;
2679 goto again;
2680 }
2681 }
2682 break;
Eric Dumazetec5efe72011-08-24 10:41:19 +00002683 case IPPROTO_IPIP:
2684 goto again;
Tom Herberte971b722011-08-14 19:46:12 +00002685 default:
2686 break;
2687 }
2688
Changli Gao12fcdef2010-08-17 19:04:32 +00002689 ports.v32 = 0;
2690 poff = proto_ports_offset(ip_proto);
2691 if (poff >= 0) {
Tom Herbert792df222011-08-14 19:45:04 +00002692 nhoff += poff;
Changli Gao12fcdef2010-08-17 19:04:32 +00002693 if (pskb_may_pull(skb, nhoff + 4)) {
2694 ports.v32 = * (__force u32 *) (skb->data + nhoff);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002695 if (ports.v16[1] < ports.v16[0])
2696 swap(ports.v16[0], ports.v16[1]);
Tom Herbertbdeab992011-08-14 19:45:55 +00002697 skb->l4_rxhash = 1;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002698 }
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002699 }
2700
2701 /* get a consistent hash (same value on both flow directions) */
2702 if (addr2 < addr1)
2703 swap(addr1, addr2);
2704
2705 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2706 if (!hash)
2707 hash = 1;
2708
2709done:
Tom Herbertbdeab992011-08-14 19:45:55 +00002710 skb->rxhash = hash;
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002711}
2712EXPORT_SYMBOL(__skb_get_rxhash);
2713
Eric Dumazetdf334542010-03-24 19:13:54 +00002714#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07002715
2716/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002717struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07002718EXPORT_SYMBOL(rps_sock_flow_table);
2719
Ben Hutchingsc4454772011-01-19 11:03:53 +00002720static struct rps_dev_flow *
2721set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2722 struct rps_dev_flow *rflow, u16 next_cpu)
2723{
Ben Hutchings09994d12011-10-03 04:42:46 +00002724 if (next_cpu != RPS_NO_CPU) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00002725#ifdef CONFIG_RFS_ACCEL
2726 struct netdev_rx_queue *rxqueue;
2727 struct rps_dev_flow_table *flow_table;
2728 struct rps_dev_flow *old_rflow;
2729 u32 flow_id;
2730 u16 rxq_index;
2731 int rc;
2732
2733 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00002734 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2735 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00002736 goto out;
2737 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2738 if (rxq_index == skb_get_rx_queue(skb))
2739 goto out;
2740
2741 rxqueue = dev->_rx + rxq_index;
2742 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2743 if (!flow_table)
2744 goto out;
2745 flow_id = skb->rxhash & flow_table->mask;
2746 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2747 rxq_index, flow_id);
2748 if (rc < 0)
2749 goto out;
2750 old_rflow = rflow;
2751 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00002752 rflow->filter = rc;
2753 if (old_rflow->filter == rflow->filter)
2754 old_rflow->filter = RPS_NO_FILTER;
2755 out:
2756#endif
2757 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00002758 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002759 }
2760
Ben Hutchings09994d12011-10-03 04:42:46 +00002761 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00002762 return rflow;
2763}
2764
Tom Herbert0a9627f2010-03-16 08:03:29 +00002765/*
2766 * get_rps_cpu is called from netif_receive_skb and returns the target
2767 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002768 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00002769 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002770static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2771 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002772{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002773 struct netdev_rx_queue *rxqueue;
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002774 struct rps_map *map;
Tom Herbertfec5e652010-04-16 16:01:27 -07002775 struct rps_dev_flow_table *flow_table;
2776 struct rps_sock_flow_table *sock_flow_table;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002777 int cpu = -1;
Tom Herbertfec5e652010-04-16 16:01:27 -07002778 u16 tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002779
Tom Herbert0a9627f2010-03-16 08:03:29 +00002780 if (skb_rx_queue_recorded(skb)) {
2781 u16 index = skb_get_rx_queue(skb);
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002782 if (unlikely(index >= dev->real_num_rx_queues)) {
2783 WARN_ONCE(dev->real_num_rx_queues > 1,
2784 "%s received packet on queue %u, but number "
2785 "of RX queues is %u\n",
2786 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002787 goto done;
2788 }
2789 rxqueue = dev->_rx + index;
2790 } else
2791 rxqueue = dev->_rx;
2792
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00002793 map = rcu_dereference(rxqueue->rps_map);
2794 if (map) {
Tom Herbert85875232011-01-31 16:23:42 -08002795 if (map->len == 1 &&
Eric Dumazet33d480c2011-08-11 19:30:52 +00002796 !rcu_access_pointer(rxqueue->rps_flow_table)) {
Changli Gao6febfca2010-09-03 23:12:37 +00002797 tcpu = map->cpus[0];
2798 if (cpu_online(tcpu))
2799 cpu = tcpu;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002800 goto done;
Eric Dumazetb249dcb2010-04-19 21:56:38 +00002801 }
Eric Dumazet33d480c2011-08-11 19:30:52 +00002802 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002803 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002804 }
2805
Changli Gao2d47b452010-08-17 19:00:56 +00002806 skb_reset_network_header(skb);
Krishna Kumarbfb564e2010-08-04 06:15:52 +00002807 if (!skb_get_rxhash(skb))
Tom Herbert0a9627f2010-03-16 08:03:29 +00002808 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002809
Tom Herbertfec5e652010-04-16 16:01:27 -07002810 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2811 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2812 if (flow_table && sock_flow_table) {
2813 u16 next_cpu;
2814 struct rps_dev_flow *rflow;
2815
2816 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2817 tcpu = rflow->cpu;
2818
2819 next_cpu = sock_flow_table->ents[skb->rxhash &
2820 sock_flow_table->mask];
2821
2822 /*
2823 * If the desired CPU (where last recvmsg was done) is
2824 * different from current CPU (one in the rx-queue flow
2825 * table entry), switch if one of the following holds:
2826 * - Current CPU is unset (equal to RPS_NO_CPU).
2827 * - Current CPU is offline.
2828 * - The current CPU's queue tail has advanced beyond the
2829 * last packet that was enqueued using this table entry.
2830 * This guarantees that all previous packets for the flow
2831 * have been dequeued, thus preserving in order delivery.
2832 */
2833 if (unlikely(tcpu != next_cpu) &&
2834 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2835 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Ben Hutchingsc4454772011-01-19 11:03:53 +00002836 rflow->last_qtail)) >= 0))
2837 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2838
Tom Herbertfec5e652010-04-16 16:01:27 -07002839 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2840 *rflowp = rflow;
2841 cpu = tcpu;
2842 goto done;
2843 }
2844 }
2845
Tom Herbert0a9627f2010-03-16 08:03:29 +00002846 if (map) {
Tom Herbertfec5e652010-04-16 16:01:27 -07002847 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
Tom Herbert0a9627f2010-03-16 08:03:29 +00002848
2849 if (cpu_online(tcpu)) {
2850 cpu = tcpu;
2851 goto done;
2852 }
2853 }
2854
2855done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00002856 return cpu;
2857}
2858
Ben Hutchingsc4454772011-01-19 11:03:53 +00002859#ifdef CONFIG_RFS_ACCEL
2860
2861/**
2862 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2863 * @dev: Device on which the filter was set
2864 * @rxq_index: RX queue index
2865 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2866 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2867 *
2868 * Drivers that implement ndo_rx_flow_steer() should periodically call
2869 * this function for each installed filter and remove the filters for
2870 * which it returns %true.
2871 */
2872bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2873 u32 flow_id, u16 filter_id)
2874{
2875 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2876 struct rps_dev_flow_table *flow_table;
2877 struct rps_dev_flow *rflow;
2878 bool expire = true;
2879 int cpu;
2880
2881 rcu_read_lock();
2882 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2883 if (flow_table && flow_id <= flow_table->mask) {
2884 rflow = &flow_table->flows[flow_id];
2885 cpu = ACCESS_ONCE(rflow->cpu);
2886 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2887 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2888 rflow->last_qtail) <
2889 (int)(10 * flow_table->mask)))
2890 expire = false;
2891 }
2892 rcu_read_unlock();
2893 return expire;
2894}
2895EXPORT_SYMBOL(rps_may_expire_flow);
2896
2897#endif /* CONFIG_RFS_ACCEL */
2898
Tom Herbert0a9627f2010-03-16 08:03:29 +00002899/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002900static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002901{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002902 struct softnet_data *sd = data;
2903
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002904 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00002905 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002906}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002907
Tom Herbertfec5e652010-04-16 16:01:27 -07002908#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002909
2910/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002911 * Check if this softnet_data structure is another cpu one
2912 * If yes, queue it to our IPI list and return 1
2913 * If no, return 0
2914 */
2915static int rps_ipi_queued(struct softnet_data *sd)
2916{
2917#ifdef CONFIG_RPS
2918 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2919
2920 if (sd != mysd) {
2921 sd->rps_ipi_next = mysd->rps_ipi_list;
2922 mysd->rps_ipi_list = sd;
2923
2924 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2925 return 1;
2926 }
2927#endif /* CONFIG_RPS */
2928 return 0;
2929}
2930
2931/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00002932 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2933 * queue (may be a remote CPU queue).
2934 */
Tom Herbertfec5e652010-04-16 16:01:27 -07002935static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2936 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00002937{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002938 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00002939 unsigned long flags;
2940
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002941 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002942
2943 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002944
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002945 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07002946 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2947 if (skb_queue_len(&sd->input_pkt_queue)) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00002948enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002949 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00002950 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002951 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00002952 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002953 return NET_RX_SUCCESS;
2954 }
2955
Eric Dumazetebda37c22010-05-06 23:51:21 +00002956 /* Schedule NAPI for backlog device
2957 * We can use non atomic operation since we own the queue lock
2958 */
2959 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002960 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07002961 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002962 }
2963 goto enqueue;
2964 }
2965
Changli Gaodee42872010-05-02 05:42:16 +00002966 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00002967 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002968
Tom Herbert0a9627f2010-03-16 08:03:29 +00002969 local_irq_restore(flags);
2970
Eric Dumazetcaf586e2010-09-30 21:06:55 +00002971 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00002972 kfree_skb(skb);
2973 return NET_RX_DROP;
2974}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002975
Linus Torvalds1da177e2005-04-16 15:20:36 -07002976/**
2977 * netif_rx - post buffer to the network code
2978 * @skb: buffer to post
2979 *
2980 * This function receives a packet from a device driver and queues it for
2981 * the upper (protocol) levels to process. It always succeeds. The buffer
2982 * may be dropped during processing for congestion control or by the
2983 * protocol layers.
2984 *
2985 * return values:
2986 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002987 * NET_RX_DROP (packet was dropped)
2988 *
2989 */
2990
2991int netif_rx(struct sk_buff *skb)
2992{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07002993 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002994
2995 /* if netpoll wants it, pretend we never saw it */
2996 if (netpoll_rx(skb))
2997 return NET_RX_DROP;
2998
Eric Dumazet588f0332011-11-15 04:12:55 +00002999 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003000
Koki Sanagicf66ba52010-08-23 18:45:02 +09003001 trace_netif_rx(skb);
Eric Dumazetdf334542010-03-24 19:13:54 +00003002#ifdef CONFIG_RPS
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003003 {
Tom Herbertfec5e652010-04-16 16:01:27 -07003004 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003005 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003006
Changli Gaocece1942010-08-07 20:35:43 -07003007 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003008 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07003009
3010 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003011 if (cpu < 0)
3012 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07003013
3014 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3015
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003016 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07003017 preempt_enable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003018 }
3019#else
Tom Herbertfec5e652010-04-16 16:01:27 -07003020 {
3021 unsigned int qtail;
3022 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3023 put_cpu();
3024 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07003025#endif
3026 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003027}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003028EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003029
3030int netif_rx_ni(struct sk_buff *skb)
3031{
3032 int err;
3033
3034 preempt_disable();
3035 err = netif_rx(skb);
3036 if (local_softirq_pending())
3037 do_softirq();
3038 preempt_enable();
3039
3040 return err;
3041}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003042EXPORT_SYMBOL(netif_rx_ni);
3043
Linus Torvalds1da177e2005-04-16 15:20:36 -07003044static void net_tx_action(struct softirq_action *h)
3045{
3046 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3047
3048 if (sd->completion_queue) {
3049 struct sk_buff *clist;
3050
3051 local_irq_disable();
3052 clist = sd->completion_queue;
3053 sd->completion_queue = NULL;
3054 local_irq_enable();
3055
3056 while (clist) {
3057 struct sk_buff *skb = clist;
3058 clist = clist->next;
3059
Ilpo Järvinen547b7922008-07-25 21:43:18 -07003060 WARN_ON(atomic_read(&skb->users));
Koki Sanagi07dc22e2010-08-23 18:46:12 +09003061 trace_kfree_skb(skb, net_tx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062 __kfree_skb(skb);
3063 }
3064 }
3065
3066 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07003067 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003068
3069 local_irq_disable();
3070 head = sd->output_queue;
3071 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00003072 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073 local_irq_enable();
3074
3075 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07003076 struct Qdisc *q = head;
3077 spinlock_t *root_lock;
3078
Linus Torvalds1da177e2005-04-16 15:20:36 -07003079 head = head->next_sched;
3080
David S. Miller5fb66222008-08-02 20:02:43 -07003081 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07003082 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07003083 smp_mb__before_clear_bit();
3084 clear_bit(__QDISC_STATE_SCHED,
3085 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07003086 qdisc_run(q);
3087 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088 } else {
David S. Miller195648b2008-08-19 04:00:36 -07003089 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003090 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07003091 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07003092 } else {
3093 smp_mb__before_clear_bit();
3094 clear_bit(__QDISC_STATE_SCHED,
3095 &q->state);
3096 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003097 }
3098 }
3099 }
3100}
3101
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003102#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3103 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
Michał Mirosławda678292009-06-05 05:35:28 +00003104/* This hook is defined here for ATM LANE */
3105int (*br_fdb_test_addr_hook)(struct net_device *dev,
3106 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07003107EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00003108#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003109
Linus Torvalds1da177e2005-04-16 15:20:36 -07003110#ifdef CONFIG_NET_CLS_ACT
3111/* TODO: Maybe we should just force sch_ingress to be compiled in
3112 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3113 * a compare and 2 stores extra right now if we dont have it on
3114 * but have CONFIG_NET_CLS_ACT
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003115 * NOTE: This doesn't stop any functionality; if you dont have
3116 * the ingress scheduler, you just can't add policies on ingress.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003117 *
3118 */
Eric Dumazet24824a02010-10-02 06:11:55 +00003119static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003120{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003121 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003122 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07003123 int result = TC_ACT_OK;
3124 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003125
Stephen Hemmingerde384832010-08-01 00:33:23 -07003126 if (unlikely(MAX_RED_LOOP < ttl++)) {
3127 if (net_ratelimit())
3128 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
3129 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07003130 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003131 }
3132
Herbert Xuf697c3e2007-10-14 00:38:47 -07003133 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3134 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3135
David S. Miller83874002008-07-17 00:53:03 -07003136 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07003137 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07003138 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07003139 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3140 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07003141 spin_unlock(qdisc_lock(q));
3142 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07003143
Linus Torvalds1da177e2005-04-16 15:20:36 -07003144 return result;
3145}
Herbert Xuf697c3e2007-10-14 00:38:47 -07003146
3147static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3148 struct packet_type **pt_prev,
3149 int *ret, struct net_device *orig_dev)
3150{
Eric Dumazet24824a02010-10-02 06:11:55 +00003151 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3152
3153 if (!rxq || rxq->qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07003154 goto out;
3155
3156 if (*pt_prev) {
3157 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3158 *pt_prev = NULL;
Herbert Xuf697c3e2007-10-14 00:38:47 -07003159 }
3160
Eric Dumazet24824a02010-10-02 06:11:55 +00003161 switch (ing_filter(skb, rxq)) {
Herbert Xuf697c3e2007-10-14 00:38:47 -07003162 case TC_ACT_SHOT:
3163 case TC_ACT_STOLEN:
3164 kfree_skb(skb);
3165 return NULL;
3166 }
3167
3168out:
3169 skb->tc_verd = 0;
3170 return skb;
3171}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172#endif
3173
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003174/**
3175 * netdev_rx_handler_register - register receive handler
3176 * @dev: device to register a handler for
3177 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00003178 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003179 *
3180 * Register a receive hander for a device. This handler will then be
3181 * called from __netif_receive_skb. A negative errno code is returned
3182 * on a failure.
3183 *
3184 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003185 *
3186 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003187 */
3188int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00003189 rx_handler_func_t *rx_handler,
3190 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003191{
3192 ASSERT_RTNL();
3193
3194 if (dev->rx_handler)
3195 return -EBUSY;
3196
Jiri Pirko93e2c322010-06-10 03:34:59 +00003197 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003198 rcu_assign_pointer(dev->rx_handler, rx_handler);
3199
3200 return 0;
3201}
3202EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3203
3204/**
3205 * netdev_rx_handler_unregister - unregister receive handler
3206 * @dev: device to unregister a handler from
3207 *
3208 * Unregister a receive hander from a device.
3209 *
3210 * The caller must hold the rtnl_mutex.
3211 */
3212void netdev_rx_handler_unregister(struct net_device *dev)
3213{
3214
3215 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00003216 RCU_INIT_POINTER(dev->rx_handler, NULL);
3217 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003218}
3219EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3220
Eric Dumazet10f744d2010-03-28 23:07:20 -07003221static int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003222{
3223 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003224 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003225 struct net_device *orig_dev;
David S. Miller63d8ea72011-02-28 10:48:59 -08003226 struct net_device *null_or_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003227 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003228 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08003229 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003230
Eric Dumazet588f0332011-11-15 04:12:55 +00003231 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07003232
Koki Sanagicf66ba52010-08-23 18:45:02 +09003233 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08003234
Linus Torvalds1da177e2005-04-16 15:20:36 -07003235 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003236 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003237 return NET_RX_DROP;
3238
Eric Dumazet8964be42009-11-20 15:35:04 -08003239 if (!skb->skb_iif)
3240 skb->skb_iif = skb->dev->ifindex;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07003241 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00003242
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07003243 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03003244 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00003245 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003246
3247 pt_prev = NULL;
3248
3249 rcu_read_lock();
3250
David S. Miller63d8ea72011-02-28 10:48:59 -08003251another_round:
3252
3253 __this_cpu_inc(softnet_data.processed);
3254
Jiri Pirkobcc6d472011-04-07 19:48:33 +00003255 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3256 skb = vlan_untag(skb);
3257 if (unlikely(!skb))
3258 goto out;
3259 }
3260
Linus Torvalds1da177e2005-04-16 15:20:36 -07003261#ifdef CONFIG_NET_CLS_ACT
3262 if (skb->tc_verd & TC_NCLS) {
3263 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3264 goto ncls;
3265 }
3266#endif
3267
3268 list_for_each_entry_rcu(ptype, &ptype_all, list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003269 if (!ptype->dev || ptype->dev == skb->dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003270 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003271 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003272 pt_prev = ptype;
3273 }
3274 }
3275
3276#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07003277 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3278 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003279 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003280ncls:
3281#endif
3282
Eric Dumazet6a32e4f2011-10-29 06:13:39 +00003283 rx_handler = rcu_dereference(skb->dev->rx_handler);
John Fastabend24257172011-10-10 09:16:41 +00003284 if (vlan_tx_tag_present(skb)) {
3285 if (pt_prev) {
3286 ret = deliver_skb(skb, pt_prev, orig_dev);
3287 pt_prev = NULL;
3288 }
Eric Dumazet6a32e4f2011-10-29 06:13:39 +00003289 if (vlan_do_receive(&skb, !rx_handler))
John Fastabend24257172011-10-10 09:16:41 +00003290 goto another_round;
3291 else if (unlikely(!skb))
3292 goto out;
3293 }
3294
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003295 if (rx_handler) {
3296 if (pt_prev) {
3297 ret = deliver_skb(skb, pt_prev, orig_dev);
3298 pt_prev = NULL;
3299 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003300 switch (rx_handler(&skb)) {
3301 case RX_HANDLER_CONSUMED:
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003302 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003303 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08003304 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003305 case RX_HANDLER_EXACT:
3306 deliver_exact = true;
3307 case RX_HANDLER_PASS:
3308 break;
3309 default:
3310 BUG();
3311 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00003312 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003313
David S. Miller63d8ea72011-02-28 10:48:59 -08003314 /* deliver only exact match when indicated */
Jiri Pirko8a4eb572011-03-12 03:14:39 +00003315 null_or_dev = deliver_exact ? skb->dev : NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00003316
Linus Torvalds1da177e2005-04-16 15:20:36 -07003317 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003318 list_for_each_entry_rcu(ptype,
3319 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
David S. Miller63d8ea72011-02-28 10:48:59 -08003320 if (ptype->type == type &&
Jiri Pirkoe3f48d32011-02-28 20:26:31 +00003321 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3322 ptype->dev == orig_dev)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003323 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003324 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003325 pt_prev = ptype;
3326 }
3327 }
3328
3329 if (pt_prev) {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07003330 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003331 } else {
Eric Dumazetcaf586e2010-09-30 21:06:55 +00003332 atomic_long_inc(&skb->dev->rx_dropped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003333 kfree_skb(skb);
3334 /* Jamal, now you will not able to escape explaining
3335 * me how you were going to use this. :-)
3336 */
3337 ret = NET_RX_DROP;
3338 }
3339
3340out:
3341 rcu_read_unlock();
3342 return ret;
3343}
Tom Herbert0a9627f2010-03-16 08:03:29 +00003344
3345/**
3346 * netif_receive_skb - process receive buffer from network
3347 * @skb: buffer to process
3348 *
3349 * netif_receive_skb() is the main receive data processing function.
3350 * It always succeeds. The buffer may be dropped during processing
3351 * for congestion control or by the protocol layers.
3352 *
3353 * This function may only be called from softirq context and interrupts
3354 * should be enabled.
3355 *
3356 * Return values (usually ignored):
3357 * NET_RX_SUCCESS: no congestion
3358 * NET_RX_DROP: packet was dropped
3359 */
3360int netif_receive_skb(struct sk_buff *skb)
3361{
Eric Dumazet588f0332011-11-15 04:12:55 +00003362 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07003363
Richard Cochranc1f19b52010-07-17 08:49:36 +00003364 if (skb_defer_rx_timestamp(skb))
3365 return NET_RX_SUCCESS;
3366
Eric Dumazetdf334542010-03-24 19:13:54 +00003367#ifdef CONFIG_RPS
Eric Dumazet3b098e22010-05-15 23:57:10 -07003368 {
3369 struct rps_dev_flow voidflow, *rflow = &voidflow;
3370 int cpu, ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00003371
Eric Dumazet3b098e22010-05-15 23:57:10 -07003372 rcu_read_lock();
Tom Herbert0a9627f2010-03-16 08:03:29 +00003373
Eric Dumazet3b098e22010-05-15 23:57:10 -07003374 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07003375
Eric Dumazet3b098e22010-05-15 23:57:10 -07003376 if (cpu >= 0) {
3377 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3378 rcu_read_unlock();
3379 } else {
3380 rcu_read_unlock();
3381 ret = __netif_receive_skb(skb);
3382 }
3383
3384 return ret;
Tom Herbertfec5e652010-04-16 16:01:27 -07003385 }
Tom Herbert1e94d722010-03-18 17:45:44 -07003386#else
3387 return __netif_receive_skb(skb);
3388#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00003389}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003390EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003391
Eric Dumazet88751272010-04-19 05:07:33 +00003392/* Network device is going away, flush any packets still pending
3393 * Called with irqs disabled.
3394 */
Changli Gao152102c2010-03-30 20:16:22 +00003395static void flush_backlog(void *arg)
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003396{
Changli Gao152102c2010-03-30 20:16:22 +00003397 struct net_device *dev = arg;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003398 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003399 struct sk_buff *skb, *tmp;
3400
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003401 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003402 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003403 if (skb->dev == dev) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003404 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003405 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003406 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003407 }
Changli Gao6e7676c2010-04-27 15:07:33 -07003408 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00003409 rps_unlock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003410
3411 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3412 if (skb->dev == dev) {
3413 __skb_unlink(skb, &sd->process_queue);
3414 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003415 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07003416 }
3417 }
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07003418}
3419
Herbert Xud565b0a2008-12-15 23:38:52 -08003420static int napi_gro_complete(struct sk_buff *skb)
3421{
3422 struct packet_type *ptype;
3423 __be16 type = skb->protocol;
3424 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3425 int err = -ENOENT;
3426
Herbert Xufc59f9a2009-04-14 15:11:06 -07003427 if (NAPI_GRO_CB(skb)->count == 1) {
3428 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003429 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07003430 }
Herbert Xud565b0a2008-12-15 23:38:52 -08003431
3432 rcu_read_lock();
3433 list_for_each_entry_rcu(ptype, head, list) {
3434 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3435 continue;
3436
3437 err = ptype->gro_complete(skb);
3438 break;
3439 }
3440 rcu_read_unlock();
3441
3442 if (err) {
3443 WARN_ON(&ptype->list == head);
3444 kfree_skb(skb);
3445 return NET_RX_SUCCESS;
3446 }
3447
3448out:
Herbert Xud565b0a2008-12-15 23:38:52 -08003449 return netif_receive_skb(skb);
3450}
3451
Eric Dumazet86cac582010-08-31 18:25:32 +00003452inline void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08003453{
3454 struct sk_buff *skb, *next;
3455
3456 for (skb = napi->gro_list; skb; skb = next) {
3457 next = skb->next;
3458 skb->next = NULL;
3459 napi_gro_complete(skb);
3460 }
3461
Herbert Xu4ae55442009-02-08 18:00:36 +00003462 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003463 napi->gro_list = NULL;
3464}
Eric Dumazet86cac582010-08-31 18:25:32 +00003465EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08003466
Ben Hutchings5b252f02009-10-29 07:17:09 +00003467enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08003468{
3469 struct sk_buff **pp = NULL;
3470 struct packet_type *ptype;
3471 __be16 type = skb->protocol;
3472 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08003473 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08003474 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003475 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003476
Jarek Poplawskice9e76c2010-08-05 01:19:11 +00003477 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
Herbert Xud565b0a2008-12-15 23:38:52 -08003478 goto normal;
3479
David S. Miller21dc3302010-08-23 00:13:46 -07003480 if (skb_is_gso(skb) || skb_has_frag_list(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08003481 goto normal;
3482
Herbert Xud565b0a2008-12-15 23:38:52 -08003483 rcu_read_lock();
3484 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08003485 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3486 continue;
3487
Herbert Xu86911732009-01-29 14:19:50 +00003488 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08003489 mac_len = skb->network_header - skb->mac_header;
3490 skb->mac_len = mac_len;
3491 NAPI_GRO_CB(skb)->same_flow = 0;
3492 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08003493 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003494
Herbert Xud565b0a2008-12-15 23:38:52 -08003495 pp = ptype->gro_receive(&napi->gro_list, skb);
3496 break;
3497 }
3498 rcu_read_unlock();
3499
3500 if (&ptype->list == head)
3501 goto normal;
3502
Herbert Xu0da2afd52008-12-26 14:57:42 -08003503 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003504 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08003505
Herbert Xud565b0a2008-12-15 23:38:52 -08003506 if (pp) {
3507 struct sk_buff *nskb = *pp;
3508
3509 *pp = nskb->next;
3510 nskb->next = NULL;
3511 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00003512 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08003513 }
3514
Herbert Xu0da2afd52008-12-26 14:57:42 -08003515 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08003516 goto ok;
3517
Herbert Xu4ae55442009-02-08 18:00:36 +00003518 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08003519 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08003520
Herbert Xu4ae55442009-02-08 18:00:36 +00003521 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08003522 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00003523 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003524 skb->next = napi->gro_list;
3525 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003526 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08003527
Herbert Xuad0f9902009-02-01 01:24:55 -08003528pull:
Herbert Xucb189782009-05-26 18:50:31 +00003529 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3530 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3531
3532 BUG_ON(skb->end - skb->tail < grow);
3533
3534 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3535
3536 skb->tail += grow;
3537 skb->data_len -= grow;
3538
3539 skb_shinfo(skb)->frags[0].page_offset += grow;
Eric Dumazet9e903e02011-10-18 21:00:24 +00003540 skb_frag_size_sub(&skb_shinfo(skb)->frags[0], grow);
Herbert Xucb189782009-05-26 18:50:31 +00003541
Eric Dumazet9e903e02011-10-18 21:00:24 +00003542 if (unlikely(!skb_frag_size(&skb_shinfo(skb)->frags[0]))) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003543 skb_frag_unref(skb, 0);
Herbert Xucb189782009-05-26 18:50:31 +00003544 memmove(skb_shinfo(skb)->frags,
3545 skb_shinfo(skb)->frags + 1,
Jarek Poplawskie5093ae2010-08-11 02:02:10 +00003546 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
Herbert Xucb189782009-05-26 18:50:31 +00003547 }
Herbert Xuad0f9902009-02-01 01:24:55 -08003548 }
3549
Herbert Xud565b0a2008-12-15 23:38:52 -08003550ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003551 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08003552
3553normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08003554 ret = GRO_NORMAL;
3555 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08003556}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003557EXPORT_SYMBOL(dev_gro_receive);
3558
Eric Dumazet40d08022010-08-26 22:03:08 -07003559static inline gro_result_t
Ben Hutchings5b252f02009-10-29 07:17:09 +00003560__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003561{
3562 struct sk_buff *p;
3563
3564 for (p = napi->gro_list; p; p = p->next) {
Eric Dumazet40d08022010-08-26 22:03:08 -07003565 unsigned long diffs;
3566
3567 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Jesse Gross3701e512010-10-20 13:56:06 +00003568 diffs |= p->vlan_tci ^ skb->vlan_tci;
Eric Dumazet40d08022010-08-26 22:03:08 -07003569 diffs |= compare_ether_header(skb_mac_header(p),
Joe Perchesf64f9e72009-11-29 16:55:45 -08003570 skb_gro_mac_header(skb));
Eric Dumazet40d08022010-08-26 22:03:08 -07003571 NAPI_GRO_CB(p)->same_flow = !diffs;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003572 NAPI_GRO_CB(p)->flush = 0;
3573 }
3574
3575 return dev_gro_receive(napi, skb);
3576}
Herbert Xu5d38a072009-01-04 16:13:40 -08003577
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003578gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08003579{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003580 switch (ret) {
3581 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003582 if (netif_receive_skb(skb))
3583 ret = GRO_DROP;
3584 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003585
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003586 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003587 case GRO_MERGED_FREE:
Herbert Xu5d38a072009-01-04 16:13:40 -08003588 kfree_skb(skb);
3589 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003590
3591 case GRO_HELD:
3592 case GRO_MERGED:
3593 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08003594 }
3595
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003596 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003597}
3598EXPORT_SYMBOL(napi_skb_finish);
3599
Herbert Xu78a478d2009-05-26 18:50:21 +00003600void skb_gro_reset_offset(struct sk_buff *skb)
3601{
3602 NAPI_GRO_CB(skb)->data_offset = 0;
3603 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00003604 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00003605
Herbert Xu78d3fd02009-05-26 18:50:23 +00003606 if (skb->mac_header == skb->tail &&
Ian Campbellea2ab692011-08-22 23:44:58 +00003607 !PageHighMem(skb_frag_page(&skb_shinfo(skb)->frags[0]))) {
Herbert Xu78a478d2009-05-26 18:50:21 +00003608 NAPI_GRO_CB(skb)->frag0 =
Ian Campbellea2ab692011-08-22 23:44:58 +00003609 skb_frag_address(&skb_shinfo(skb)->frags[0]);
Eric Dumazet9e903e02011-10-18 21:00:24 +00003610 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(&skb_shinfo(skb)->frags[0]);
Herbert Xu74895942009-05-26 18:50:27 +00003611 }
Herbert Xu78a478d2009-05-26 18:50:21 +00003612}
3613EXPORT_SYMBOL(skb_gro_reset_offset);
3614
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003615gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003616{
Herbert Xu86911732009-01-29 14:19:50 +00003617 skb_gro_reset_offset(skb);
3618
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003619 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08003620}
3621EXPORT_SYMBOL(napi_gro_receive);
3622
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00003623static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003624{
Herbert Xu96e93ea2009-01-06 10:49:34 -08003625 __skb_pull(skb, skb_headlen(skb));
3626 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
Jesse Gross3701e512010-10-20 13:56:06 +00003627 skb->vlan_tci = 0;
Herbert Xu66c46d72011-01-29 20:44:54 -08003628 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08003629 skb->skb_iif = 0;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003630
3631 napi->skb = skb;
3632}
Herbert Xu96e93ea2009-01-06 10:49:34 -08003633
Herbert Xu76620aa2009-04-16 02:02:07 -07003634struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08003635{
Herbert Xu5d38a072009-01-04 16:13:40 -08003636 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003637
3638 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00003639 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3640 if (skb)
3641 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003642 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003643 return skb;
3644}
Herbert Xu76620aa2009-04-16 02:02:07 -07003645EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003646
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003647gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3648 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003649{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003650 switch (ret) {
3651 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003652 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003653 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003654
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003655 if (ret == GRO_HELD)
3656 skb_gro_pull(skb, -ETH_HLEN);
3657 else if (netif_receive_skb(skb))
3658 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003659 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003660
3661 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003662 case GRO_MERGED_FREE:
3663 napi_reuse_skb(napi, skb);
3664 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003665
3666 case GRO_MERGED:
3667 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003668 }
3669
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003670 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003671}
3672EXPORT_SYMBOL(napi_frags_finish);
3673
Herbert Xu76620aa2009-04-16 02:02:07 -07003674struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003675{
Herbert Xu76620aa2009-04-16 02:02:07 -07003676 struct sk_buff *skb = napi->skb;
3677 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003678 unsigned int hlen;
3679 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003680
3681 napi->skb = NULL;
3682
3683 skb_reset_mac_header(skb);
3684 skb_gro_reset_offset(skb);
3685
Herbert Xua5b1cf22009-05-26 18:50:28 +00003686 off = skb_gro_offset(skb);
3687 hlen = off + sizeof(*eth);
3688 eth = skb_gro_header_fast(skb, off);
3689 if (skb_gro_header_hard(skb, hlen)) {
3690 eth = skb_gro_header_slow(skb, hlen, off);
3691 if (unlikely(!eth)) {
3692 napi_reuse_skb(napi, skb);
3693 skb = NULL;
3694 goto out;
3695 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003696 }
3697
3698 skb_gro_pull(skb, sizeof(*eth));
3699
3700 /*
3701 * This works because the only protocols we care about don't require
3702 * special handling. We'll fix it up properly at the end.
3703 */
3704 skb->protocol = eth->h_proto;
3705
3706out:
3707 return skb;
3708}
3709EXPORT_SYMBOL(napi_frags_skb);
3710
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003711gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003712{
3713 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003714
3715 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003716 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003717
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003718 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003719}
3720EXPORT_SYMBOL(napi_gro_frags);
3721
Eric Dumazete326bed2010-04-22 00:22:45 -07003722/*
3723 * net_rps_action sends any pending IPI's for rps.
3724 * Note: called with local irq disabled, but exits with local irq enabled.
3725 */
3726static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3727{
3728#ifdef CONFIG_RPS
3729 struct softnet_data *remsd = sd->rps_ipi_list;
3730
3731 if (remsd) {
3732 sd->rps_ipi_list = NULL;
3733
3734 local_irq_enable();
3735
3736 /* Send pending IPI's to kick RPS processing on remote cpus. */
3737 while (remsd) {
3738 struct softnet_data *next = remsd->rps_ipi_next;
3739
3740 if (cpu_online(remsd->cpu))
3741 __smp_call_function_single(remsd->cpu,
3742 &remsd->csd, 0);
3743 remsd = next;
3744 }
3745 } else
3746#endif
3747 local_irq_enable();
3748}
3749
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003750static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003751{
3752 int work = 0;
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003753 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003754
Eric Dumazete326bed2010-04-22 00:22:45 -07003755#ifdef CONFIG_RPS
3756 /* Check if we have pending ipi, its better to send them now,
3757 * not waiting net_rx_action() end.
3758 */
3759 if (sd->rps_ipi_list) {
3760 local_irq_disable();
3761 net_rps_action_and_irq_enable(sd);
3762 }
3763#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003764 napi->weight = weight_p;
Changli Gao6e7676c2010-04-27 15:07:33 -07003765 local_irq_disable();
3766 while (work < quota) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003767 struct sk_buff *skb;
Changli Gao6e7676c2010-04-27 15:07:33 -07003768 unsigned int qlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003769
Changli Gao6e7676c2010-04-27 15:07:33 -07003770 while ((skb = __skb_dequeue(&sd->process_queue))) {
Eric Dumazete4008272010-04-05 15:42:39 -07003771 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07003772 __netif_receive_skb(skb);
Changli Gao6e7676c2010-04-27 15:07:33 -07003773 local_irq_disable();
Tom Herbert76cc8b12010-05-20 18:37:59 +00003774 input_queue_head_incr(sd);
3775 if (++work >= quota) {
3776 local_irq_enable();
3777 return work;
3778 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003779 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003780
Changli Gao6e7676c2010-04-27 15:07:33 -07003781 rps_lock(sd);
3782 qlen = skb_queue_len(&sd->input_pkt_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003783 if (qlen)
Changli Gao6e7676c2010-04-27 15:07:33 -07003784 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3785 &sd->process_queue);
Tom Herbert76cc8b12010-05-20 18:37:59 +00003786
Changli Gao6e7676c2010-04-27 15:07:33 -07003787 if (qlen < quota - work) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003788 /*
3789 * Inline a custom version of __napi_complete().
3790 * only current cpu owns and manipulates this napi,
3791 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3792 * we can use a plain write instead of clear_bit(),
3793 * and we dont need an smp_mb() memory barrier.
3794 */
3795 list_del(&napi->poll_list);
3796 napi->state = 0;
3797
Changli Gao6e7676c2010-04-27 15:07:33 -07003798 quota = work + qlen;
3799 }
3800 rps_unlock(sd);
3801 }
3802 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003803
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003804 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003805}
3806
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003807/**
3808 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003809 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003810 *
3811 * The entry's receive function will be scheduled to run
3812 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003813void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003814{
3815 unsigned long flags;
3816
3817 local_irq_save(flags);
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003818 ____napi_schedule(&__get_cpu_var(softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003819 local_irq_restore(flags);
3820}
3821EXPORT_SYMBOL(__napi_schedule);
3822
Herbert Xud565b0a2008-12-15 23:38:52 -08003823void __napi_complete(struct napi_struct *n)
3824{
3825 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3826 BUG_ON(n->gro_list);
3827
3828 list_del(&n->poll_list);
3829 smp_mb__before_clear_bit();
3830 clear_bit(NAPI_STATE_SCHED, &n->state);
3831}
3832EXPORT_SYMBOL(__napi_complete);
3833
3834void napi_complete(struct napi_struct *n)
3835{
3836 unsigned long flags;
3837
3838 /*
3839 * don't let napi dequeue from the cpu poll list
3840 * just in case its running on a different cpu
3841 */
3842 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3843 return;
3844
3845 napi_gro_flush(n);
3846 local_irq_save(flags);
3847 __napi_complete(n);
3848 local_irq_restore(flags);
3849}
3850EXPORT_SYMBOL(napi_complete);
3851
3852void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3853 int (*poll)(struct napi_struct *, int), int weight)
3854{
3855 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003856 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003857 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003858 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003859 napi->poll = poll;
3860 napi->weight = weight;
3861 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003862 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003863#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003864 spin_lock_init(&napi->poll_lock);
3865 napi->poll_owner = -1;
3866#endif
3867 set_bit(NAPI_STATE_SCHED, &napi->state);
3868}
3869EXPORT_SYMBOL(netif_napi_add);
3870
3871void netif_napi_del(struct napi_struct *napi)
3872{
3873 struct sk_buff *skb, *next;
3874
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003875 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003876 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003877
3878 for (skb = napi->gro_list; skb; skb = next) {
3879 next = skb->next;
3880 skb->next = NULL;
3881 kfree_skb(skb);
3882 }
3883
3884 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003885 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003886}
3887EXPORT_SYMBOL(netif_napi_del);
3888
Linus Torvalds1da177e2005-04-16 15:20:36 -07003889static void net_rx_action(struct softirq_action *h)
3890{
Eric Dumazete326bed2010-04-22 00:22:45 -07003891 struct softnet_data *sd = &__get_cpu_var(softnet_data);
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003892 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003893 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003894 void *have;
3895
Linus Torvalds1da177e2005-04-16 15:20:36 -07003896 local_irq_disable();
3897
Eric Dumazete326bed2010-04-22 00:22:45 -07003898 while (!list_empty(&sd->poll_list)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003899 struct napi_struct *n;
3900 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003901
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003902 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003903 * Allow this to run for 2 jiffies since which will allow
3904 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003905 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003906 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003907 goto softnet_break;
3908
3909 local_irq_enable();
3910
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003911 /* Even though interrupts have been re-enabled, this
3912 * access is safe because interrupts can only add new
3913 * entries to the tail of this list, and only ->poll()
3914 * calls can remove this head entry from the list.
3915 */
Eric Dumazete326bed2010-04-22 00:22:45 -07003916 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003917
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003918 have = netpoll_poll_lock(n);
3919
3920 weight = n->weight;
3921
David S. Miller0a7606c2007-10-29 21:28:47 -07003922 /* This NAPI_STATE_SCHED test is for avoiding a race
3923 * with netpoll's poll_napi(). Only the entity which
3924 * obtains the lock and sees NAPI_STATE_SCHED set will
3925 * actually make the ->poll() call. Therefore we avoid
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003926 * accidentally calling ->poll() when NAPI is not scheduled.
David S. Miller0a7606c2007-10-29 21:28:47 -07003927 */
3928 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003929 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003930 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003931 trace_napi_poll(n);
3932 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003933
3934 WARN_ON_ONCE(work > weight);
3935
3936 budget -= work;
3937
3938 local_irq_disable();
3939
3940 /* Drivers must not modify the NAPI state if they
3941 * consume the entire weight. In such cases this code
3942 * still "owns" the NAPI instance and therefore can
3943 * move the instance around on the list at-will.
3944 */
David S. Millerfed17f32008-01-07 21:00:40 -08003945 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003946 if (unlikely(napi_disable_pending(n))) {
3947 local_irq_enable();
3948 napi_complete(n);
3949 local_irq_disable();
3950 } else
Eric Dumazete326bed2010-04-22 00:22:45 -07003951 list_move_tail(&n->poll_list, &sd->poll_list);
David S. Millerfed17f32008-01-07 21:00:40 -08003952 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003953
3954 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003955 }
3956out:
Eric Dumazete326bed2010-04-22 00:22:45 -07003957 net_rps_action_and_irq_enable(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003958
Chris Leechdb217332006-06-17 21:24:58 -07003959#ifdef CONFIG_NET_DMA
3960 /*
3961 * There may not be any more sk_buffs coming right now, so push
3962 * any pending DMA copies to hardware
3963 */
Dan Williams2ba05622009-01-06 11:38:14 -07003964 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003965#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003966
Linus Torvalds1da177e2005-04-16 15:20:36 -07003967 return;
3968
3969softnet_break:
Changli Gaodee42872010-05-02 05:42:16 +00003970 sd->time_squeeze++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003971 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3972 goto out;
3973}
3974
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003975static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976
3977/**
3978 * register_gifconf - register a SIOCGIF handler
3979 * @family: Address family
3980 * @gifconf: Function handler
3981 *
3982 * Register protocol dependent address dumping routines. The handler
3983 * that is passed must not be freed or reused until it has been replaced
3984 * by another handler.
3985 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003986int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003987{
3988 if (family >= NPROTO)
3989 return -EINVAL;
3990 gifconf_list[family] = gifconf;
3991 return 0;
3992}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003993EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003994
3995
3996/*
3997 * Map an interface index to its name (SIOCGIFNAME)
3998 */
3999
4000/*
4001 * We need this ioctl for efficient implementation of the
4002 * if_indextoname() function required by the IPv6 API. Without
4003 * it, we would have to search all the interfaces to find a
4004 * match. --pb
4005 */
4006
Eric W. Biederman881d9662007-09-17 11:56:21 -07004007static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004008{
4009 struct net_device *dev;
4010 struct ifreq ifr;
4011
4012 /*
4013 * Fetch the caller's info block.
4014 */
4015
4016 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4017 return -EFAULT;
4018
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004019 rcu_read_lock();
4020 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004021 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004022 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004023 return -ENODEV;
4024 }
4025
4026 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00004027 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004028
4029 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
4030 return -EFAULT;
4031 return 0;
4032}
4033
4034/*
4035 * Perform a SIOCGIFCONF call. This structure will change
4036 * size eventually, and there is nothing I can do about it.
4037 * Thus we will need a 'compatibility mode'.
4038 */
4039
Eric W. Biederman881d9662007-09-17 11:56:21 -07004040static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004041{
4042 struct ifconf ifc;
4043 struct net_device *dev;
4044 char __user *pos;
4045 int len;
4046 int total;
4047 int i;
4048
4049 /*
4050 * Fetch the caller's info block.
4051 */
4052
4053 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
4054 return -EFAULT;
4055
4056 pos = ifc.ifc_buf;
4057 len = ifc.ifc_len;
4058
4059 /*
4060 * Loop over the interfaces, and write an info block for each.
4061 */
4062
4063 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004064 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004065 for (i = 0; i < NPROTO; i++) {
4066 if (gifconf_list[i]) {
4067 int done;
4068 if (!pos)
4069 done = gifconf_list[i](dev, NULL, 0);
4070 else
4071 done = gifconf_list[i](dev, pos + total,
4072 len - total);
4073 if (done < 0)
4074 return -EFAULT;
4075 total += done;
4076 }
4077 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004078 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004079
4080 /*
4081 * All done. Write the updated control block back to the caller.
4082 */
4083 ifc.ifc_len = total;
4084
4085 /*
4086 * Both BSD and Solaris return 0 here, so we do too.
4087 */
4088 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
4089}
4090
4091#ifdef CONFIG_PROC_FS
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004092
4093#define BUCKET_SPACE (32 - NETDEV_HASHBITS)
4094
4095struct dev_iter_state {
4096 struct seq_net_private p;
4097 unsigned int pos; /* bucket << BUCKET_SPACE + offset */
4098};
4099
4100#define get_bucket(x) ((x) >> BUCKET_SPACE)
4101#define get_offset(x) ((x) & ((1 << BUCKET_SPACE) - 1))
4102#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))
4103
4104static inline struct net_device *dev_from_same_bucket(struct seq_file *seq)
4105{
4106 struct dev_iter_state *state = seq->private;
4107 struct net *net = seq_file_net(seq);
4108 struct net_device *dev;
4109 struct hlist_node *p;
4110 struct hlist_head *h;
4111 unsigned int count, bucket, offset;
4112
4113 bucket = get_bucket(state->pos);
4114 offset = get_offset(state->pos);
4115 h = &net->dev_name_head[bucket];
4116 count = 0;
4117 hlist_for_each_entry_rcu(dev, p, h, name_hlist) {
4118 if (count++ == offset) {
4119 state->pos = set_bucket_offset(bucket, count);
4120 return dev;
4121 }
4122 }
4123
4124 return NULL;
4125}
4126
4127static inline struct net_device *dev_from_new_bucket(struct seq_file *seq)
4128{
4129 struct dev_iter_state *state = seq->private;
4130 struct net_device *dev;
4131 unsigned int bucket;
4132
4133 bucket = get_bucket(state->pos);
4134 do {
4135 dev = dev_from_same_bucket(seq);
4136 if (dev)
4137 return dev;
4138
4139 bucket++;
4140 state->pos = set_bucket_offset(bucket, 0);
4141 } while (bucket < NETDEV_HASHENTRIES);
4142
4143 return NULL;
4144}
4145
Linus Torvalds1da177e2005-04-16 15:20:36 -07004146/*
4147 * This is invoked by the /proc filesystem handler to display a device
4148 * in detail.
4149 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004150void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004151 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004152{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004153 struct dev_iter_state *state = seq->private;
Pavel Emelianov7562f872007-05-03 15:13:45 -07004154
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004155 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07004156 if (!*pos)
4157 return SEQ_START_TOKEN;
4158
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004159 /* check for end of the hash */
4160 if (state->pos == 0 && *pos > 1)
4161 return NULL;
Pavel Emelianov7562f872007-05-03 15:13:45 -07004162
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004163 return dev_from_new_bucket(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004164}
4165
4166void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4167{
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004168 struct net_device *dev;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004169
Linus Torvalds1da177e2005-04-16 15:20:36 -07004170 ++*pos;
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004171
4172 if (v == SEQ_START_TOKEN)
4173 return dev_from_new_bucket(seq);
4174
4175 dev = dev_from_same_bucket(seq);
4176 if (dev)
4177 return dev;
4178
4179 return dev_from_new_bucket(seq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004180}
4181
4182void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004183 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004184{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08004185 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004186}
4187
4188static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4189{
Eric Dumazet28172732010-07-07 14:58:56 -07004190 struct rtnl_link_stats64 temp;
4191 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004192
Ben Hutchingsbe1f3c22010-06-08 07:19:54 +00004193 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4194 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
Rusty Russell5a1b5892007-04-28 21:04:03 -07004195 dev->name, stats->rx_bytes, stats->rx_packets,
4196 stats->rx_errors,
4197 stats->rx_dropped + stats->rx_missed_errors,
4198 stats->rx_fifo_errors,
4199 stats->rx_length_errors + stats->rx_over_errors +
4200 stats->rx_crc_errors + stats->rx_frame_errors,
4201 stats->rx_compressed, stats->multicast,
4202 stats->tx_bytes, stats->tx_packets,
4203 stats->tx_errors, stats->tx_dropped,
4204 stats->tx_fifo_errors, stats->collisions,
4205 stats->tx_carrier_errors +
4206 stats->tx_aborted_errors +
4207 stats->tx_window_errors +
4208 stats->tx_heartbeat_errors,
4209 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004210}
4211
4212/*
4213 * Called from the PROCfs module. This now uses the new arbitrary sized
4214 * /proc/net interface to create /proc/net/dev
4215 */
4216static int dev_seq_show(struct seq_file *seq, void *v)
4217{
4218 if (v == SEQ_START_TOKEN)
4219 seq_puts(seq, "Inter-| Receive "
4220 " | Transmit\n"
4221 " face |bytes packets errs drop fifo frame "
4222 "compressed multicast|bytes packets errs "
4223 "drop fifo colls carrier compressed\n");
4224 else
4225 dev_seq_printf_stats(seq, v);
4226 return 0;
4227}
4228
Changli Gaodee42872010-05-02 05:42:16 +00004229static struct softnet_data *softnet_get_online(loff_t *pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004230{
Changli Gaodee42872010-05-02 05:42:16 +00004231 struct softnet_data *sd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004232
Mike Travis0c0b0ac2008-05-02 16:43:08 -07004233 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004234 if (cpu_online(*pos)) {
Changli Gaodee42872010-05-02 05:42:16 +00004235 sd = &per_cpu(softnet_data, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004236 break;
4237 } else
4238 ++*pos;
Changli Gaodee42872010-05-02 05:42:16 +00004239 return sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004240}
4241
4242static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4243{
4244 return softnet_get_online(pos);
4245}
4246
4247static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4248{
4249 ++*pos;
4250 return softnet_get_online(pos);
4251}
4252
4253static void softnet_seq_stop(struct seq_file *seq, void *v)
4254{
4255}
4256
4257static int softnet_seq_show(struct seq_file *seq, void *v)
4258{
Changli Gaodee42872010-05-02 05:42:16 +00004259 struct softnet_data *sd = v;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004260
Tom Herbert0a9627f2010-03-16 08:03:29 +00004261 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Changli Gaodee42872010-05-02 05:42:16 +00004262 sd->processed, sd->dropped, sd->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07004263 0, 0, 0, 0, /* was fastroute */
Changli Gaodee42872010-05-02 05:42:16 +00004264 sd->cpu_collision, sd->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004265 return 0;
4266}
4267
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004268static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004269 .start = dev_seq_start,
4270 .next = dev_seq_next,
4271 .stop = dev_seq_stop,
4272 .show = dev_seq_show,
4273};
4274
4275static int dev_seq_open(struct inode *inode, struct file *file)
4276{
Denis V. Luneve372c412007-11-19 22:31:54 -08004277 return seq_open_net(inode, file, &dev_seq_ops,
Mihai Maruseacf04565d2011-10-20 20:45:10 +00004278 sizeof(struct dev_iter_state));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004279}
4280
Arjan van de Ven9a321442007-02-12 00:55:35 -08004281static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004282 .owner = THIS_MODULE,
4283 .open = dev_seq_open,
4284 .read = seq_read,
4285 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004286 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004287};
4288
Stephen Hemmingerf6908082007-03-12 14:34:29 -07004289static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004290 .start = softnet_seq_start,
4291 .next = softnet_seq_next,
4292 .stop = softnet_seq_stop,
4293 .show = softnet_seq_show,
4294};
4295
4296static int softnet_seq_open(struct inode *inode, struct file *file)
4297{
4298 return seq_open(file, &softnet_seq_ops);
4299}
4300
Arjan van de Ven9a321442007-02-12 00:55:35 -08004301static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004302 .owner = THIS_MODULE,
4303 .open = softnet_seq_open,
4304 .read = seq_read,
4305 .llseek = seq_lseek,
4306 .release = seq_release,
4307};
4308
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004309static void *ptype_get_idx(loff_t pos)
4310{
4311 struct packet_type *pt = NULL;
4312 loff_t i = 0;
4313 int t;
4314
4315 list_for_each_entry_rcu(pt, &ptype_all, list) {
4316 if (i == pos)
4317 return pt;
4318 ++i;
4319 }
4320
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004321 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004322 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4323 if (i == pos)
4324 return pt;
4325 ++i;
4326 }
4327 }
4328 return NULL;
4329}
4330
4331static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004332 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004333{
4334 rcu_read_lock();
4335 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4336}
4337
4338static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4339{
4340 struct packet_type *pt;
4341 struct list_head *nxt;
4342 int hash;
4343
4344 ++*pos;
4345 if (v == SEQ_START_TOKEN)
4346 return ptype_get_idx(0);
4347
4348 pt = v;
4349 nxt = pt->list.next;
4350 if (pt->type == htons(ETH_P_ALL)) {
4351 if (nxt != &ptype_all)
4352 goto found;
4353 hash = 0;
4354 nxt = ptype_base[0].next;
4355 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004356 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004357
4358 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08004359 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004360 return NULL;
4361 nxt = ptype_base[hash].next;
4362 }
4363found:
4364 return list_entry(nxt, struct packet_type, list);
4365}
4366
4367static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08004368 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004369{
4370 rcu_read_unlock();
4371}
4372
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004373static int ptype_seq_show(struct seq_file *seq, void *v)
4374{
4375 struct packet_type *pt = v;
4376
4377 if (v == SEQ_START_TOKEN)
4378 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004379 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004380 if (pt->type == htons(ETH_P_ALL))
4381 seq_puts(seq, "ALL ");
4382 else
4383 seq_printf(seq, "%04x", ntohs(pt->type));
4384
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08004385 seq_printf(seq, " %-8s %pF\n",
4386 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004387 }
4388
4389 return 0;
4390}
4391
4392static const struct seq_operations ptype_seq_ops = {
4393 .start = ptype_seq_start,
4394 .next = ptype_seq_next,
4395 .stop = ptype_seq_stop,
4396 .show = ptype_seq_show,
4397};
4398
4399static int ptype_seq_open(struct inode *inode, struct file *file)
4400{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004401 return seq_open_net(inode, file, &ptype_seq_ops,
4402 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004403}
4404
4405static const struct file_operations ptype_seq_fops = {
4406 .owner = THIS_MODULE,
4407 .open = ptype_seq_open,
4408 .read = seq_read,
4409 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07004410 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004411};
4412
4413
Pavel Emelyanov46650792007-10-08 20:38:39 -07004414static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004415{
4416 int rc = -ENOMEM;
4417
Eric W. Biederman881d9662007-09-17 11:56:21 -07004418 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004419 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004420 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07004422 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004423 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07004424
Eric W. Biederman881d9662007-09-17 11:56:21 -07004425 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004426 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004427 rc = 0;
4428out:
4429 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02004430out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004431 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004432out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004433 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004434out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07004435 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004436 goto out;
4437}
Eric W. Biederman881d9662007-09-17 11:56:21 -07004438
Pavel Emelyanov46650792007-10-08 20:38:39 -07004439static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004440{
4441 wext_proc_exit(net);
4442
4443 proc_net_remove(net, "ptype");
4444 proc_net_remove(net, "softnet_stat");
4445 proc_net_remove(net, "dev");
4446}
4447
Denis V. Lunev022cbae2007-11-13 03:23:50 -08004448static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07004449 .init = dev_proc_net_init,
4450 .exit = dev_proc_net_exit,
4451};
4452
4453static int __init dev_proc_init(void)
4454{
4455 return register_pernet_subsys(&dev_proc_ops);
4456}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004457#else
4458#define dev_proc_init() 0
4459#endif /* CONFIG_PROC_FS */
4460
4461
4462/**
Jiri Pirko1765a572011-02-12 06:48:36 +00004463 * netdev_set_master - set up master pointer
Linus Torvalds1da177e2005-04-16 15:20:36 -07004464 * @slave: slave device
4465 * @master: new master device
4466 *
4467 * Changes the master device of the slave. Pass %NULL to break the
4468 * bonding. The caller must hold the RTNL semaphore. On a failure
4469 * a negative errno code is returned. On success the reference counts
Jiri Pirko1765a572011-02-12 06:48:36 +00004470 * are adjusted and the function returns zero.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004471 */
4472int netdev_set_master(struct net_device *slave, struct net_device *master)
4473{
4474 struct net_device *old = slave->master;
4475
4476 ASSERT_RTNL();
4477
4478 if (master) {
4479 if (old)
4480 return -EBUSY;
4481 dev_hold(master);
4482 }
4483
4484 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09004485
Eric Dumazet6df427f2011-05-19 19:37:40 +00004486 if (old)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004487 dev_put(old);
Jiri Pirko1765a572011-02-12 06:48:36 +00004488 return 0;
4489}
4490EXPORT_SYMBOL(netdev_set_master);
4491
4492/**
4493 * netdev_set_bond_master - set up bonding master/slave pair
4494 * @slave: slave device
4495 * @master: new master device
4496 *
4497 * Changes the master device of the slave. Pass %NULL to break the
4498 * bonding. The caller must hold the RTNL semaphore. On a failure
4499 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4500 * to the routing socket and the function returns zero.
4501 */
4502int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4503{
4504 int err;
4505
4506 ASSERT_RTNL();
4507
4508 err = netdev_set_master(slave, master);
4509 if (err)
4510 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004511 if (master)
4512 slave->flags |= IFF_SLAVE;
4513 else
4514 slave->flags &= ~IFF_SLAVE;
4515
4516 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4517 return 0;
4518}
Jiri Pirko1765a572011-02-12 06:48:36 +00004519EXPORT_SYMBOL(netdev_set_bond_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004520
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004521static void dev_change_rx_flags(struct net_device *dev, int flags)
4522{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004523 const struct net_device_ops *ops = dev->netdev_ops;
4524
4525 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4526 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004527}
4528
Wang Chendad9b332008-06-18 01:48:28 -07004529static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07004530{
4531 unsigned short old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11004532 uid_t uid;
4533 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07004534
Patrick McHardy24023452007-07-14 18:51:31 -07004535 ASSERT_RTNL();
4536
Wang Chendad9b332008-06-18 01:48:28 -07004537 dev->flags |= IFF_PROMISC;
4538 dev->promiscuity += inc;
4539 if (dev->promiscuity == 0) {
4540 /*
4541 * Avoid overflow.
4542 * If inc causes overflow, untouch promisc and return error.
4543 */
4544 if (inc < 0)
4545 dev->flags &= ~IFF_PROMISC;
4546 else {
4547 dev->promiscuity -= inc;
4548 printk(KERN_WARNING "%s: promiscuity touches roof, "
4549 "set promiscuity failed, promiscuity feature "
4550 "of device might be broken.\n", dev->name);
4551 return -EOVERFLOW;
4552 }
4553 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004554 if (dev->flags != old_flags) {
4555 printk(KERN_INFO "device %s %s promiscuous mode\n",
4556 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4557 "left");
David Howells8192b0c2008-11-14 10:39:10 +11004558 if (audit_enabled) {
4559 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004560 audit_log(current->audit_context, GFP_ATOMIC,
4561 AUDIT_ANOM_PROMISCUOUS,
4562 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4563 dev->name, (dev->flags & IFF_PROMISC),
4564 (old_flags & IFF_PROMISC),
4565 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11004566 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05004567 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11004568 }
Patrick McHardy24023452007-07-14 18:51:31 -07004569
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004570 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07004571 }
Wang Chendad9b332008-06-18 01:48:28 -07004572 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004573}
4574
Linus Torvalds1da177e2005-04-16 15:20:36 -07004575/**
4576 * dev_set_promiscuity - update promiscuity count on a device
4577 * @dev: device
4578 * @inc: modifier
4579 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07004580 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07004581 * remains above zero the interface remains promiscuous. Once it hits zero
4582 * the device reverts back to normal filtering operation. A negative inc
4583 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07004584 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004585 */
Wang Chendad9b332008-06-18 01:48:28 -07004586int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004587{
4588 unsigned short old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07004589 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004590
Wang Chendad9b332008-06-18 01:48:28 -07004591 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07004592 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07004593 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07004594 if (dev->flags != old_flags)
4595 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07004596 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004597}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004598EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004599
4600/**
4601 * dev_set_allmulti - update allmulti count on a device
4602 * @dev: device
4603 * @inc: modifier
4604 *
4605 * Add or remove reception of all multicast frames to a device. While the
4606 * count in the device remains above zero the interface remains listening
4607 * to all interfaces. Once it hits zero the device reverts back to normal
4608 * filtering operation. A negative @inc value is used to drop the counter
4609 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07004610 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004611 */
4612
Wang Chendad9b332008-06-18 01:48:28 -07004613int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004614{
4615 unsigned short old_flags = dev->flags;
4616
Patrick McHardy24023452007-07-14 18:51:31 -07004617 ASSERT_RTNL();
4618
Linus Torvalds1da177e2005-04-16 15:20:36 -07004619 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07004620 dev->allmulti += inc;
4621 if (dev->allmulti == 0) {
4622 /*
4623 * Avoid overflow.
4624 * If inc causes overflow, untouch allmulti and return error.
4625 */
4626 if (inc < 0)
4627 dev->flags &= ~IFF_ALLMULTI;
4628 else {
4629 dev->allmulti -= inc;
4630 printk(KERN_WARNING "%s: allmulti touches roof, "
4631 "set allmulti failed, allmulti feature of "
4632 "device might be broken.\n", dev->name);
4633 return -EOVERFLOW;
4634 }
4635 }
Patrick McHardy24023452007-07-14 18:51:31 -07004636 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004637 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07004638 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07004639 }
Wang Chendad9b332008-06-18 01:48:28 -07004640 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07004641}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004642EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07004643
4644/*
4645 * Upload unicast and multicast address lists to device and
4646 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08004647 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07004648 * are present.
4649 */
4650void __dev_set_rx_mode(struct net_device *dev)
4651{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004652 const struct net_device_ops *ops = dev->netdev_ops;
4653
Patrick McHardy4417da62007-06-27 01:28:10 -07004654 /* dev_open will call this function so the list will stay sane. */
4655 if (!(dev->flags&IFF_UP))
4656 return;
4657
4658 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09004659 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07004660
Jiri Pirko01789342011-08-16 06:29:00 +00004661 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004662 /* Unicast addresses changes may only happen under the rtnl,
4663 * therefore calling __dev_set_promiscuity here is safe.
4664 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004665 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004666 __dev_set_promiscuity(dev, 1);
Joe Perches2d348d12011-07-25 16:17:35 -07004667 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08004668 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07004669 __dev_set_promiscuity(dev, -1);
Joe Perches2d348d12011-07-25 16:17:35 -07004670 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07004671 }
Patrick McHardy4417da62007-06-27 01:28:10 -07004672 }
Jiri Pirko01789342011-08-16 06:29:00 +00004673
4674 if (ops->ndo_set_rx_mode)
4675 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004676}
4677
4678void dev_set_rx_mode(struct net_device *dev)
4679{
David S. Millerb9e40852008-07-15 00:15:08 -07004680 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004681 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07004682 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004683}
4684
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004685/**
4686 * dev_get_flags - get flags reported to userspace
4687 * @dev: device
4688 *
4689 * Get the combination of flag bits exported through APIs to userspace.
4690 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004691unsigned dev_get_flags(const struct net_device *dev)
4692{
4693 unsigned flags;
4694
4695 flags = (dev->flags & ~(IFF_PROMISC |
4696 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004697 IFF_RUNNING |
4698 IFF_LOWER_UP |
4699 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004700 (dev->gflags & (IFF_PROMISC |
4701 IFF_ALLMULTI));
4702
Stefan Rompfb00055a2006-03-20 17:09:11 -08004703 if (netif_running(dev)) {
4704 if (netif_oper_up(dev))
4705 flags |= IFF_RUNNING;
4706 if (netif_carrier_ok(dev))
4707 flags |= IFF_LOWER_UP;
4708 if (netif_dormant(dev))
4709 flags |= IFF_DORMANT;
4710 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004711
4712 return flags;
4713}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004714EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004715
Patrick McHardybd380812010-02-26 06:34:53 +00004716int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004717{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004718 int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004719 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004720
Patrick McHardy24023452007-07-14 18:51:31 -07004721 ASSERT_RTNL();
4722
Linus Torvalds1da177e2005-04-16 15:20:36 -07004723 /*
4724 * Set the flags on our device.
4725 */
4726
4727 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4728 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4729 IFF_AUTOMEDIA)) |
4730 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4731 IFF_ALLMULTI));
4732
4733 /*
4734 * Load in the correct multicast list now the flags have changed.
4735 */
4736
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004737 if ((old_flags ^ flags) & IFF_MULTICAST)
4738 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004739
Patrick McHardy4417da62007-06-27 01:28:10 -07004740 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004741
4742 /*
4743 * Have we downed the interface. We handle IFF_UP ourselves
4744 * according to user attempts to set it, rather than blindly
4745 * setting it.
4746 */
4747
4748 ret = 0;
4749 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004750 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004751
4752 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004753 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004754 }
4755
Linus Torvalds1da177e2005-04-16 15:20:36 -07004756 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004757 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4758
Linus Torvalds1da177e2005-04-16 15:20:36 -07004759 dev->gflags ^= IFF_PROMISC;
4760 dev_set_promiscuity(dev, inc);
4761 }
4762
4763 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4764 is important. Some (broken) drivers set IFF_PROMISC, when
4765 IFF_ALLMULTI is requested not asking us and not reporting.
4766 */
4767 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004768 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4769
Linus Torvalds1da177e2005-04-16 15:20:36 -07004770 dev->gflags ^= IFF_ALLMULTI;
4771 dev_set_allmulti(dev, inc);
4772 }
4773
Patrick McHardybd380812010-02-26 06:34:53 +00004774 return ret;
4775}
4776
4777void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4778{
4779 unsigned int changes = dev->flags ^ old_flags;
4780
4781 if (changes & IFF_UP) {
4782 if (dev->flags & IFF_UP)
4783 call_netdevice_notifiers(NETDEV_UP, dev);
4784 else
4785 call_netdevice_notifiers(NETDEV_DOWN, dev);
4786 }
4787
4788 if (dev->flags & IFF_UP &&
4789 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4790 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4791}
4792
4793/**
4794 * dev_change_flags - change device settings
4795 * @dev: device
4796 * @flags: device state flags
4797 *
4798 * Change settings on device based state flags. The flags are
4799 * in the userspace exported format.
4800 */
4801int dev_change_flags(struct net_device *dev, unsigned flags)
4802{
4803 int ret, changes;
4804 int old_flags = dev->flags;
4805
4806 ret = __dev_change_flags(dev, flags);
4807 if (ret < 0)
4808 return ret;
4809
4810 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004811 if (changes)
4812 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004813
Patrick McHardybd380812010-02-26 06:34:53 +00004814 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004815 return ret;
4816}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004817EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004818
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004819/**
4820 * dev_set_mtu - Change maximum transfer unit
4821 * @dev: device
4822 * @new_mtu: new transfer unit
4823 *
4824 * Change the maximum transfer size of the network device.
4825 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004826int dev_set_mtu(struct net_device *dev, int new_mtu)
4827{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004828 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004829 int err;
4830
4831 if (new_mtu == dev->mtu)
4832 return 0;
4833
4834 /* MTU must be positive. */
4835 if (new_mtu < 0)
4836 return -EINVAL;
4837
4838 if (!netif_device_present(dev))
4839 return -ENODEV;
4840
4841 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004842 if (ops->ndo_change_mtu)
4843 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004844 else
4845 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004846
Linus Torvalds1da177e2005-04-16 15:20:36 -07004847 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004848 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004849 return err;
4850}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004851EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004852
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004853/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00004854 * dev_set_group - Change group this device belongs to
4855 * @dev: device
4856 * @new_group: group this device should belong to
4857 */
4858void dev_set_group(struct net_device *dev, int new_group)
4859{
4860 dev->group = new_group;
4861}
4862EXPORT_SYMBOL(dev_set_group);
4863
4864/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004865 * dev_set_mac_address - Change Media Access Control Address
4866 * @dev: device
4867 * @sa: new address
4868 *
4869 * Change the hardware (MAC) address of the device
4870 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004871int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4872{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004873 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004874 int err;
4875
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004876 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004877 return -EOPNOTSUPP;
4878 if (sa->sa_family != dev->type)
4879 return -EINVAL;
4880 if (!netif_device_present(dev))
4881 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004882 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004883 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004884 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004885 return err;
4886}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004887EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004888
4889/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004890 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004891 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004892static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004893{
4894 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004895 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004896
4897 if (!dev)
4898 return -ENODEV;
4899
4900 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004901 case SIOCGIFFLAGS: /* Get interface flags */
4902 ifr->ifr_flags = (short) dev_get_flags(dev);
4903 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004904
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004905 case SIOCGIFMETRIC: /* Get the metric on the interface
4906 (currently unused) */
4907 ifr->ifr_metric = 0;
4908 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004909
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004910 case SIOCGIFMTU: /* Get the MTU of a device */
4911 ifr->ifr_mtu = dev->mtu;
4912 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004913
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004914 case SIOCGIFHWADDR:
4915 if (!dev->addr_len)
4916 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4917 else
4918 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4919 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4920 ifr->ifr_hwaddr.sa_family = dev->type;
4921 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004922
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004923 case SIOCGIFSLAVE:
4924 err = -EINVAL;
4925 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004926
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004927 case SIOCGIFMAP:
4928 ifr->ifr_map.mem_start = dev->mem_start;
4929 ifr->ifr_map.mem_end = dev->mem_end;
4930 ifr->ifr_map.base_addr = dev->base_addr;
4931 ifr->ifr_map.irq = dev->irq;
4932 ifr->ifr_map.dma = dev->dma;
4933 ifr->ifr_map.port = dev->if_port;
4934 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004935
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004936 case SIOCGIFINDEX:
4937 ifr->ifr_ifindex = dev->ifindex;
4938 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004939
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004940 case SIOCGIFTXQLEN:
4941 ifr->ifr_qlen = dev->tx_queue_len;
4942 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004943
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004944 default:
4945 /* dev_ioctl() should ensure this case
4946 * is never reached
4947 */
4948 WARN_ON(1);
Lifeng Sun41c31f32011-04-27 22:04:51 +00004949 err = -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004950 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004951
4952 }
4953 return err;
4954}
4955
4956/*
4957 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4958 */
4959static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4960{
4961 int err;
4962 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004963 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004964
4965 if (!dev)
4966 return -ENODEV;
4967
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004968 ops = dev->netdev_ops;
4969
Jeff Garzik14e3e072007-10-08 00:06:32 -07004970 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004971 case SIOCSIFFLAGS: /* Set interface flags */
4972 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004973
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004974 case SIOCSIFMETRIC: /* Set the metric on the interface
4975 (currently unused) */
4976 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004977
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004978 case SIOCSIFMTU: /* Set the MTU of a device */
4979 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004980
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004981 case SIOCSIFHWADDR:
4982 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004983
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004984 case SIOCSIFHWBROADCAST:
4985 if (ifr->ifr_hwaddr.sa_family != dev->type)
4986 return -EINVAL;
4987 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4988 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4989 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4990 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004991
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004992 case SIOCSIFMAP:
4993 if (ops->ndo_set_config) {
4994 if (!netif_device_present(dev))
4995 return -ENODEV;
4996 return ops->ndo_set_config(dev, &ifr->ifr_map);
4997 }
4998 return -EOPNOTSUPP;
4999
5000 case SIOCADDMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00005001 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005002 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
5003 return -EINVAL;
5004 if (!netif_device_present(dev))
5005 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00005006 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005007
5008 case SIOCDELMULTI:
Jiri Pirkob81693d2011-08-16 06:29:02 +00005009 if (!ops->ndo_set_rx_mode ||
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005010 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
5011 return -EINVAL;
5012 if (!netif_device_present(dev))
5013 return -ENODEV;
Jiri Pirko22bedad2010-04-01 21:22:57 +00005014 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005015
5016 case SIOCSIFTXQLEN:
5017 if (ifr->ifr_qlen < 0)
5018 return -EINVAL;
5019 dev->tx_queue_len = ifr->ifr_qlen;
5020 return 0;
5021
5022 case SIOCSIFNAME:
5023 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
5024 return dev_change_name(dev, ifr->ifr_newname);
5025
Richard Cochran4dc360c2011-10-19 17:00:35 -04005026 case SIOCSHWTSTAMP:
5027 err = net_hwtstamp_validate(ifr);
5028 if (err)
5029 return err;
5030 /* fall through */
5031
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005032 /*
5033 * Unknown or private ioctl
5034 */
5035 default:
5036 if ((cmd >= SIOCDEVPRIVATE &&
5037 cmd <= SIOCDEVPRIVATE + 15) ||
5038 cmd == SIOCBONDENSLAVE ||
5039 cmd == SIOCBONDRELEASE ||
5040 cmd == SIOCBONDSETHWADDR ||
5041 cmd == SIOCBONDSLAVEINFOQUERY ||
5042 cmd == SIOCBONDINFOQUERY ||
5043 cmd == SIOCBONDCHANGEACTIVE ||
5044 cmd == SIOCGMIIPHY ||
5045 cmd == SIOCGMIIREG ||
5046 cmd == SIOCSMIIREG ||
5047 cmd == SIOCBRADDIF ||
5048 cmd == SIOCBRDELIF ||
5049 cmd == SIOCSHWTSTAMP ||
5050 cmd == SIOCWANDEV) {
5051 err = -EOPNOTSUPP;
5052 if (ops->ndo_do_ioctl) {
5053 if (netif_device_present(dev))
5054 err = ops->ndo_do_ioctl(dev, ifr, cmd);
5055 else
5056 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005057 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005058 } else
5059 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005060
5061 }
5062 return err;
5063}
5064
5065/*
5066 * This function handles all "interface"-type I/O control requests. The actual
5067 * 'doing' part of this is dev_ifsioc above.
5068 */
5069
5070/**
5071 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005072 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005073 * @cmd: command to issue
5074 * @arg: pointer to a struct ifreq in user space
5075 *
5076 * Issue ioctl functions to devices. This is normally called by the
5077 * user space syscall interfaces but can sometimes be useful for
5078 * other purposes. The return value is the return from the syscall if
5079 * positive or a negative errno code on error.
5080 */
5081
Eric W. Biederman881d9662007-09-17 11:56:21 -07005082int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005083{
5084 struct ifreq ifr;
5085 int ret;
5086 char *colon;
5087
5088 /* One special case: SIOCGIFCONF takes ifconf argument
5089 and requires shared lock, because it sleeps writing
5090 to user space.
5091 */
5092
5093 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005094 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005095 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005096 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005097 return ret;
5098 }
5099 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07005100 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005101
5102 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
5103 return -EFAULT;
5104
5105 ifr.ifr_name[IFNAMSIZ-1] = 0;
5106
5107 colon = strchr(ifr.ifr_name, ':');
5108 if (colon)
5109 *colon = 0;
5110
5111 /*
5112 * See which interface the caller is talking about.
5113 */
5114
5115 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005116 /*
5117 * These ioctl calls:
5118 * - can be done by all.
5119 * - atomic and do not require locking.
5120 * - return a value
5121 */
5122 case SIOCGIFFLAGS:
5123 case SIOCGIFMETRIC:
5124 case SIOCGIFMTU:
5125 case SIOCGIFHWADDR:
5126 case SIOCGIFSLAVE:
5127 case SIOCGIFMAP:
5128 case SIOCGIFINDEX:
5129 case SIOCGIFTXQLEN:
5130 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005131 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005132 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00005133 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005134 if (!ret) {
5135 if (colon)
5136 *colon = ':';
5137 if (copy_to_user(arg, &ifr,
5138 sizeof(struct ifreq)))
5139 ret = -EFAULT;
5140 }
5141 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005142
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005143 case SIOCETHTOOL:
5144 dev_load(net, ifr.ifr_name);
5145 rtnl_lock();
5146 ret = dev_ethtool(net, &ifr);
5147 rtnl_unlock();
5148 if (!ret) {
5149 if (colon)
5150 *colon = ':';
5151 if (copy_to_user(arg, &ifr,
5152 sizeof(struct ifreq)))
5153 ret = -EFAULT;
5154 }
5155 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005156
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005157 /*
5158 * These ioctl calls:
5159 * - require superuser power.
5160 * - require strict serialization.
5161 * - return a value
5162 */
5163 case SIOCGMIIPHY:
5164 case SIOCGMIIREG:
5165 case SIOCSIFNAME:
5166 if (!capable(CAP_NET_ADMIN))
5167 return -EPERM;
5168 dev_load(net, ifr.ifr_name);
5169 rtnl_lock();
5170 ret = dev_ifsioc(net, &ifr, cmd);
5171 rtnl_unlock();
5172 if (!ret) {
5173 if (colon)
5174 *colon = ':';
5175 if (copy_to_user(arg, &ifr,
5176 sizeof(struct ifreq)))
5177 ret = -EFAULT;
5178 }
5179 return ret;
5180
5181 /*
5182 * These ioctl calls:
5183 * - require superuser power.
5184 * - require strict serialization.
5185 * - do not return a value
5186 */
5187 case SIOCSIFFLAGS:
5188 case SIOCSIFMETRIC:
5189 case SIOCSIFMTU:
5190 case SIOCSIFMAP:
5191 case SIOCSIFHWADDR:
5192 case SIOCSIFSLAVE:
5193 case SIOCADDMULTI:
5194 case SIOCDELMULTI:
5195 case SIOCSIFHWBROADCAST:
5196 case SIOCSIFTXQLEN:
5197 case SIOCSMIIREG:
5198 case SIOCBONDENSLAVE:
5199 case SIOCBONDRELEASE:
5200 case SIOCBONDSETHWADDR:
5201 case SIOCBONDCHANGEACTIVE:
5202 case SIOCBRADDIF:
5203 case SIOCBRDELIF:
5204 case SIOCSHWTSTAMP:
5205 if (!capable(CAP_NET_ADMIN))
5206 return -EPERM;
5207 /* fall through */
5208 case SIOCBONDSLAVEINFOQUERY:
5209 case SIOCBONDINFOQUERY:
5210 dev_load(net, ifr.ifr_name);
5211 rtnl_lock();
5212 ret = dev_ifsioc(net, &ifr, cmd);
5213 rtnl_unlock();
5214 return ret;
5215
5216 case SIOCGIFMEM:
5217 /* Get the per device memory space. We can add this but
5218 * currently do not support it */
5219 case SIOCSIFMEM:
5220 /* Set the per device memory buffer space.
5221 * Not applicable in our case */
5222 case SIOCSIFLINK:
Lifeng Sun41c31f32011-04-27 22:04:51 +00005223 return -ENOTTY;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005224
5225 /*
5226 * Unknown or private ioctl.
5227 */
5228 default:
5229 if (cmd == SIOCWANDEV ||
5230 (cmd >= SIOCDEVPRIVATE &&
5231 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005232 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005233 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005234 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005235 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005236 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005237 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005238 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005239 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005240 }
5241 /* Take care of Wireless Extensions */
5242 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5243 return wext_handle_ioctl(net, &ifr, cmd, arg);
Lifeng Sun41c31f32011-04-27 22:04:51 +00005244 return -ENOTTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005245 }
5246}
5247
5248
5249/**
5250 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005251 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005252 *
5253 * Returns a suitable unique value for a new device interface
5254 * number. The caller must hold the rtnl semaphore or the
5255 * dev_base_lock to be sure it remains unique.
5256 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005257static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005258{
5259 static int ifindex;
5260 for (;;) {
5261 if (++ifindex <= 0)
5262 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005263 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005264 return ifindex;
5265 }
5266}
5267
Linus Torvalds1da177e2005-04-16 15:20:36 -07005268/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005269static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005270
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005271static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005272{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005273 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005274}
5275
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005276static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005277{
Krishna Kumare93737b2009-12-08 22:26:02 +00005278 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005279
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005280 BUG_ON(dev_boot_phase);
5281 ASSERT_RTNL();
5282
Krishna Kumare93737b2009-12-08 22:26:02 +00005283 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005284 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005285 * for initialization unwind. Remove those
5286 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005287 */
5288 if (dev->reg_state == NETREG_UNINITIALIZED) {
5289 pr_debug("unregister_netdevice: device %s/%p never "
5290 "was registered\n", dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005291
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005292 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005293 list_del(&dev->unreg_list);
5294 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005295 }
Eric Dumazet449f4542011-05-19 12:24:16 +00005296 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005297 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00005298 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005299
Octavian Purdila44345722010-12-13 12:44:07 +00005300 /* If device is running, close it first. */
5301 dev_close_many(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005302
Octavian Purdila44345722010-12-13 12:44:07 +00005303 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005304 /* And unlink it from device chain. */
5305 unlist_netdevice(dev);
5306
5307 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005308 }
5309
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005310 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005311
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005312 list_for_each_entry(dev, head, unreg_list) {
5313 /* Shutdown queueing discipline. */
5314 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005315
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005316
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005317 /* Notify protocols, that we are about to destroy
5318 this device. They should clean all the things.
5319 */
5320 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5321
Patrick McHardya2835762010-02-26 06:34:51 +00005322 if (!dev->rtnl_link_ops ||
5323 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5324 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5325
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005326 /*
5327 * Flush the unicast and multicast chains
5328 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00005329 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00005330 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005331
5332 if (dev->netdev_ops->ndo_uninit)
5333 dev->netdev_ops->ndo_uninit(dev);
5334
5335 /* Notifier chain MUST detach us from master device. */
5336 WARN_ON(dev->master);
5337
5338 /* Remove entries from kobject tree */
5339 netdev_unregister_kobject(dev);
5340 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005341
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005342 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00005343 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005344 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5345
Eric W. Biederman850a5452011-10-13 22:25:23 +00005346 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005347
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005348 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005349 dev_put(dev);
5350}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005351
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005352static void rollback_registered(struct net_device *dev)
5353{
5354 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005355
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005356 list_add(&dev->unreg_list, &single);
5357 rollback_registered_many(&single);
Eric Dumazetceaaec92011-02-17 22:59:19 +00005358 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005359}
5360
Michał Mirosławfec30c32011-07-13 14:10:30 +00005361static u32 netdev_fix_features(struct net_device *dev, u32 features)
Herbert Xub63365a2008-10-23 01:11:29 -07005362{
Michał Mirosław57422dc2011-01-22 12:14:12 +00005363 /* Fix illegal checksum combinations */
5364 if ((features & NETIF_F_HW_CSUM) &&
5365 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005366 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005367 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5368 }
5369
5370 if ((features & NETIF_F_NO_CSUM) &&
5371 (features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005372 netdev_warn(dev, "mixed no checksumming and other settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00005373 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5374 }
5375
Herbert Xub63365a2008-10-23 01:11:29 -07005376 /* Fix illegal SG+CSUM combinations. */
5377 if ((features & NETIF_F_SG) &&
5378 !(features & NETIF_F_ALL_CSUM)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005379 netdev_dbg(dev,
5380 "Dropping NETIF_F_SG since no checksum feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005381 features &= ~NETIF_F_SG;
5382 }
5383
5384 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005385 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005386 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00005387 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07005388 }
5389
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00005390 /* TSO ECN requires that TSO is present as well. */
5391 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5392 features &= ~NETIF_F_TSO_ECN;
5393
Michał Mirosław212b5732011-02-15 16:59:16 +00005394 /* Software GSO depends on SG. */
5395 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005396 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00005397 features &= ~NETIF_F_GSO;
5398 }
5399
Michał Mirosławacd11302011-01-24 15:45:15 -08005400 /* UFO needs SG and checksumming */
Herbert Xub63365a2008-10-23 01:11:29 -07005401 if (features & NETIF_F_UFO) {
Michał Mirosław79032642010-11-30 06:38:00 +00005402 /* maybe split UFO into V4 and V6? */
5403 if (!((features & NETIF_F_GEN_CSUM) ||
5404 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5405 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005406 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005407 "Dropping NETIF_F_UFO since no checksum offload features.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005408 features &= ~NETIF_F_UFO;
5409 }
5410
5411 if (!(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04005412 netdev_dbg(dev,
Michał Mirosławacd11302011-01-24 15:45:15 -08005413 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
Herbert Xub63365a2008-10-23 01:11:29 -07005414 features &= ~NETIF_F_UFO;
5415 }
5416 }
5417
5418 return features;
5419}
Herbert Xub63365a2008-10-23 01:11:29 -07005420
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005421int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00005422{
5423 u32 features;
5424 int err = 0;
5425
Michał Mirosław87267482011-04-12 09:56:38 +00005426 ASSERT_RTNL();
5427
Michał Mirosław5455c692011-02-15 16:59:17 +00005428 features = netdev_get_wanted_features(dev);
5429
5430 if (dev->netdev_ops->ndo_fix_features)
5431 features = dev->netdev_ops->ndo_fix_features(dev, features);
5432
5433 /* driver might be less strict about feature dependencies */
5434 features = netdev_fix_features(dev, features);
5435
5436 if (dev->features == features)
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005437 return 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00005438
Michael S. Tsirkin604ae142011-05-16 10:37:39 +00005439 netdev_dbg(dev, "Features changed: 0x%08x -> 0x%08x\n",
Michał Mirosław5455c692011-02-15 16:59:17 +00005440 dev->features, features);
5441
5442 if (dev->netdev_ops->ndo_set_features)
5443 err = dev->netdev_ops->ndo_set_features(dev, features);
5444
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005445 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00005446 netdev_err(dev,
5447 "set_features() failed (%d); wanted 0x%08x, left 0x%08x\n",
5448 err, features, dev->features);
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005449 return -1;
5450 }
5451
5452 if (!err)
5453 dev->features = features;
5454
5455 return 1;
5456}
5457
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005458/**
5459 * netdev_update_features - recalculate device features
5460 * @dev: the device to check
5461 *
5462 * Recalculate dev->features set and send notifications if it
5463 * has changed. Should be called after driver or hardware dependent
5464 * conditions might have changed that influence the features.
5465 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005466void netdev_update_features(struct net_device *dev)
5467{
5468 if (__netdev_update_features(dev))
5469 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00005470}
5471EXPORT_SYMBOL(netdev_update_features);
5472
Linus Torvalds1da177e2005-04-16 15:20:36 -07005473/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00005474 * netdev_change_features - recalculate device features
5475 * @dev: the device to check
5476 *
5477 * Recalculate dev->features set and send notifications even
5478 * if they have not changed. Should be called instead of
5479 * netdev_update_features() if also dev->vlan_features might
5480 * have changed to allow the changes to be propagated to stacked
5481 * VLAN devices.
5482 */
5483void netdev_change_features(struct net_device *dev)
5484{
5485 __netdev_update_features(dev);
5486 netdev_features_change(dev);
5487}
5488EXPORT_SYMBOL(netdev_change_features);
5489
5490/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005491 * netif_stacked_transfer_operstate - transfer operstate
5492 * @rootdev: the root or lower level device to transfer state from
5493 * @dev: the device to transfer operstate to
5494 *
5495 * Transfer operational state from root to device. This is normally
5496 * called when a stacking relationship exists between the root
5497 * device and the device(a leaf device).
5498 */
5499void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5500 struct net_device *dev)
5501{
5502 if (rootdev->operstate == IF_OPER_DORMANT)
5503 netif_dormant_on(dev);
5504 else
5505 netif_dormant_off(dev);
5506
5507 if (netif_carrier_ok(rootdev)) {
5508 if (!netif_carrier_ok(dev))
5509 netif_carrier_on(dev);
5510 } else {
5511 if (netif_carrier_ok(dev))
5512 netif_carrier_off(dev);
5513 }
5514}
5515EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5516
Tom Herbertbf264142010-11-26 08:36:09 +00005517#ifdef CONFIG_RPS
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005518static int netif_alloc_rx_queues(struct net_device *dev)
5519{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005520 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00005521 struct netdev_rx_queue *rx;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005522
Tom Herbertbd25fa72010-10-18 18:00:16 +00005523 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005524
Tom Herbertbd25fa72010-10-18 18:00:16 +00005525 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5526 if (!rx) {
5527 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5528 return -ENOMEM;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005529 }
Tom Herbertbd25fa72010-10-18 18:00:16 +00005530 dev->_rx = rx;
5531
Tom Herbertbd25fa72010-10-18 18:00:16 +00005532 for (i = 0; i < count; i++)
Tom Herbertfe822242010-11-09 10:47:38 +00005533 rx[i].dev = dev;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005534 return 0;
5535}
Tom Herbertbf264142010-11-26 08:36:09 +00005536#endif
Eric Dumazet1b4bf462010-09-23 17:26:35 +00005537
Changli Gaoaa942102010-12-04 02:31:41 +00005538static void netdev_init_one_queue(struct net_device *dev,
5539 struct netdev_queue *queue, void *_unused)
5540{
5541 /* Initialize queue lock */
5542 spin_lock_init(&queue->_xmit_lock);
5543 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5544 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00005545 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00005546 queue->dev = dev;
5547}
5548
Tom Herberte6484932010-10-18 18:04:39 +00005549static int netif_alloc_netdev_queues(struct net_device *dev)
5550{
5551 unsigned int count = dev->num_tx_queues;
5552 struct netdev_queue *tx;
5553
5554 BUG_ON(count < 1);
5555
5556 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5557 if (!tx) {
5558 pr_err("netdev: Unable to allocate %u tx queues.\n",
5559 count);
5560 return -ENOMEM;
5561 }
5562 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00005563
Tom Herberte6484932010-10-18 18:04:39 +00005564 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5565 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00005566
5567 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00005568}
5569
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005570/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005571 * register_netdevice - register a network device
5572 * @dev: device to register
5573 *
5574 * Take a completed network device structure and add it to the kernel
5575 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5576 * chain. 0 is returned on success. A negative errno code is returned
5577 * on a failure to set up the device, or if the name is a duplicate.
5578 *
5579 * Callers must hold the rtnl semaphore. You may want
5580 * register_netdev() instead of this.
5581 *
5582 * BUGS:
5583 * The locking appears insufficient to guarantee two parallel registers
5584 * will not get the same name.
5585 */
5586
5587int register_netdevice(struct net_device *dev)
5588{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005589 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005590 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005591
5592 BUG_ON(dev_boot_phase);
5593 ASSERT_RTNL();
5594
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005595 might_sleep();
5596
Linus Torvalds1da177e2005-04-16 15:20:36 -07005597 /* When net_device's are persistent, this will be fatal. */
5598 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005599 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005600
David S. Millerf1f28aa2008-07-15 00:08:33 -07005601 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005602 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005603
Linus Torvalds1da177e2005-04-16 15:20:36 -07005604 dev->iflink = -1;
5605
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00005606 ret = dev_get_valid_name(dev, dev->name);
5607 if (ret < 0)
5608 goto out;
5609
Linus Torvalds1da177e2005-04-16 15:20:36 -07005610 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005611 if (dev->netdev_ops->ndo_init) {
5612 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005613 if (ret) {
5614 if (ret > 0)
5615 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005616 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005617 }
5618 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005619
Eric W. Biederman881d9662007-09-17 11:56:21 -07005620 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005621 if (dev->iflink == -1)
5622 dev->iflink = dev->ifindex;
5623
Michał Mirosław5455c692011-02-15 16:59:17 +00005624 /* Transfer changeable features to wanted_features and enable
5625 * software offloads (GSO and GRO).
5626 */
5627 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00005628 dev->features |= NETIF_F_SOFT_FEATURES;
5629 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005630
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07005631 /* Turn on no cache copy if HW is doing checksum */
5632 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5633 if ((dev->features & NETIF_F_ALL_CSUM) &&
5634 !(dev->features & NETIF_F_NO_CSUM)) {
5635 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5636 dev->features |= NETIF_F_NOCACHE_COPY;
5637 }
5638
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005639 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00005640 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07005641 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00005642
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005643 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5644 ret = notifier_to_errno(ret);
5645 if (ret)
5646 goto err_uninit;
5647
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005648 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005649 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005650 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005651 dev->reg_state = NETREG_REGISTERED;
5652
Michał Mirosław6cb6a272011-04-02 22:48:47 -07005653 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00005654
Linus Torvalds1da177e2005-04-16 15:20:36 -07005655 /*
5656 * Default initial state at registry is that the
5657 * device is present.
5658 */
5659
5660 set_bit(__LINK_STATE_PRESENT, &dev->state);
5661
Linus Torvalds1da177e2005-04-16 15:20:36 -07005662 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005663 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005664 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005665
5666 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005667 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005668 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005669 if (ret) {
5670 rollback_registered(dev);
5671 dev->reg_state = NETREG_UNREGISTERED;
5672 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005673 /*
5674 * Prevent userspace races by waiting until the network
5675 * device is fully setup before sending notifications.
5676 */
Patrick McHardya2835762010-02-26 06:34:51 +00005677 if (!dev->rtnl_link_ops ||
5678 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5679 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005680
5681out:
5682 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005683
5684err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005685 if (dev->netdev_ops->ndo_uninit)
5686 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005687 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005688}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005689EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005690
5691/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005692 * init_dummy_netdev - init a dummy network device for NAPI
5693 * @dev: device to init
5694 *
5695 * This takes a network device structure and initialize the minimum
5696 * amount of fields so it can be used to schedule NAPI polls without
5697 * registering a full blown interface. This is to be used by drivers
5698 * that need to tie several hardware interfaces to a single NAPI
5699 * poll scheduler due to HW limitations.
5700 */
5701int init_dummy_netdev(struct net_device *dev)
5702{
5703 /* Clear everything. Note we don't initialize spinlocks
5704 * are they aren't supposed to be taken by any of the
5705 * NAPI code and this dummy netdev is supposed to be
5706 * only ever used for NAPI polls
5707 */
5708 memset(dev, 0, sizeof(struct net_device));
5709
5710 /* make sure we BUG if trying to hit standard
5711 * register/unregister code path
5712 */
5713 dev->reg_state = NETREG_DUMMY;
5714
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005715 /* NAPI wants this */
5716 INIT_LIST_HEAD(&dev->napi_list);
5717
5718 /* a dummy interface is started by default */
5719 set_bit(__LINK_STATE_PRESENT, &dev->state);
5720 set_bit(__LINK_STATE_START, &dev->state);
5721
Eric Dumazet29b44332010-10-11 10:22:12 +00005722 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5723 * because users of this 'device' dont need to change
5724 * its refcount.
5725 */
5726
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005727 return 0;
5728}
5729EXPORT_SYMBOL_GPL(init_dummy_netdev);
5730
5731
5732/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005733 * register_netdev - register a network device
5734 * @dev: device to register
5735 *
5736 * Take a completed network device structure and add it to the kernel
5737 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5738 * chain. 0 is returned on success. A negative errno code is returned
5739 * on a failure to set up the device, or if the name is a duplicate.
5740 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005741 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005742 * and expands the device name if you passed a format string to
5743 * alloc_netdev.
5744 */
5745int register_netdev(struct net_device *dev)
5746{
5747 int err;
5748
5749 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005750 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005751 rtnl_unlock();
5752 return err;
5753}
5754EXPORT_SYMBOL(register_netdev);
5755
Eric Dumazet29b44332010-10-11 10:22:12 +00005756int netdev_refcnt_read(const struct net_device *dev)
5757{
5758 int i, refcnt = 0;
5759
5760 for_each_possible_cpu(i)
5761 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5762 return refcnt;
5763}
5764EXPORT_SYMBOL(netdev_refcnt_read);
5765
Linus Torvalds1da177e2005-04-16 15:20:36 -07005766/*
5767 * netdev_wait_allrefs - wait until all references are gone.
5768 *
5769 * This is called when unregistering network devices.
5770 *
5771 * Any protocol or device that holds a reference should register
5772 * for netdevice notification, and cleanup and put back the
5773 * reference if they receive an UNREGISTER event.
5774 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005775 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005776 */
5777static void netdev_wait_allrefs(struct net_device *dev)
5778{
5779 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00005780 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005781
Eric Dumazete014deb2009-11-17 05:59:21 +00005782 linkwatch_forget_dev(dev);
5783
Linus Torvalds1da177e2005-04-16 15:20:36 -07005784 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00005785 refcnt = netdev_refcnt_read(dev);
5786
5787 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005788 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005789 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005790
5791 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005792 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005793 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005794 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005795
5796 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5797 &dev->state)) {
5798 /* We must not have linkwatch events
5799 * pending on unregister. If this
5800 * happens, we simply run the queue
5801 * unscheduled, resulting in a noop
5802 * for this device.
5803 */
5804 linkwatch_run_queue();
5805 }
5806
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005807 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005808
5809 rebroadcast_time = jiffies;
5810 }
5811
5812 msleep(250);
5813
Eric Dumazet29b44332010-10-11 10:22:12 +00005814 refcnt = netdev_refcnt_read(dev);
5815
Linus Torvalds1da177e2005-04-16 15:20:36 -07005816 if (time_after(jiffies, warning_time + 10 * HZ)) {
5817 printk(KERN_EMERG "unregister_netdevice: "
5818 "waiting for %s to become free. Usage "
5819 "count = %d\n",
Eric Dumazet29b44332010-10-11 10:22:12 +00005820 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005821 warning_time = jiffies;
5822 }
5823 }
5824}
5825
5826/* The sequence is:
5827 *
5828 * rtnl_lock();
5829 * ...
5830 * register_netdevice(x1);
5831 * register_netdevice(x2);
5832 * ...
5833 * unregister_netdevice(y1);
5834 * unregister_netdevice(y2);
5835 * ...
5836 * rtnl_unlock();
5837 * free_netdev(y1);
5838 * free_netdev(y2);
5839 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005840 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005841 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005842 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005843 * without deadlocking with linkwatch via keventd.
5844 * 2) Since we run with the RTNL semaphore not held, we can sleep
5845 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005846 *
5847 * We must not return until all unregister events added during
5848 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005849 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005850void netdev_run_todo(void)
5851{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005852 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005853
Linus Torvalds1da177e2005-04-16 15:20:36 -07005854 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005855 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005856
5857 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005858
Eric W. Biederman850a5452011-10-13 22:25:23 +00005859 /* Wait for rcu callbacks to finish before attempting to drain
5860 * the device list. This usually avoids a 250ms wait.
5861 */
5862 if (!list_empty(&list))
5863 rcu_barrier();
5864
Linus Torvalds1da177e2005-04-16 15:20:36 -07005865 while (!list_empty(&list)) {
5866 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005867 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005868 list_del(&dev->todo_list);
5869
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005870 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005871 printk(KERN_ERR "network todo '%s' but state %d\n",
5872 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005873 dump_stack();
5874 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005875 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005876
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005877 dev->reg_state = NETREG_UNREGISTERED;
5878
Changli Gao152102c2010-03-30 20:16:22 +00005879 on_each_cpu(flush_backlog, dev, 1);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005880
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005881 netdev_wait_allrefs(dev);
5882
5883 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00005884 BUG_ON(netdev_refcnt_read(dev));
Eric Dumazet33d480c2011-08-11 19:30:52 +00005885 WARN_ON(rcu_access_pointer(dev->ip_ptr));
5886 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005887 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005888
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005889 if (dev->destructor)
5890 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005891
5892 /* Free network device */
5893 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005894 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005895}
5896
Ben Hutchings3cfde792010-07-09 09:11:52 +00005897/* Convert net_device_stats to rtnl_link_stats64. They have the same
5898 * fields in the same order, with only the type differing.
5899 */
5900static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5901 const struct net_device_stats *netdev_stats)
5902{
5903#if BITS_PER_LONG == 64
5904 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5905 memcpy(stats64, netdev_stats, sizeof(*stats64));
5906#else
5907 size_t i, n = sizeof(*stats64) / sizeof(u64);
5908 const unsigned long *src = (const unsigned long *)netdev_stats;
5909 u64 *dst = (u64 *)stats64;
5910
5911 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5912 sizeof(*stats64) / sizeof(u64));
5913 for (i = 0; i < n; i++)
5914 dst[i] = src[i];
5915#endif
5916}
5917
Eric Dumazetd83345a2009-11-16 03:36:51 +00005918/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005919 * dev_get_stats - get network device statistics
5920 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07005921 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005922 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00005923 * Get network statistics from device. Return @storage.
5924 * The device driver may provide its own method by setting
5925 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5926 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005927 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00005928struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5929 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005930{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005931 const struct net_device_ops *ops = dev->netdev_ops;
5932
Eric Dumazet28172732010-07-07 14:58:56 -07005933 if (ops->ndo_get_stats64) {
5934 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005935 ops->ndo_get_stats64(dev, storage);
5936 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00005937 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005938 } else {
5939 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07005940 }
Eric Dumazetcaf586e2010-09-30 21:06:55 +00005941 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
Eric Dumazet28172732010-07-07 14:58:56 -07005942 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07005943}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005944EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005945
Eric Dumazet24824a02010-10-02 06:11:55 +00005946struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07005947{
Eric Dumazet24824a02010-10-02 06:11:55 +00005948 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07005949
Eric Dumazet24824a02010-10-02 06:11:55 +00005950#ifdef CONFIG_NET_CLS_ACT
5951 if (queue)
5952 return queue;
5953 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5954 if (!queue)
5955 return NULL;
5956 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet24824a02010-10-02 06:11:55 +00005957 queue->qdisc = &noop_qdisc;
5958 queue->qdisc_sleeping = &noop_qdisc;
5959 rcu_assign_pointer(dev->ingress_queue, queue);
5960#endif
5961 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07005962}
5963
Linus Torvalds1da177e2005-04-16 15:20:36 -07005964/**
Tom Herbert36909ea2011-01-09 19:36:31 +00005965 * alloc_netdev_mqs - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005966 * @sizeof_priv: size of private data to allocate space for
5967 * @name: device name format string
5968 * @setup: callback to initialize device
Tom Herbert36909ea2011-01-09 19:36:31 +00005969 * @txqs: the number of TX subqueues to allocate
5970 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005971 *
5972 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005973 * and performs basic initialization. Also allocates subquue structs
Tom Herbert36909ea2011-01-09 19:36:31 +00005974 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005975 */
Tom Herbert36909ea2011-01-09 19:36:31 +00005976struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5977 void (*setup)(struct net_device *),
5978 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005979{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005980 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005981 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005982 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005983
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005984 BUG_ON(strlen(name) >= sizeof(dev->name));
5985
Tom Herbert36909ea2011-01-09 19:36:31 +00005986 if (txqs < 1) {
Tom Herbert55513fb2010-10-18 17:55:58 +00005987 pr_err("alloc_netdev: Unable to allocate device "
5988 "with zero queues.\n");
5989 return NULL;
5990 }
5991
Tom Herbert36909ea2011-01-09 19:36:31 +00005992#ifdef CONFIG_RPS
5993 if (rxqs < 1) {
5994 pr_err("alloc_netdev: Unable to allocate device "
5995 "with zero RX queues.\n");
5996 return NULL;
5997 }
5998#endif
5999
David S. Millerfd2ea0a2008-07-17 01:56:23 -07006000 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006001 if (sizeof_priv) {
6002 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006003 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07006004 alloc_size += sizeof_priv;
6005 }
6006 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006007 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006008
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07006009 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006010 if (!p) {
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07006011 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07006012 return NULL;
6013 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006014
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00006015 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006016 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006017
Eric Dumazet29b44332010-10-11 10:22:12 +00006018 dev->pcpu_refcnt = alloc_percpu(int);
6019 if (!dev->pcpu_refcnt)
Tom Herberte6484932010-10-18 18:04:39 +00006020 goto free_p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006021
Linus Torvalds1da177e2005-04-16 15:20:36 -07006022 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00006023 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006024
Jiri Pirko22bedad2010-04-01 21:22:57 +00006025 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006026 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00006027
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006028 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006029
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07006030 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006031
Herbert Xud565b0a2008-12-15 23:38:52 -08006032 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006033 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00006034 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07006035 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006036 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006037
6038 dev->num_tx_queues = txqs;
6039 dev->real_num_tx_queues = txqs;
6040 if (netif_alloc_netdev_queues(dev))
6041 goto free_all;
6042
6043#ifdef CONFIG_RPS
6044 dev->num_rx_queues = rxqs;
6045 dev->real_num_rx_queues = rxqs;
6046 if (netif_alloc_rx_queues(dev))
6047 goto free_all;
6048#endif
6049
Linus Torvalds1da177e2005-04-16 15:20:36 -07006050 strcpy(dev->name, name);
Vlad Dogarucbda10f2011-01-13 23:38:30 +00006051 dev->group = INIT_NETDEV_GROUP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006052 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006053
David S. Miller8d3bdbd2011-02-08 15:02:50 -08006054free_all:
6055 free_netdev(dev);
6056 return NULL;
6057
Eric Dumazet29b44332010-10-11 10:22:12 +00006058free_pcpu:
6059 free_percpu(dev->pcpu_refcnt);
Tom Herberted9af2e2010-11-09 10:47:30 +00006060 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006061#ifdef CONFIG_RPS
6062 kfree(dev->_rx);
6063#endif
6064
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00006065free_p:
6066 kfree(p);
6067 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006068}
Tom Herbert36909ea2011-01-09 19:36:31 +00006069EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006070
6071/**
6072 * free_netdev - free network device
6073 * @dev: device
6074 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006075 * This function does the last stage of destroying an allocated device
6076 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006077 * If this is the last reference then it will be freed.
6078 */
6079void free_netdev(struct net_device *dev)
6080{
Herbert Xud565b0a2008-12-15 23:38:52 -08006081 struct napi_struct *p, *n;
6082
Denis V. Lunevf3005d72008-04-16 02:02:18 -07006083 release_net(dev_net(dev));
6084
David S. Millere8a04642008-07-17 00:34:19 -07006085 kfree(dev->_tx);
Tom Herbertfe822242010-11-09 10:47:38 +00006086#ifdef CONFIG_RPS
6087 kfree(dev->_rx);
6088#endif
David S. Millere8a04642008-07-17 00:34:19 -07006089
Eric Dumazet33d480c2011-08-11 19:30:52 +00006090 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00006091
Jiri Pirkof001fde2009-05-05 02:48:28 +00006092 /* Flush device addresses */
6093 dev_addr_flush(dev);
6094
Herbert Xud565b0a2008-12-15 23:38:52 -08006095 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6096 netif_napi_del(p);
6097
Eric Dumazet29b44332010-10-11 10:22:12 +00006098 free_percpu(dev->pcpu_refcnt);
6099 dev->pcpu_refcnt = NULL;
6100
Stephen Hemminger3041a062006-05-26 13:25:24 -07006101 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006102 if (dev->reg_state == NETREG_UNINITIALIZED) {
6103 kfree((char *)dev - dev->padded);
6104 return;
6105 }
6106
6107 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6108 dev->reg_state = NETREG_RELEASED;
6109
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07006110 /* will free via device release */
6111 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006112}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006113EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006114
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006115/**
6116 * synchronize_net - Synchronize with packet receive processing
6117 *
6118 * Wait for packets currently being received to be done.
6119 * Does not block later packets from starting.
6120 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09006121void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006122{
6123 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00006124 if (rtnl_is_locked())
6125 synchronize_rcu_expedited();
6126 else
6127 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006128}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006129EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006130
6131/**
Eric Dumazet44a08732009-10-27 07:03:04 +00006132 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07006133 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00006134 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08006135 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07006136 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006137 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00006138 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006139 *
6140 * Callers must hold the rtnl semaphore. You may want
6141 * unregister_netdev() instead of this.
6142 */
6143
Eric Dumazet44a08732009-10-27 07:03:04 +00006144void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006145{
Herbert Xua6620712007-12-12 19:21:56 -08006146 ASSERT_RTNL();
6147
Eric Dumazet44a08732009-10-27 07:03:04 +00006148 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00006149 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00006150 } else {
6151 rollback_registered(dev);
6152 /* Finish processing unregister after unlock */
6153 net_set_todo(dev);
6154 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006155}
Eric Dumazet44a08732009-10-27 07:03:04 +00006156EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006157
6158/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006159 * unregister_netdevice_many - unregister many devices
6160 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006161 */
6162void unregister_netdevice_many(struct list_head *head)
6163{
6164 struct net_device *dev;
6165
6166 if (!list_empty(head)) {
6167 rollback_registered_many(head);
6168 list_for_each_entry(dev, head, unreg_list)
6169 net_set_todo(dev);
6170 }
6171}
Eric Dumazet63c80992009-10-27 07:06:49 +00006172EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00006173
6174/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07006175 * unregister_netdev - remove device from the kernel
6176 * @dev: device
6177 *
6178 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08006179 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006180 *
6181 * This is just a wrapper for unregister_netdevice that takes
6182 * the rtnl semaphore. In general you want to use this and not
6183 * unregister_netdevice.
6184 */
6185void unregister_netdev(struct net_device *dev)
6186{
6187 rtnl_lock();
6188 unregister_netdevice(dev);
6189 rtnl_unlock();
6190}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006191EXPORT_SYMBOL(unregister_netdev);
6192
Eric W. Biedermance286d32007-09-12 13:53:49 +02006193/**
6194 * dev_change_net_namespace - move device to different nethost namespace
6195 * @dev: device
6196 * @net: network namespace
6197 * @pat: If not NULL name pattern to try if the current device name
6198 * is already taken in the destination network namespace.
6199 *
6200 * This function shuts down a device interface and moves it
6201 * to a new network namespace. On success 0 is returned, on
6202 * a failure a netagive errno code is returned.
6203 *
6204 * Callers must hold the rtnl semaphore.
6205 */
6206
6207int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6208{
Eric W. Biedermance286d32007-09-12 13:53:49 +02006209 int err;
6210
6211 ASSERT_RTNL();
6212
6213 /* Don't allow namespace local devices to be moved. */
6214 err = -EINVAL;
6215 if (dev->features & NETIF_F_NETNS_LOCAL)
6216 goto out;
6217
6218 /* Ensure the device has been registrered */
6219 err = -EINVAL;
6220 if (dev->reg_state != NETREG_REGISTERED)
6221 goto out;
6222
6223 /* Get out if there is nothing todo */
6224 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09006225 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02006226 goto out;
6227
6228 /* Pick the destination device name, and ensure
6229 * we can use it in the destination network namespace.
6230 */
6231 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00006232 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006233 /* We get here if we can't use the current device name */
6234 if (!pat)
6235 goto out;
Jiri Pirko1c5cae82011-04-30 01:21:32 +00006236 if (dev_get_valid_name(dev, pat) < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006237 goto out;
6238 }
6239
6240 /*
6241 * And now a mini version of register_netdevice unregister_netdevice.
6242 */
6243
6244 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07006245 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006246
6247 /* And unlink it from device chain */
6248 err = -ENODEV;
6249 unlist_netdevice(dev);
6250
6251 synchronize_net();
6252
6253 /* Shutdown queueing discipline. */
6254 dev_shutdown(dev);
6255
6256 /* Notify protocols, that we are about to destroy
6257 this device. They should clean all the things.
David Lamparter3b27e102010-09-17 03:22:19 +00006258
6259 Note that dev->reg_state stays at NETREG_REGISTERED.
6260 This is wanted because this way 8021q and macvlan know
6261 the device is just moving and can keep their slaves up.
Eric W. Biedermance286d32007-09-12 13:53:49 +02006262 */
6263 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00006264 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermand2237d32011-10-21 06:24:20 +00006265 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006266
6267 /*
6268 * Flush the unicast and multicast chains
6269 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00006270 dev_uc_flush(dev);
Jiri Pirko22bedad2010-04-01 21:22:57 +00006271 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006272
6273 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09006274 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006275
Eric W. Biedermance286d32007-09-12 13:53:49 +02006276 /* If there is an ifindex conflict assign a new one */
6277 if (__dev_get_by_index(net, dev->ifindex)) {
6278 int iflink = (dev->iflink == dev->ifindex);
6279 dev->ifindex = dev_new_index(net);
6280 if (iflink)
6281 dev->iflink = dev->ifindex;
6282 }
6283
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006284 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07006285 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006286 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006287
6288 /* Add the device back in the hashes */
6289 list_netdevice(dev);
6290
6291 /* Notify protocols, that a new device appeared. */
6292 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6293
Eric W. Biedermand90a9092009-12-12 22:11:15 +00006294 /*
6295 * Prevent userspace races by waiting until the network
6296 * device is fully setup before sending notifications.
6297 */
6298 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6299
Eric W. Biedermance286d32007-09-12 13:53:49 +02006300 synchronize_net();
6301 err = 0;
6302out:
6303 return err;
6304}
Johannes Berg463d0182009-07-14 00:33:35 +02006305EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006306
Linus Torvalds1da177e2005-04-16 15:20:36 -07006307static int dev_cpu_callback(struct notifier_block *nfb,
6308 unsigned long action,
6309 void *ocpu)
6310{
6311 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006312 struct sk_buff *skb;
6313 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6314 struct softnet_data *sd, *oldsd;
6315
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07006316 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006317 return NOTIFY_OK;
6318
6319 local_irq_disable();
6320 cpu = smp_processor_id();
6321 sd = &per_cpu(softnet_data, cpu);
6322 oldsd = &per_cpu(softnet_data, oldcpu);
6323
6324 /* Find end of our completion_queue. */
6325 list_skb = &sd->completion_queue;
6326 while (*list_skb)
6327 list_skb = &(*list_skb)->next;
6328 /* Append completion queue from offline CPU. */
6329 *list_skb = oldsd->completion_queue;
6330 oldsd->completion_queue = NULL;
6331
Linus Torvalds1da177e2005-04-16 15:20:36 -07006332 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00006333 if (oldsd->output_queue) {
6334 *sd->output_queue_tailp = oldsd->output_queue;
6335 sd->output_queue_tailp = oldsd->output_queue_tailp;
6336 oldsd->output_queue = NULL;
6337 oldsd->output_queue_tailp = &oldsd->output_queue;
6338 }
Heiko Carstens264524d2011-06-06 20:50:03 +00006339 /* Append NAPI poll list from offline CPU. */
6340 if (!list_empty(&oldsd->poll_list)) {
6341 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6342 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6343 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006344
6345 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6346 local_irq_enable();
6347
6348 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00006349 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6350 netif_rx(skb);
6351 input_queue_head_incr(oldsd);
6352 }
Tom Herbertfec5e652010-04-16 16:01:27 -07006353 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006354 netif_rx(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00006355 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07006356 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006357
6358 return NOTIFY_OK;
6359}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006360
6361
Herbert Xu7f353bf2007-08-10 15:47:58 -07006362/**
Herbert Xub63365a2008-10-23 01:11:29 -07006363 * netdev_increment_features - increment feature set by one
6364 * @all: current feature set
6365 * @one: new feature set
6366 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006367 *
6368 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006369 * @one to the master device with current feature set @all. Will not
6370 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006371 */
Michał Mirosław04ed3e72011-01-24 15:32:47 -08006372u32 netdev_increment_features(u32 all, u32 one, u32 mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006373{
Michał Mirosław1742f182011-04-22 06:31:16 +00006374 if (mask & NETIF_F_GEN_CSUM)
6375 mask |= NETIF_F_ALL_CSUM;
6376 mask |= NETIF_F_VLAN_CHALLENGED;
6377
6378 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6379 all &= one | ~NETIF_F_ALL_FOR_ALL;
6380
Herbert Xub63365a2008-10-23 01:11:29 -07006381 /* If device needs checksumming, downgrade to it. */
Michał Mirosław1742f182011-04-22 06:31:16 +00006382 if (all & (NETIF_F_ALL_CSUM & ~NETIF_F_NO_CSUM))
6383 all &= ~NETIF_F_NO_CSUM;
Herbert Xu7f353bf2007-08-10 15:47:58 -07006384
Michał Mirosław1742f182011-04-22 06:31:16 +00006385 /* If one device supports hw checksumming, set for all. */
6386 if (all & NETIF_F_GEN_CSUM)
6387 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006388
6389 return all;
6390}
Herbert Xub63365a2008-10-23 01:11:29 -07006391EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006392
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006393static struct hlist_head *netdev_create_hash(void)
6394{
6395 int i;
6396 struct hlist_head *hash;
6397
6398 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6399 if (hash != NULL)
6400 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6401 INIT_HLIST_HEAD(&hash[i]);
6402
6403 return hash;
6404}
6405
Eric W. Biederman881d9662007-09-17 11:56:21 -07006406/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006407static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006408{
Eric W. Biederman881d9662007-09-17 11:56:21 -07006409 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006410
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006411 net->dev_name_head = netdev_create_hash();
6412 if (net->dev_name_head == NULL)
6413 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006414
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006415 net->dev_index_head = netdev_create_hash();
6416 if (net->dev_index_head == NULL)
6417 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006418
6419 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006420
6421err_idx:
6422 kfree(net->dev_name_head);
6423err_name:
6424 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006425}
6426
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006427/**
6428 * netdev_drivername - network driver for the device
6429 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006430 *
6431 * Determine network driver for device.
6432 */
David S. Miller3019de12011-06-06 16:41:33 -07006433const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006434{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006435 const struct device_driver *driver;
6436 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07006437 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07006438
6439 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006440 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07006441 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006442
6443 driver = parent->driver;
6444 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07006445 return driver->name;
6446 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006447}
6448
Jason Baronffa10cb2011-08-11 14:36:48 -04006449int __netdev_printk(const char *level, const struct net_device *dev,
Joe Perches256df2f2010-06-27 01:02:35 +00006450 struct va_format *vaf)
6451{
6452 int r;
6453
6454 if (dev && dev->dev.parent)
6455 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6456 netdev_name(dev), vaf);
6457 else if (dev)
6458 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6459 else
6460 r = printk("%s(NULL net_device): %pV", level, vaf);
6461
6462 return r;
6463}
Jason Baronffa10cb2011-08-11 14:36:48 -04006464EXPORT_SYMBOL(__netdev_printk);
Joe Perches256df2f2010-06-27 01:02:35 +00006465
6466int netdev_printk(const char *level, const struct net_device *dev,
6467 const char *format, ...)
6468{
6469 struct va_format vaf;
6470 va_list args;
6471 int r;
6472
6473 va_start(args, format);
6474
6475 vaf.fmt = format;
6476 vaf.va = &args;
6477
6478 r = __netdev_printk(level, dev, &vaf);
6479 va_end(args);
6480
6481 return r;
6482}
6483EXPORT_SYMBOL(netdev_printk);
6484
6485#define define_netdev_printk_level(func, level) \
6486int func(const struct net_device *dev, const char *fmt, ...) \
6487{ \
6488 int r; \
6489 struct va_format vaf; \
6490 va_list args; \
6491 \
6492 va_start(args, fmt); \
6493 \
6494 vaf.fmt = fmt; \
6495 vaf.va = &args; \
6496 \
6497 r = __netdev_printk(level, dev, &vaf); \
6498 va_end(args); \
6499 \
6500 return r; \
6501} \
6502EXPORT_SYMBOL(func);
6503
6504define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6505define_netdev_printk_level(netdev_alert, KERN_ALERT);
6506define_netdev_printk_level(netdev_crit, KERN_CRIT);
6507define_netdev_printk_level(netdev_err, KERN_ERR);
6508define_netdev_printk_level(netdev_warn, KERN_WARNING);
6509define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6510define_netdev_printk_level(netdev_info, KERN_INFO);
6511
Pavel Emelyanov46650792007-10-08 20:38:39 -07006512static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006513{
6514 kfree(net->dev_name_head);
6515 kfree(net->dev_index_head);
6516}
6517
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006518static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006519 .init = netdev_init,
6520 .exit = netdev_exit,
6521};
6522
Pavel Emelyanov46650792007-10-08 20:38:39 -07006523static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006524{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006525 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006526 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006527 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006528 * initial network namespace
6529 */
6530 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006531 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006532 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006533 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006534
6535 /* Ignore unmoveable devices (i.e. loopback) */
6536 if (dev->features & NETIF_F_NETNS_LOCAL)
6537 continue;
6538
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006539 /* Leave virtual devices for the generic cleanup */
6540 if (dev->rtnl_link_ops)
6541 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006542
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006543 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006544 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6545 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006546 if (err) {
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006547 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
Eric W. Biedermance286d32007-09-12 13:53:49 +02006548 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006549 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006550 }
6551 }
6552 rtnl_unlock();
6553}
6554
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006555static void __net_exit default_device_exit_batch(struct list_head *net_list)
6556{
6557 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04006558 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006559 * Do this across as many network namespaces as possible to
6560 * improve batching efficiency.
6561 */
6562 struct net_device *dev;
6563 struct net *net;
6564 LIST_HEAD(dev_kill_list);
6565
6566 rtnl_lock();
6567 list_for_each_entry(net, net_list, exit_list) {
6568 for_each_netdev_reverse(net, dev) {
6569 if (dev->rtnl_link_ops)
6570 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6571 else
6572 unregister_netdevice_queue(dev, &dev_kill_list);
6573 }
6574 }
6575 unregister_netdevice_many(&dev_kill_list);
Eric Dumazetceaaec92011-02-17 22:59:19 +00006576 list_del(&dev_kill_list);
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006577 rtnl_unlock();
6578}
6579
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006580static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006581 .exit = default_device_exit,
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006582 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006583};
6584
Linus Torvalds1da177e2005-04-16 15:20:36 -07006585/*
6586 * Initialize the DEV module. At boot time this walks the device list and
6587 * unhooks any devices that fail to initialise (normally hardware not
6588 * present) and leaves us with a valid list of present and active devices.
6589 *
6590 */
6591
6592/*
6593 * This is called single threaded during boot, so no need
6594 * to take the rtnl semaphore.
6595 */
6596static int __init net_dev_init(void)
6597{
6598 int i, rc = -ENOMEM;
6599
6600 BUG_ON(!dev_boot_phase);
6601
Linus Torvalds1da177e2005-04-16 15:20:36 -07006602 if (dev_proc_init())
6603 goto out;
6604
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006605 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006606 goto out;
6607
6608 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006609 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006610 INIT_LIST_HEAD(&ptype_base[i]);
6611
Eric W. Biederman881d9662007-09-17 11:56:21 -07006612 if (register_pernet_subsys(&netdev_net_ops))
6613 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006614
6615 /*
6616 * Initialise the packet receive queues.
6617 */
6618
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006619 for_each_possible_cpu(i) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006620 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006621
Changli Gaodee42872010-05-02 05:42:16 +00006622 memset(sd, 0, sizeof(*sd));
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006623 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07006624 skb_queue_head_init(&sd->process_queue);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006625 sd->completion_queue = NULL;
6626 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00006627 sd->output_queue = NULL;
6628 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00006629#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006630 sd->csd.func = rps_trigger_softirq;
6631 sd->csd.info = sd;
6632 sd->csd.flags = 0;
6633 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07006634#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006635
Eric Dumazete36fa2f2010-04-19 21:17:14 +00006636 sd->backlog.poll = process_backlog;
6637 sd->backlog.weight = weight_p;
6638 sd->backlog.gro_list = NULL;
6639 sd->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006640 }
6641
Linus Torvalds1da177e2005-04-16 15:20:36 -07006642 dev_boot_phase = 0;
6643
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006644 /* The loopback device is special if any other network devices
6645 * is present in a network namespace the loopback device must
6646 * be present. Since we now dynamically allocate and free the
6647 * loopback device ensure this invariant is maintained by
6648 * keeping the loopback device as the first device on the
6649 * list of network devices. Ensuring the loopback devices
6650 * is the first device that appears and the last network device
6651 * that disappears.
6652 */
6653 if (register_pernet_device(&loopback_net_ops))
6654 goto out;
6655
6656 if (register_pernet_device(&default_device_ops))
6657 goto out;
6658
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006659 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6660 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006661
6662 hotcpu_notifier(dev_cpu_callback, 0);
6663 dst_init();
6664 dev_mcast_init();
6665 rc = 0;
6666out:
6667 return rc;
6668}
6669
6670subsys_initcall(net_dev_init);
6671
Krishna Kumare88721f2009-02-18 17:55:02 -08006672static int __init initialize_hashrnd(void)
6673{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006674 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006675 return 0;
6676}
6677
6678late_initcall_sync(initialize_hashrnd);
6679