blob: a03aab45e84faa04eac4d12124eb5e5d5ed48d91 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
Jesper Juhl02c30a82005-05-05 16:16:16 -070010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080078#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070079#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000082#include <linux/hash.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080084#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070085#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070094#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070095#include <linux/notifier.h>
96#include <linux/skbuff.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020097#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#include <net/sock.h>
99#include <linux/rtnetlink.h>
100#include <linux/proc_fs.h>
101#include <linux/seq_file.h>
102#include <linux/stat.h>
103#include <linux/if_bridge.h>
Patrick McHardyb863ceb2007-07-14 18:55:06 -0700104#include <linux/if_macvlan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#include <net/dst.h>
106#include <net/pkt_sched.h>
107#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000108#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109#include <linux/highmem.h>
110#include <linux/init.h>
111#include <linux/kmod.h>
112#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
Johannes Berg295f4a12007-04-26 20:43:56 -0700116#include <net/wext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500119#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700120#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700121#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700122#include <linux/ctype.h>
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700123#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700124#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700125#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700126#include <net/ip.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700127#include <linux/ipv6.h>
128#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700129#include <linux/jhash.h>
130#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700131#include <trace/events/napi.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700133#include "net-sysfs.h"
134
Herbert Xud565b0a2008-12-15 23:38:52 -0800135/* Instead of increasing this, you should create a hash table. */
136#define MAX_GRO_SKBS 8
137
Herbert Xu5d38a072009-01-04 16:13:40 -0800138/* This should be increased if a protocol with a bigger head is added. */
139#define GRO_MAX_HEAD (MAX_HEADER + 128)
140
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141/*
142 * The list of packet types we will receive (as opposed to discard)
143 * and the routines to invoke.
144 *
145 * Why 16. Because with 16 the only overlap we get on a hash of the
146 * low nibble of the protocol value is RARP/SNAP/X.25.
147 *
148 * NOTE: That is no longer true with the addition of VLAN tags. Not
149 * sure which should go first, but I bet it won't make much
150 * difference if we are running VLANs. The good news is that
151 * this protocol won't be in the list unless compiled in, so
Stephen Hemminger3041a062006-05-26 13:25:24 -0700152 * the average user (w/out VLANs) will not be adversely affected.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 * --BLG
154 *
155 * 0800 IP
156 * 8100 802.1Q VLAN
157 * 0001 802.3
158 * 0002 AX.25
159 * 0004 802.2
160 * 8035 RARP
161 * 0005 SNAP
162 * 0805 X.25
163 * 0806 ARP
164 * 8137 IPX
165 * 0009 Localtalk
166 * 86DD IPv6
167 */
168
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800169#define PTYPE_HASH_SIZE (16)
170#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
171
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172static DEFINE_SPINLOCK(ptype_lock);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800173static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -0700174static struct list_head ptype_all __read_mostly; /* Taps */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700177 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 * semaphore.
179 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800180 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 *
182 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700183 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 * actual updates. This allows pure readers to access the list even
185 * while a writer is preparing to update it.
186 *
187 * To put it another way, dev_base_lock is held for writing only to
188 * protect against pure readers; the rtnl semaphore provides the
189 * protection against other writers.
190 *
191 * See, for example usages, register_netdevice() and
192 * unregister_netdevice(), which must be called with the rtnl
193 * semaphore held.
194 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196EXPORT_SYMBOL(dev_base_lock);
197
Eric W. Biederman881d9662007-09-17 11:56:21 -0700198static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199{
200 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
stephen hemminger08e98972009-11-10 07:20:34 +0000201 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202}
203
Eric W. Biederman881d9662007-09-17 11:56:21 -0700204static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700206 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207}
208
Eric W. Biedermance286d32007-09-12 13:53:49 +0200209/* Device list insertion */
210static int list_netdevice(struct net_device *dev)
211{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900212 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200213
214 ASSERT_RTNL();
215
216 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800217 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000218 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000219 hlist_add_head_rcu(&dev->index_hlist,
220 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200221 write_unlock_bh(&dev_base_lock);
222 return 0;
223}
224
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000225/* Device list removal
226 * caller must respect a RCU grace period before freeing/reusing dev
227 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200228static void unlist_netdevice(struct net_device *dev)
229{
230 ASSERT_RTNL();
231
232 /* Unlink dev from the device chain */
233 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800234 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000235 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000236 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200237 write_unlock_bh(&dev_base_lock);
238}
239
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240/*
241 * Our notifier list
242 */
243
Alan Sternf07d5b92006-05-09 15:23:03 -0700244static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
246/*
247 * Device drivers call our routines to queue packets here. We empty the
248 * queue in the local softnet handler.
249 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700250
251DEFINE_PER_CPU(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700252EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253
David S. Millercf508b12008-07-22 14:16:42 -0700254#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700255/*
David S. Millerc773e842008-07-08 23:13:53 -0700256 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700257 * according to dev->type
258 */
259static const unsigned short netdev_lock_type[] =
260 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
261 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
262 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
263 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
264 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
265 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
266 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
267 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
268 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
269 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
270 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
271 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
272 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800273 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
Dmitry Eremin-Solenikov929122c2009-08-14 20:00:20 +0400274 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000275 ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700276
Jan Engelhardt36cbd3d2009-08-05 10:42:58 -0700277static const char *const netdev_lock_name[] =
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700278 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
279 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
280 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
281 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
282 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
283 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
284 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
285 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
286 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
287 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
288 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
289 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
290 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
Rémi Denis-Courmont2d91d782008-12-17 15:47:29 -0800291 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
Dmitry Eremin-Solenikov929122c2009-08-14 20:00:20 +0400292 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
Sergey Lapinfcb94e42009-06-08 12:18:47 +0000293 "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700294
295static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700296static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700297
298static inline unsigned short netdev_lock_pos(unsigned short dev_type)
299{
300 int i;
301
302 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
303 if (netdev_lock_type[i] == dev_type)
304 return i;
305 /* the last key is used by default */
306 return ARRAY_SIZE(netdev_lock_type) - 1;
307}
308
David S. Millercf508b12008-07-22 14:16:42 -0700309static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
310 unsigned short dev_type)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700311{
312 int i;
313
314 i = netdev_lock_pos(dev_type);
315 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
316 netdev_lock_name[i]);
317}
David S. Millercf508b12008-07-22 14:16:42 -0700318
319static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
320{
321 int i;
322
323 i = netdev_lock_pos(dev->type);
324 lockdep_set_class_and_name(&dev->addr_list_lock,
325 &netdev_addr_lock_key[i],
326 netdev_lock_name[i]);
327}
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700328#else
David S. Millercf508b12008-07-22 14:16:42 -0700329static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
330 unsigned short dev_type)
331{
332}
333static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b2007-05-15 22:46:18 -0700334{
335}
336#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337
338/*******************************************************************************
339
340 Protocol management and registration routines
341
342*******************************************************************************/
343
344/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 * Add a protocol ID to the list. Now that the input handler is
346 * smarter we can dispense with all the messy stuff that used to be
347 * here.
348 *
349 * BEWARE!!! Protocol handlers, mangling input packets,
350 * MUST BE last in hash buckets and checking protocol handlers
351 * MUST start from promiscuous ptype_all chain in net_bh.
352 * It is true now, do not change it.
353 * Explanation follows: if protocol handler, mangling packet, will
354 * be the first on list, it is not able to sense, that packet
355 * is cloned and should be copied-on-write, so that it will
356 * change it and subsequent readers will get broken packet.
357 * --ANK (980803)
358 */
359
360/**
361 * dev_add_pack - add packet handler
362 * @pt: packet type declaration
363 *
364 * Add a protocol handler to the networking stack. The passed &packet_type
365 * is linked into kernel lists and may not be freed until it has been
366 * removed from the kernel lists.
367 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900368 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369 * guarantee all CPU's that are in middle of receiving packets
370 * will see the new packet type (until the next received packet).
371 */
372
373void dev_add_pack(struct packet_type *pt)
374{
375 int hash;
376
377 spin_lock_bh(&ptype_lock);
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700378 if (pt->type == htons(ETH_P_ALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 list_add_rcu(&pt->list, &ptype_all);
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700380 else {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800381 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 list_add_rcu(&pt->list, &ptype_base[hash]);
383 }
384 spin_unlock_bh(&ptype_lock);
385}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700386EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388/**
389 * __dev_remove_pack - remove packet handler
390 * @pt: packet type declaration
391 *
392 * Remove a protocol handler that was previously added to the kernel
393 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
394 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900395 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 *
397 * The packet type might still be in use by receivers
398 * and must not be freed until after all the CPU's have gone
399 * through a quiescent state.
400 */
401void __dev_remove_pack(struct packet_type *pt)
402{
403 struct list_head *head;
404 struct packet_type *pt1;
405
406 spin_lock_bh(&ptype_lock);
407
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700408 if (pt->type == htons(ETH_P_ALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 head = &ptype_all;
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -0700410 else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +0800411 head = &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
413 list_for_each_entry(pt1, head, list) {
414 if (pt == pt1) {
415 list_del_rcu(&pt->list);
416 goto out;
417 }
418 }
419
420 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
421out:
422 spin_unlock_bh(&ptype_lock);
423}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700424EXPORT_SYMBOL(__dev_remove_pack);
425
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426/**
427 * dev_remove_pack - remove packet handler
428 * @pt: packet type declaration
429 *
430 * Remove a protocol handler that was previously added to the kernel
431 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
432 * from the kernel lists and can be freed or reused once this function
433 * returns.
434 *
435 * This call sleeps to guarantee that no CPU is looking at the packet
436 * type after return.
437 */
438void dev_remove_pack(struct packet_type *pt)
439{
440 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 synchronize_net();
443}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700444EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
446/******************************************************************************
447
448 Device Boot-time Settings Routines
449
450*******************************************************************************/
451
452/* Boot time configuration table */
453static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
454
455/**
456 * netdev_boot_setup_add - add new setup entry
457 * @name: name of the device
458 * @map: configured settings for the device
459 *
460 * Adds new setup entry to the dev_boot_setup list. The function
461 * returns 0 on error and 1 on success. This is a generic routine to
462 * all netdevices.
463 */
464static int netdev_boot_setup_add(char *name, struct ifmap *map)
465{
466 struct netdev_boot_setup *s;
467 int i;
468
469 s = dev_boot_setup;
470 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
471 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
472 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff2008-07-01 19:57:19 -0700473 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 memcpy(&s[i].map, map, sizeof(s[i].map));
475 break;
476 }
477 }
478
479 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
480}
481
482/**
483 * netdev_boot_setup_check - check boot time settings
484 * @dev: the netdevice
485 *
486 * Check boot time settings for the device.
487 * The found settings are set for the device to be used
488 * later in the device probing.
489 * Returns 0 if no settings found, 1 if they are.
490 */
491int netdev_boot_setup_check(struct net_device *dev)
492{
493 struct netdev_boot_setup *s = dev_boot_setup;
494 int i;
495
496 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
497 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff2008-07-01 19:57:19 -0700498 !strcmp(dev->name, s[i].name)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 dev->irq = s[i].map.irq;
500 dev->base_addr = s[i].map.base_addr;
501 dev->mem_start = s[i].map.mem_start;
502 dev->mem_end = s[i].map.mem_end;
503 return 1;
504 }
505 }
506 return 0;
507}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700508EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509
510
511/**
512 * netdev_boot_base - get address from boot time settings
513 * @prefix: prefix for network device
514 * @unit: id for network device
515 *
516 * Check boot time settings for the base address of device.
517 * The found settings are set for the device to be used
518 * later in the device probing.
519 * Returns 0 if no settings found.
520 */
521unsigned long netdev_boot_base(const char *prefix, int unit)
522{
523 const struct netdev_boot_setup *s = dev_boot_setup;
524 char name[IFNAMSIZ];
525 int i;
526
527 sprintf(name, "%s%d", prefix, unit);
528
529 /*
530 * If device already registered then return base of 1
531 * to indicate not to probe for this interface
532 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700533 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 return 1;
535
536 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
537 if (!strcmp(name, s[i].name))
538 return s[i].map.base_addr;
539 return 0;
540}
541
542/*
543 * Saves at boot time configured settings for any netdevice.
544 */
545int __init netdev_boot_setup(char *str)
546{
547 int ints[5];
548 struct ifmap map;
549
550 str = get_options(str, ARRAY_SIZE(ints), ints);
551 if (!str || !*str)
552 return 0;
553
554 /* Save settings */
555 memset(&map, 0, sizeof(map));
556 if (ints[0] > 0)
557 map.irq = ints[1];
558 if (ints[0] > 1)
559 map.base_addr = ints[2];
560 if (ints[0] > 2)
561 map.mem_start = ints[3];
562 if (ints[0] > 3)
563 map.mem_end = ints[4];
564
565 /* Add new entry to the list */
566 return netdev_boot_setup_add(str, &map);
567}
568
569__setup("netdev=", netdev_boot_setup);
570
571/*******************************************************************************
572
573 Device Interface Subroutines
574
575*******************************************************************************/
576
577/**
578 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700579 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 * @name: name to find
581 *
582 * Find an interface by name. Must be called under RTNL semaphore
583 * or @dev_base_lock. If the name is found a pointer to the device
584 * is returned. If the name is not found then %NULL is returned. The
585 * reference counters are not incremented so the caller must be
586 * careful with locks.
587 */
588
Eric W. Biederman881d9662007-09-17 11:56:21 -0700589struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590{
591 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700592 struct net_device *dev;
593 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700595 hlist_for_each_entry(dev, p, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 if (!strncmp(dev->name, name, IFNAMSIZ))
597 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700598
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 return NULL;
600}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700601EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602
603/**
Eric Dumazet72c95282009-10-30 07:11:27 +0000604 * dev_get_by_name_rcu - find a device by its name
605 * @net: the applicable net namespace
606 * @name: name to find
607 *
608 * Find an interface by name.
609 * If the name is found a pointer to the device is returned.
610 * If the name is not found then %NULL is returned.
611 * The reference counters are not incremented so the caller must be
612 * careful with locks. The caller must hold RCU lock.
613 */
614
615struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
616{
617 struct hlist_node *p;
618 struct net_device *dev;
619 struct hlist_head *head = dev_name_hash(net, name);
620
621 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
622 if (!strncmp(dev->name, name, IFNAMSIZ))
623 return dev;
624
625 return NULL;
626}
627EXPORT_SYMBOL(dev_get_by_name_rcu);
628
629/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700631 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 * @name: name to find
633 *
634 * Find an interface by name. This can be called from any
635 * context and does its own locking. The returned handle has
636 * the usage count incremented and the caller must use dev_put() to
637 * release it when it is no longer needed. %NULL is returned if no
638 * matching device is found.
639 */
640
Eric W. Biederman881d9662007-09-17 11:56:21 -0700641struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642{
643 struct net_device *dev;
644
Eric Dumazet72c95282009-10-30 07:11:27 +0000645 rcu_read_lock();
646 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 if (dev)
648 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000649 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 return dev;
651}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700652EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653
654/**
655 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700656 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 * @ifindex: index of device
658 *
659 * Search for an interface by index. Returns %NULL if the device
660 * is not found or a pointer to the device. The device has not
661 * had its reference counter increased so the caller must be careful
662 * about locking. The caller must hold either the RTNL semaphore
663 * or @dev_base_lock.
664 */
665
Eric W. Biederman881d9662007-09-17 11:56:21 -0700666struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667{
668 struct hlist_node *p;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700669 struct net_device *dev;
670 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700672 hlist_for_each_entry(dev, p, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 if (dev->ifindex == ifindex)
674 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700675
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676 return NULL;
677}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700678EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000680/**
681 * dev_get_by_index_rcu - find a device by its ifindex
682 * @net: the applicable net namespace
683 * @ifindex: index of device
684 *
685 * Search for an interface by index. Returns %NULL if the device
686 * is not found or a pointer to the device. The device has not
687 * had its reference counter increased so the caller must be careful
688 * about locking. The caller must hold RCU lock.
689 */
690
691struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
692{
693 struct hlist_node *p;
694 struct net_device *dev;
695 struct hlist_head *head = dev_index_hash(net, ifindex);
696
697 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
698 if (dev->ifindex == ifindex)
699 return dev;
700
701 return NULL;
702}
703EXPORT_SYMBOL(dev_get_by_index_rcu);
704
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705
706/**
707 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700708 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 * @ifindex: index of device
710 *
711 * Search for an interface by index. Returns NULL if the device
712 * is not found or a pointer to the device. The device returned has
713 * had a reference added and the pointer is safe until the user calls
714 * dev_put to indicate they have finished with it.
715 */
716
Eric W. Biederman881d9662007-09-17 11:56:21 -0700717struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718{
719 struct net_device *dev;
720
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000721 rcu_read_lock();
722 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 if (dev)
724 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000725 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 return dev;
727}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700728EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729
730/**
731 * dev_getbyhwaddr - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700732 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 * @type: media type of device
734 * @ha: hardware address
735 *
736 * Search for an interface by MAC address. Returns NULL if the device
737 * is not found or a pointer to the device. The caller must hold the
738 * rtnl semaphore. The returned device has not had its ref count increased
739 * and the caller must therefore be careful about locking
740 *
741 * BUGS:
742 * If the API was consistent this would be __dev_get_by_hwaddr
743 */
744
Eric W. Biederman881d9662007-09-17 11:56:21 -0700745struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746{
747 struct net_device *dev;
748
749 ASSERT_RTNL();
750
Denis V. Lunev81103a52007-12-12 10:47:38 -0800751 for_each_netdev(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 if (dev->type == type &&
753 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700754 return dev;
755
756 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757}
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300758EXPORT_SYMBOL(dev_getbyhwaddr);
759
Eric W. Biederman881d9662007-09-17 11:56:21 -0700760struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700761{
762 struct net_device *dev;
763
764 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700765 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700766 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700767 return dev;
768
769 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700770}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700771EXPORT_SYMBOL(__dev_getfirstbyhwtype);
772
Eric W. Biederman881d9662007-09-17 11:56:21 -0700773struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000775 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000777 rcu_read_lock();
778 for_each_netdev_rcu(net, dev)
779 if (dev->type == type) {
780 dev_hold(dev);
781 ret = dev;
782 break;
783 }
784 rcu_read_unlock();
785 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787EXPORT_SYMBOL(dev_getfirstbyhwtype);
788
789/**
790 * dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700791 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 * @if_flags: IFF_* values
793 * @mask: bitmask of bits in if_flags to check
794 *
795 * Search for any interface with the given flags. Returns NULL if a device
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900796 * is not found or a pointer to the device. The device returned has
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 * had a reference added and the pointer is safe until the user calls
798 * dev_put to indicate they have finished with it.
799 */
800
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700801struct net_device *dev_get_by_flags(struct net *net, unsigned short if_flags,
802 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803{
Pavel Emelianov7562f872007-05-03 15:13:45 -0700804 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805
Pavel Emelianov7562f872007-05-03 15:13:45 -0700806 ret = NULL;
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800807 rcu_read_lock();
808 for_each_netdev_rcu(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 if (((dev->flags ^ if_flags) & mask) == 0) {
810 dev_hold(dev);
Pavel Emelianov7562f872007-05-03 15:13:45 -0700811 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 break;
813 }
814 }
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800815 rcu_read_unlock();
Pavel Emelianov7562f872007-05-03 15:13:45 -0700816 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700818EXPORT_SYMBOL(dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819
820/**
821 * dev_valid_name - check if name is okay for network device
822 * @name: name string
823 *
824 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -0700825 * to allow sysfs to work. We also disallow any kind of
826 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 */
Mitch Williamsc2373ee2005-11-09 10:34:45 -0800828int dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829{
David S. Millerc7fa9d12006-08-15 16:34:13 -0700830 if (*name == '\0')
831 return 0;
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -0700832 if (strlen(name) >= IFNAMSIZ)
833 return 0;
David S. Millerc7fa9d12006-08-15 16:34:13 -0700834 if (!strcmp(name, ".") || !strcmp(name, ".."))
835 return 0;
836
837 while (*name) {
838 if (*name == '/' || isspace(*name))
839 return 0;
840 name++;
841 }
842 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700844EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845
846/**
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200847 * __dev_alloc_name - allocate a name for a device
848 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 * @name: name format string
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200850 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 *
852 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -0700853 * id. It scans list of devices to build up a free map, then chooses
854 * the first empty slot. The caller must hold the dev_base or rtnl lock
855 * while allocating the name and adding the device in order to avoid
856 * duplicates.
857 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
858 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 */
860
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200861static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862{
863 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864 const char *p;
865 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700866 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867 struct net_device *d;
868
869 p = strnchr(name, IFNAMSIZ-1, '%');
870 if (p) {
871 /*
872 * Verify the string as this thing may have come from
873 * the user. There must be either one "%d" and no other "%"
874 * characters.
875 */
876 if (p[1] != 'd' || strchr(p + 2, '%'))
877 return -EINVAL;
878
879 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -0700880 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 if (!inuse)
882 return -ENOMEM;
883
Eric W. Biederman881d9662007-09-17 11:56:21 -0700884 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700885 if (!sscanf(d->name, name, &i))
886 continue;
887 if (i < 0 || i >= max_netdevices)
888 continue;
889
890 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200891 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 if (!strncmp(buf, d->name, IFNAMSIZ))
893 set_bit(i, inuse);
894 }
895
896 i = find_first_zero_bit(inuse, max_netdevices);
897 free_page((unsigned long) inuse);
898 }
899
Octavian Purdilad9031022009-11-18 02:36:59 +0000900 if (buf != name)
901 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200902 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904
905 /* It is possible to run out of possible slots
906 * when the name is long and there isn't enough space left
907 * for the digits, or if all bits are used.
908 */
909 return -ENFILE;
910}
911
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200912/**
913 * dev_alloc_name - allocate a name for a device
914 * @dev: device
915 * @name: name format string
916 *
917 * Passed a format string - eg "lt%d" it will try and find a suitable
918 * id. It scans list of devices to build up a free map, then chooses
919 * the first empty slot. The caller must hold the dev_base or rtnl lock
920 * while allocating the name and adding the device in order to avoid
921 * duplicates.
922 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
923 * Returns the number of the unit assigned or a negative errno code.
924 */
925
926int dev_alloc_name(struct net_device *dev, const char *name)
927{
928 char buf[IFNAMSIZ];
929 struct net *net;
930 int ret;
931
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900932 BUG_ON(!dev_net(dev));
933 net = dev_net(dev);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200934 ret = __dev_alloc_name(net, name, buf);
935 if (ret >= 0)
936 strlcpy(dev->name, buf, IFNAMSIZ);
937 return ret;
938}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700939EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b172007-09-12 13:48:45 +0200940
Octavian Purdilad9031022009-11-18 02:36:59 +0000941static int dev_get_valid_name(struct net *net, const char *name, char *buf,
942 bool fmt)
943{
944 if (!dev_valid_name(name))
945 return -EINVAL;
946
947 if (fmt && strchr(name, '%'))
948 return __dev_alloc_name(net, name, buf);
949 else if (__dev_get_by_name(net, name))
950 return -EEXIST;
951 else if (buf != name)
952 strlcpy(buf, name, IFNAMSIZ);
953
954 return 0;
955}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956
957/**
958 * dev_change_name - change name of a device
959 * @dev: device
960 * @newname: name (or format string) must be at least IFNAMSIZ
961 *
962 * Change name of a device, can pass format strings "eth%d".
963 * for wildcarding.
964 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -0700965int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966{
Herbert Xufcc5a032007-07-30 17:03:38 -0700967 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -0700969 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -0700970 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971
972 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900973 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900975 net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 if (dev->flags & IFF_UP)
977 return -EBUSY;
978
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -0700979 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
980 return 0;
981
Herbert Xufcc5a032007-07-30 17:03:38 -0700982 memcpy(oldname, dev->name, IFNAMSIZ);
983
Octavian Purdilad9031022009-11-18 02:36:59 +0000984 err = dev_get_valid_name(net, newname, dev->name, 1);
985 if (err < 0)
986 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987
Herbert Xufcc5a032007-07-30 17:03:38 -0700988rollback:
Eric W. Biederman38918452008-10-27 17:51:47 -0700989 /* For now only devices in the initial network namespace
990 * are in sysfs.
991 */
Octavian Purdila09ad9bc2009-11-25 15:14:13 -0800992 if (net_eq(net, &init_net)) {
Eric W. Biederman38918452008-10-27 17:51:47 -0700993 ret = device_rename(&dev->dev, dev->name);
994 if (ret) {
995 memcpy(dev->name, oldname, IFNAMSIZ);
996 return ret;
997 }
Stephen Hemmingerdcc99772008-05-14 22:33:38 -0700998 }
Herbert Xu7f988ea2007-07-30 16:35:46 -0700999
1000 write_lock_bh(&dev_base_lock);
Eric W. Biederman92749822007-04-03 00:07:30 -06001001 hlist_del(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001002 write_unlock_bh(&dev_base_lock);
1003
1004 synchronize_rcu();
1005
1006 write_lock_bh(&dev_base_lock);
1007 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988ea2007-07-30 16:35:46 -07001008 write_unlock_bh(&dev_base_lock);
1009
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001010 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001011 ret = notifier_to_errno(ret);
1012
1013 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001014 /* err >= 0 after dev_alloc_name() or stores the first errno */
1015 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001016 err = ret;
1017 memcpy(dev->name, oldname, IFNAMSIZ);
1018 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001019 } else {
1020 printk(KERN_ERR
1021 "%s: name change rollback failed: %d.\n",
1022 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001023 }
1024 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025
1026 return err;
1027}
1028
1029/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001030 * dev_set_alias - change ifalias of a device
1031 * @dev: device
1032 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001033 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001034 *
1035 * Set ifalias for a device,
1036 */
1037int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1038{
1039 ASSERT_RTNL();
1040
1041 if (len >= IFALIASZ)
1042 return -EINVAL;
1043
Oliver Hartkopp96ca4a22008-09-23 21:23:19 -07001044 if (!len) {
1045 if (dev->ifalias) {
1046 kfree(dev->ifalias);
1047 dev->ifalias = NULL;
1048 }
1049 return 0;
1050 }
1051
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001052 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001053 if (!dev->ifalias)
1054 return -ENOMEM;
1055
1056 strlcpy(dev->ifalias, alias, len+1);
1057 return len;
1058}
1059
1060
1061/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001062 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001063 * @dev: device to cause notification
1064 *
1065 * Called to indicate a device has changed features.
1066 */
1067void netdev_features_change(struct net_device *dev)
1068{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001069 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001070}
1071EXPORT_SYMBOL(netdev_features_change);
1072
1073/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 * netdev_state_change - device changes state
1075 * @dev: device to cause notification
1076 *
1077 * Called to indicate a device has changed state. This function calls
1078 * the notifier chains for netdev_chain and sends a NEWLINK message
1079 * to the routing socket.
1080 */
1081void netdev_state_change(struct net_device *dev)
1082{
1083 if (dev->flags & IFF_UP) {
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001084 call_netdevice_notifiers(NETDEV_CHANGE, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1086 }
1087}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001088EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001090int netdev_bonding_change(struct net_device *dev, unsigned long event)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001091{
Jiri Pirko3ca5b402010-03-10 10:29:35 +00001092 return call_netdevice_notifiers(event, dev);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001093}
1094EXPORT_SYMBOL(netdev_bonding_change);
1095
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096/**
1097 * dev_load - load a network module
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001098 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 * @name: name of interface
1100 *
1101 * If a network interface is not present and the process has suitable
1102 * privileges this function loads the module. If module loading is not
1103 * available in this kernel then it becomes a nop.
1104 */
1105
Eric W. Biederman881d9662007-09-17 11:56:21 -07001106void dev_load(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001107{
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001108 struct net_device *dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109
Eric Dumazet72c95282009-10-30 07:11:27 +00001110 rcu_read_lock();
1111 dev = dev_get_by_name_rcu(net, name);
1112 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113
Eric Parisa8f80e82009-08-13 09:44:51 -04001114 if (!dev && capable(CAP_NET_ADMIN))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 request_module("%s", name);
1116}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001117EXPORT_SYMBOL(dev_load);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118
Patrick McHardybd380812010-02-26 06:34:53 +00001119static int __dev_open(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001121 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001122 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001124 ASSERT_RTNL();
1125
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 * Is it even present?
1128 */
1129 if (!netif_device_present(dev))
1130 return -ENODEV;
1131
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001132 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1133 ret = notifier_to_errno(ret);
1134 if (ret)
1135 return ret;
1136
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 /*
1138 * Call device private open method
1139 */
1140 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001141
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001142 if (ops->ndo_validate_addr)
1143 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001144
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001145 if (!ret && ops->ndo_open)
1146 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001147
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001148 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149 * If it went open OK then:
1150 */
1151
Jeff Garzikbada3392007-10-23 20:19:37 -07001152 if (ret)
1153 clear_bit(__LINK_STATE_START, &dev->state);
1154 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 /*
1156 * Set the flags.
1157 */
1158 dev->flags |= IFF_UP;
1159
1160 /*
Dan Williams649274d2009-01-11 00:20:39 -08001161 * Enable NET_DMA
1162 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001163 net_dmaengine_get();
Dan Williams649274d2009-01-11 00:20:39 -08001164
1165 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 * Initialize multicasting status
1167 */
Patrick McHardy4417da62007-06-27 01:28:10 -07001168 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001169
1170 /*
1171 * Wakeup transmit queue engine
1172 */
1173 dev_activate(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001175
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 return ret;
1177}
Patrick McHardybd380812010-02-26 06:34:53 +00001178
1179/**
1180 * dev_open - prepare an interface for use.
1181 * @dev: device to open
1182 *
1183 * Takes a device from down to up state. The device's private open
1184 * function is invoked and then the multicast lists are loaded. Finally
1185 * the device is moved into the up state and a %NETDEV_UP message is
1186 * sent to the netdev notifier chain.
1187 *
1188 * Calling this function on an active interface is a nop. On a failure
1189 * a negative errno code is returned.
1190 */
1191int dev_open(struct net_device *dev)
1192{
1193 int ret;
1194
1195 /*
1196 * Is it already up?
1197 */
1198 if (dev->flags & IFF_UP)
1199 return 0;
1200
1201 /*
1202 * Open device
1203 */
1204 ret = __dev_open(dev);
1205 if (ret < 0)
1206 return ret;
1207
1208 /*
1209 * ... and announce new interface.
1210 */
1211 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1212 call_netdevice_notifiers(NETDEV_UP, dev);
1213
1214 return ret;
1215}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001216EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217
Patrick McHardybd380812010-02-26 06:34:53 +00001218static int __dev_close(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001220 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardybd380812010-02-26 06:34:53 +00001221
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001222 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001223 might_sleep();
1224
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225 /*
1226 * Tell people we are going down, so that they can
1227 * prepare to death, when device is still operating.
1228 */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001229 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 clear_bit(__LINK_STATE_START, &dev->state);
1232
1233 /* Synchronize to scheduled poll. We cannot touch poll list,
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001234 * it can be even on different cpu. So just clear netif_running().
1235 *
1236 * dev->stop() will invoke napi_disable() on all of it's
1237 * napi_struct instances on this device.
1238 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239 smp_mb__after_clear_bit(); /* Commit netif_running(). */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001241 dev_deactivate(dev);
1242
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 /*
1244 * Call the device specific close. This cannot fail.
1245 * Only if device is UP
1246 *
1247 * We allow it to be called even after a DETACH hot-plug
1248 * event.
1249 */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001250 if (ops->ndo_stop)
1251 ops->ndo_stop(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252
1253 /*
1254 * Device is now down.
1255 */
1256
1257 dev->flags &= ~IFF_UP;
1258
1259 /*
Dan Williams649274d2009-01-11 00:20:39 -08001260 * Shutdown NET_DMA
1261 */
David S. Millerb4bd07c2009-02-06 22:06:43 -08001262 net_dmaengine_put();
Dan Williams649274d2009-01-11 00:20:39 -08001263
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264 return 0;
1265}
Patrick McHardybd380812010-02-26 06:34:53 +00001266
1267/**
1268 * dev_close - shutdown an interface.
1269 * @dev: device to shutdown
1270 *
1271 * This function moves an active device into down state. A
1272 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1273 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1274 * chain.
1275 */
1276int dev_close(struct net_device *dev)
1277{
1278 if (!(dev->flags & IFF_UP))
1279 return 0;
1280
1281 __dev_close(dev);
1282
1283 /*
1284 * Tell people we are down
1285 */
1286 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1287 call_netdevice_notifiers(NETDEV_DOWN, dev);
1288
1289 return 0;
1290}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001291EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292
1293
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001294/**
1295 * dev_disable_lro - disable Large Receive Offload on a device
1296 * @dev: device
1297 *
1298 * Disable Large Receive Offload (LRO) on a net device. Must be
1299 * called under RTNL. This is needed if received packets may be
1300 * forwarded to another interface.
1301 */
1302void dev_disable_lro(struct net_device *dev)
1303{
1304 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1305 dev->ethtool_ops->set_flags) {
1306 u32 flags = dev->ethtool_ops->get_flags(dev);
1307 if (flags & ETH_FLAG_LRO) {
1308 flags &= ~ETH_FLAG_LRO;
1309 dev->ethtool_ops->set_flags(dev, flags);
1310 }
1311 }
1312 WARN_ON(dev->features & NETIF_F_LRO);
1313}
1314EXPORT_SYMBOL(dev_disable_lro);
1315
1316
Eric W. Biederman881d9662007-09-17 11:56:21 -07001317static int dev_boot_phase = 1;
1318
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319/*
1320 * Device change register/unregister. These are not inline or static
1321 * as we export them to the world.
1322 */
1323
1324/**
1325 * register_netdevice_notifier - register a network notifier block
1326 * @nb: notifier
1327 *
1328 * Register a notifier to be called when network device events occur.
1329 * The notifier passed is linked into the kernel structures and must
1330 * not be reused until it has been unregistered. A negative errno code
1331 * is returned on a failure.
1332 *
1333 * When registered all registration and up events are replayed
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001334 * to the new notifier to allow device to have a race free
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 * view of the network device list.
1336 */
1337
1338int register_netdevice_notifier(struct notifier_block *nb)
1339{
1340 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001341 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001342 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 int err;
1344
1345 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001346 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001347 if (err)
1348 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001349 if (dev_boot_phase)
1350 goto unlock;
1351 for_each_net(net) {
1352 for_each_netdev(net, dev) {
1353 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1354 err = notifier_to_errno(err);
1355 if (err)
1356 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357
Eric W. Biederman881d9662007-09-17 11:56:21 -07001358 if (!(dev->flags & IFF_UP))
1359 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001360
Eric W. Biederman881d9662007-09-17 11:56:21 -07001361 nb->notifier_call(nb, NETDEV_UP, dev);
1362 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001364
1365unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366 rtnl_unlock();
1367 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001368
1369rollback:
1370 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001371 for_each_net(net) {
1372 for_each_netdev(net, dev) {
1373 if (dev == last)
1374 break;
Herbert Xufcc5a032007-07-30 17:03:38 -07001375
Eric W. Biederman881d9662007-09-17 11:56:21 -07001376 if (dev->flags & IFF_UP) {
1377 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1378 nb->notifier_call(nb, NETDEV_DOWN, dev);
1379 }
1380 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00001381 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001382 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001383 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001384
1385 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001386 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001388EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389
1390/**
1391 * unregister_netdevice_notifier - unregister a network notifier block
1392 * @nb: notifier
1393 *
1394 * Unregister a notifier previously registered by
1395 * register_netdevice_notifier(). The notifier is unlinked into the
1396 * kernel structures and may then be reused. A negative errno code
1397 * is returned on a failure.
1398 */
1399
1400int unregister_netdevice_notifier(struct notifier_block *nb)
1401{
Herbert Xu9f514952006-03-25 01:24:25 -08001402 int err;
1403
1404 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001405 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xu9f514952006-03-25 01:24:25 -08001406 rtnl_unlock();
1407 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001409EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410
1411/**
1412 * call_netdevice_notifiers - call all network notifier blocks
1413 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001414 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 *
1416 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001417 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 */
1419
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001420int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421{
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001422 return raw_notifier_call_chain(&netdev_chain, val, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423}
1424
1425/* When > 0 there are consumers of rx skb time stamps */
1426static atomic_t netstamp_needed = ATOMIC_INIT(0);
1427
1428void net_enable_timestamp(void)
1429{
1430 atomic_inc(&netstamp_needed);
1431}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001432EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433
1434void net_disable_timestamp(void)
1435{
1436 atomic_dec(&netstamp_needed);
1437}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001438EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001440static inline void net_timestamp(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441{
1442 if (atomic_read(&netstamp_needed))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001443 __net_timestamp(skb);
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07001444 else
1445 skb->tstamp.tv64 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446}
1447
Arnd Bergmann44540962009-11-26 06:07:08 +00001448/**
1449 * dev_forward_skb - loopback an skb to another netif
1450 *
1451 * @dev: destination network device
1452 * @skb: buffer to forward
1453 *
1454 * return values:
1455 * NET_RX_SUCCESS (no congestion)
1456 * NET_RX_DROP (packet was dropped)
1457 *
1458 * dev_forward_skb can be used for injecting an skb from the
1459 * start_xmit function of one device into the receive queue
1460 * of another device.
1461 *
1462 * The receiving device may be in another namespace, so
1463 * we have to clear all information in the skb that could
1464 * impact namespace isolation.
1465 */
1466int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1467{
1468 skb_orphan(skb);
1469
1470 if (!(dev->flags & IFF_UP))
1471 return NET_RX_DROP;
1472
1473 if (skb->len > (dev->mtu + dev->hard_header_len))
1474 return NET_RX_DROP;
1475
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001476 skb_set_dev(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001477 skb->tstamp.tv64 = 0;
1478 skb->pkt_type = PACKET_HOST;
1479 skb->protocol = eth_type_trans(skb, dev);
Arnd Bergmann44540962009-11-26 06:07:08 +00001480 return netif_rx(skb);
1481}
1482EXPORT_SYMBOL_GPL(dev_forward_skb);
1483
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484/*
1485 * Support routine. Sends outgoing frames to any network
1486 * taps currently in use.
1487 */
1488
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001489static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490{
1491 struct packet_type *ptype;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001492
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001493#ifdef CONFIG_NET_CLS_ACT
1494 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
1495 net_timestamp(skb);
1496#else
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001497 net_timestamp(skb);
Jarek Poplawski8caf1532009-04-17 10:08:49 +00001498#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499
1500 rcu_read_lock();
1501 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1502 /* Never send packets back to the socket
1503 * they originated from - MvS (miquels@drinkel.ow.org)
1504 */
1505 if ((ptype->dev == dev || !ptype->dev) &&
1506 (ptype->af_packet_priv == NULL ||
1507 (struct sock *)ptype->af_packet_priv != skb->sk)) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001508 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509 if (!skb2)
1510 break;
1511
1512 /* skb->nh should be correctly
1513 set by sender, so that the second statement is
1514 just protection against buggy protocols.
1515 */
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001516 skb_reset_mac_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001518 if (skb_network_header(skb2) < skb2->data ||
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001519 skb2->network_header > skb2->tail) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 if (net_ratelimit())
1521 printk(KERN_CRIT "protocol %04x is "
1522 "buggy, dev %s\n",
1523 skb2->protocol, dev->name);
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07001524 skb_reset_network_header(skb2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525 }
1526
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001527 skb2->transport_header = skb2->network_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 skb2->pkt_type = PACKET_OUTGOING;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07001529 ptype->func(skb2, skb->dev, ptype, skb->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 }
1531 }
1532 rcu_read_unlock();
1533}
1534
Denis Vlasenko56079432006-03-29 15:57:29 -08001535
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001536static inline void __netif_reschedule(struct Qdisc *q)
1537{
1538 struct softnet_data *sd;
1539 unsigned long flags;
1540
1541 local_irq_save(flags);
1542 sd = &__get_cpu_var(softnet_data);
1543 q->next_sched = sd->output_queue;
1544 sd->output_queue = q;
1545 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1546 local_irq_restore(flags);
1547}
1548
David S. Miller37437bb2008-07-16 02:15:04 -07001549void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08001550{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07001551 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1552 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08001553}
1554EXPORT_SYMBOL(__netif_schedule);
1555
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001556void dev_kfree_skb_irq(struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08001557{
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001558 if (atomic_dec_and_test(&skb->users)) {
1559 struct softnet_data *sd;
1560 unsigned long flags;
Denis Vlasenko56079432006-03-29 15:57:29 -08001561
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001562 local_irq_save(flags);
1563 sd = &__get_cpu_var(softnet_data);
1564 skb->next = sd->completion_queue;
1565 sd->completion_queue = skb;
1566 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1567 local_irq_restore(flags);
1568 }
Denis Vlasenko56079432006-03-29 15:57:29 -08001569}
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001570EXPORT_SYMBOL(dev_kfree_skb_irq);
Denis Vlasenko56079432006-03-29 15:57:29 -08001571
1572void dev_kfree_skb_any(struct sk_buff *skb)
1573{
1574 if (in_irq() || irqs_disabled())
1575 dev_kfree_skb_irq(skb);
1576 else
1577 dev_kfree_skb(skb);
1578}
1579EXPORT_SYMBOL(dev_kfree_skb_any);
1580
1581
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001582/**
1583 * netif_device_detach - mark device as removed
1584 * @dev: network device
1585 *
1586 * Mark device as removed from system and therefore no longer available.
1587 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001588void netif_device_detach(struct net_device *dev)
1589{
1590 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1591 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001592 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001593 }
1594}
1595EXPORT_SYMBOL(netif_device_detach);
1596
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001597/**
1598 * netif_device_attach - mark device as attached
1599 * @dev: network device
1600 *
1601 * Mark device as attached from system and restart if needed.
1602 */
Denis Vlasenko56079432006-03-29 15:57:29 -08001603void netif_device_attach(struct net_device *dev)
1604{
1605 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1606 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00001607 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001608 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08001609 }
1610}
1611EXPORT_SYMBOL(netif_device_attach);
1612
Ben Hutchings6de329e2008-06-16 17:02:28 -07001613static bool can_checksum_protocol(unsigned long features, __be16 protocol)
1614{
1615 return ((features & NETIF_F_GEN_CSUM) ||
1616 ((features & NETIF_F_IP_CSUM) &&
1617 protocol == htons(ETH_P_IP)) ||
1618 ((features & NETIF_F_IPV6_CSUM) &&
Yi Zou1c8dbcf2009-02-27 14:06:54 -08001619 protocol == htons(ETH_P_IPV6)) ||
1620 ((features & NETIF_F_FCOE_CRC) &&
1621 protocol == htons(ETH_P_FCOE)));
Ben Hutchings6de329e2008-06-16 17:02:28 -07001622}
1623
1624static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
1625{
1626 if (can_checksum_protocol(dev->features, skb->protocol))
1627 return true;
1628
1629 if (skb->protocol == htons(ETH_P_8021Q)) {
1630 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1631 if (can_checksum_protocol(dev->features & dev->vlan_features,
1632 veh->h_vlan_encapsulated_proto))
1633 return true;
1634 }
1635
1636 return false;
1637}
Denis Vlasenko56079432006-03-29 15:57:29 -08001638
Arnd Bergmann8a83a002010-01-30 12:23:03 +00001639/**
1640 * skb_dev_set -- assign a new device to a buffer
1641 * @skb: buffer for the new device
1642 * @dev: network device
1643 *
1644 * If an skb is owned by a device already, we have to reset
1645 * all data private to the namespace a device belongs to
1646 * before assigning it a new device.
1647 */
1648#ifdef CONFIG_NET_NS
1649void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1650{
1651 skb_dst_drop(skb);
1652 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1653 secpath_reset(skb);
1654 nf_reset(skb);
1655 skb_init_secmark(skb);
1656 skb->mark = 0;
1657 skb->priority = 0;
1658 skb->nf_trace = 0;
1659 skb->ipvs_property = 0;
1660#ifdef CONFIG_NET_SCHED
1661 skb->tc_index = 0;
1662#endif
1663 }
1664 skb->dev = dev;
1665}
1666EXPORT_SYMBOL(skb_set_dev);
1667#endif /* CONFIG_NET_NS */
1668
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669/*
1670 * Invalidate hardware checksum when packet is to be mangled, and
1671 * complete checksum manually on outgoing path.
1672 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001673int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674{
Al Virod3bc23e2006-11-14 21:24:49 -08001675 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07001676 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677
Patrick McHardy84fa7932006-08-29 16:44:56 -07001678 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07001679 goto out_set_summed;
1680
1681 if (unlikely(skb_shinfo(skb)->gso_size)) {
Herbert Xua430a432006-07-08 13:34:56 -07001682 /* Let GSO fix up the checksum. */
1683 goto out_set_summed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001684 }
1685
Herbert Xua0308472007-10-15 01:47:15 -07001686 offset = skb->csum_start - skb_headroom(skb);
1687 BUG_ON(offset >= skb_headlen(skb));
1688 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1689
1690 offset += skb->csum_offset;
1691 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1692
1693 if (skb_cloned(skb) &&
1694 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1696 if (ret)
1697 goto out;
1698 }
1699
Herbert Xua0308472007-10-15 01:47:15 -07001700 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
Herbert Xua430a432006-07-08 13:34:56 -07001701out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001703out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 return ret;
1705}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001706EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001708/**
1709 * skb_gso_segment - Perform segmentation on skb.
1710 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07001711 * @features: features for the output path (see dev->features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001712 *
1713 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07001714 *
1715 * It may return NULL if the skb requires no segmentation. This is
1716 * only possible when GSO is used for verifying header integrity.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001717 */
Herbert Xu576a30e2006-06-27 13:22:38 -07001718struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001719{
1720 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1721 struct packet_type *ptype;
Al Viro252e3342006-11-14 20:48:11 -08001722 __be16 type = skb->protocol;
Herbert Xua430a432006-07-08 13:34:56 -07001723 int err;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001724
Arnaldo Carvalho de Melo459a98e2007-03-19 15:30:44 -07001725 skb_reset_mac_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07001726 skb->mac_len = skb->network_header - skb->mac_header;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001727 __skb_pull(skb, skb->mac_len);
1728
Herbert Xu67fd1a72009-01-19 16:26:44 -08001729 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1730 struct net_device *dev = skb->dev;
1731 struct ethtool_drvinfo info = {};
1732
1733 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1734 dev->ethtool_ops->get_drvinfo(dev, &info);
1735
1736 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
1737 "ip_summed=%d",
1738 info.driver, dev ? dev->features : 0L,
1739 skb->sk ? skb->sk->sk_route_caps : 0L,
1740 skb->len, skb->data_len, skb->ip_summed);
1741
Herbert Xua430a432006-07-08 13:34:56 -07001742 if (skb_header_cloned(skb) &&
1743 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1744 return ERR_PTR(err);
1745 }
1746
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001747 rcu_read_lock();
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08001748 list_for_each_entry_rcu(ptype,
1749 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001750 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001751 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
Herbert Xua430a432006-07-08 13:34:56 -07001752 err = ptype->gso_send_check(skb);
1753 segs = ERR_PTR(err);
1754 if (err || skb_gso_ok(skb, features))
1755 break;
Arnaldo Carvalho de Melod56f90a2007-04-10 20:50:43 -07001756 __skb_push(skb, (skb->data -
1757 skb_network_header(skb)));
Herbert Xua430a432006-07-08 13:34:56 -07001758 }
Herbert Xu576a30e2006-06-27 13:22:38 -07001759 segs = ptype->gso_segment(skb, features);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001760 break;
1761 }
1762 }
1763 rcu_read_unlock();
1764
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001765 __skb_push(skb, skb->data - skb_mac_header(skb));
Herbert Xu576a30e2006-06-27 13:22:38 -07001766
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001767 return segs;
1768}
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001769EXPORT_SYMBOL(skb_gso_segment);
1770
Herbert Xufb286bb2005-11-10 13:01:24 -08001771/* Take action when hardware reception checksum errors are detected. */
1772#ifdef CONFIG_BUG
1773void netdev_rx_csum_fault(struct net_device *dev)
1774{
1775 if (net_ratelimit()) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001776 printk(KERN_ERR "%s: hw csum failure.\n",
Stephen Hemminger246a4212005-12-08 15:21:39 -08001777 dev ? dev->name : "<unknown>");
Herbert Xufb286bb2005-11-10 13:01:24 -08001778 dump_stack();
1779 }
1780}
1781EXPORT_SYMBOL(netdev_rx_csum_fault);
1782#endif
1783
Linus Torvalds1da177e2005-04-16 15:20:36 -07001784/* Actually, we should eliminate this check as soon as we know, that:
1785 * 1. IOMMU is present and allows to map all the memory.
1786 * 2. No high memory really exists on this machine.
1787 */
1788
1789static inline int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1790{
Herbert Xu3d3a8532006-06-27 13:33:10 -07001791#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792 int i;
1793
1794 if (dev->features & NETIF_F_HIGHDMA)
1795 return 0;
1796
1797 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1798 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1799 return 1;
1800
Herbert Xu3d3a8532006-06-27 13:33:10 -07001801#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802 return 0;
1803}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001804
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001805struct dev_gso_cb {
1806 void (*destructor)(struct sk_buff *skb);
1807};
1808
1809#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1810
1811static void dev_gso_skb_destructor(struct sk_buff *skb)
1812{
1813 struct dev_gso_cb *cb;
1814
1815 do {
1816 struct sk_buff *nskb = skb->next;
1817
1818 skb->next = nskb->next;
1819 nskb->next = NULL;
1820 kfree_skb(nskb);
1821 } while (skb->next);
1822
1823 cb = DEV_GSO_CB(skb);
1824 if (cb->destructor)
1825 cb->destructor(skb);
1826}
1827
1828/**
1829 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1830 * @skb: buffer to segment
1831 *
1832 * This function segments the given skb and stores the list of segments
1833 * in skb->next.
1834 */
1835static int dev_gso_segment(struct sk_buff *skb)
1836{
1837 struct net_device *dev = skb->dev;
1838 struct sk_buff *segs;
Herbert Xu576a30e2006-06-27 13:22:38 -07001839 int features = dev->features & ~(illegal_highdma(dev, skb) ?
1840 NETIF_F_SG : 0);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001841
Herbert Xu576a30e2006-06-27 13:22:38 -07001842 segs = skb_gso_segment(skb, features);
1843
1844 /* Verifying header integrity only. */
1845 if (!segs)
1846 return 0;
1847
Hirofumi Nakagawa801678c2008-04-29 01:03:09 -07001848 if (IS_ERR(segs))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001849 return PTR_ERR(segs);
1850
1851 skb->next = segs;
1852 DEV_GSO_CB(skb)->destructor = skb->destructor;
1853 skb->destructor = dev_gso_skb_destructor;
1854
1855 return 0;
1856}
1857
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001858int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1859 struct netdev_queue *txq)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001860{
Stephen Hemminger00829822008-11-20 20:14:53 -08001861 const struct net_device_ops *ops = dev->netdev_ops;
Patrick McHardy572a9d72009-11-10 06:14:14 +00001862 int rc = NETDEV_TX_OK;
Stephen Hemminger00829822008-11-20 20:14:53 -08001863
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001864 if (likely(!skb->next)) {
Stephen Hemminger9be9a6b2007-04-20 17:02:45 -07001865 if (!list_empty(&ptype_all))
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001866 dev_queue_xmit_nit(skb, dev);
1867
Herbert Xu576a30e2006-06-27 13:22:38 -07001868 if (netif_needs_gso(dev, skb)) {
1869 if (unlikely(dev_gso_segment(skb)))
1870 goto out_kfree_skb;
1871 if (skb->next)
1872 goto gso;
1873 }
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001874
Eric Dumazet93f154b2009-05-18 22:19:19 -07001875 /*
1876 * If device doesnt need skb->dst, release it right now while
1877 * its hot in this cpu cache
1878 */
Eric Dumazetadf30902009-06-02 05:19:30 +00001879 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
1880 skb_dst_drop(skb);
1881
Patrick Ohlyac45f602009-02-12 05:03:37 +00001882 rc = ops->ndo_start_xmit(skb, dev);
Patrick McHardyec634fe2009-07-05 19:23:38 -07001883 if (rc == NETDEV_TX_OK)
Eric Dumazet08baf562009-05-25 22:58:01 -07001884 txq_trans_update(txq);
Patrick Ohlyac45f602009-02-12 05:03:37 +00001885 /*
1886 * TODO: if skb_orphan() was called by
1887 * dev->hard_start_xmit() (for example, the unmodified
1888 * igb driver does that; bnx2 doesn't), then
1889 * skb_tx_software_timestamp() will be unable to send
1890 * back the time stamp.
1891 *
1892 * How can this be prevented? Always create another
1893 * reference to the socket before calling
1894 * dev->hard_start_xmit()? Prevent that skb_orphan()
1895 * does anything in dev->hard_start_xmit() by clearing
1896 * the skb destructor before the call and restoring it
1897 * afterwards, then doing the skb_orphan() ourselves?
1898 */
Patrick Ohlyac45f602009-02-12 05:03:37 +00001899 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001900 }
1901
Herbert Xu576a30e2006-06-27 13:22:38 -07001902gso:
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001903 do {
1904 struct sk_buff *nskb = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001905
1906 skb->next = nskb->next;
1907 nskb->next = NULL;
Krishna Kumar068a2de2009-12-09 20:59:58 +00001908
1909 /*
1910 * If device doesnt need nskb->dst, release it right now while
1911 * its hot in this cpu cache
1912 */
1913 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
1914 skb_dst_drop(nskb);
1915
Stephen Hemminger00829822008-11-20 20:14:53 -08001916 rc = ops->ndo_start_xmit(nskb, dev);
Patrick McHardyec634fe2009-07-05 19:23:38 -07001917 if (unlikely(rc != NETDEV_TX_OK)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00001918 if (rc & ~NETDEV_TX_MASK)
1919 goto out_kfree_gso_skb;
Michael Chanf54d9e82006-06-25 23:57:04 -07001920 nskb->next = skb->next;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001921 skb->next = nskb;
1922 return rc;
1923 }
Eric Dumazet08baf562009-05-25 22:58:01 -07001924 txq_trans_update(txq);
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001925 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
Michael Chanf54d9e82006-06-25 23:57:04 -07001926 return NETDEV_TX_BUSY;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001927 } while (skb->next);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09001928
Patrick McHardy572a9d72009-11-10 06:14:14 +00001929out_kfree_gso_skb:
1930 if (likely(skb->next == NULL))
1931 skb->destructor = DEV_GSO_CB(skb)->destructor;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001932out_kfree_skb:
1933 kfree_skb(skb);
Patrick McHardy572a9d72009-11-10 06:14:14 +00001934 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07001935}
1936
Tom Herbert0a9627f2010-03-16 08:03:29 +00001937static u32 hashrnd __read_mostly;
David S. Millerb6b2fed2008-07-21 09:48:06 -07001938
Stephen Hemminger92477442009-03-21 13:39:26 -07001939u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
David S. Miller8f0f2222008-07-15 03:47:03 -07001940{
David S. Miller70192982009-01-27 16:34:47 -08001941 u32 hash;
David S. Millerb6b2fed2008-07-21 09:48:06 -07001942
David S. Miller513de112009-05-03 14:43:10 -07001943 if (skb_rx_queue_recorded(skb)) {
1944 hash = skb_get_rx_queue(skb);
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001945 while (unlikely(hash >= dev->real_num_tx_queues))
David S. Miller513de112009-05-03 14:43:10 -07001946 hash -= dev->real_num_tx_queues;
1947 return hash;
1948 }
Eric Dumazetec581f62009-05-01 09:05:06 -07001949
1950 if (skb->sk && skb->sk->sk_hash)
David S. Miller70192982009-01-27 16:34:47 -08001951 hash = skb->sk->sk_hash;
Eric Dumazetec581f62009-05-01 09:05:06 -07001952 else
David S. Miller70192982009-01-27 16:34:47 -08001953 hash = skb->protocol;
David S. Millerd5a9e242009-01-27 16:22:11 -08001954
Tom Herbert0a9627f2010-03-16 08:03:29 +00001955 hash = jhash_1word(hash, hashrnd);
David S. Millerd5a9e242009-01-27 16:22:11 -08001956
David S. Millerb6b2fed2008-07-21 09:48:06 -07001957 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
David S. Miller8f0f2222008-07-15 03:47:03 -07001958}
Stephen Hemminger92477442009-03-21 13:39:26 -07001959EXPORT_SYMBOL(skb_tx_hash);
David S. Miller8f0f2222008-07-15 03:47:03 -07001960
Eric Dumazeted046422009-11-13 21:54:04 +00001961static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
1962{
1963 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
1964 if (net_ratelimit()) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00001965 netdev_warn(dev, "selects TX queue %d, but "
Eric Dumazeted046422009-11-13 21:54:04 +00001966 "real number of TX queues is %d\n",
Tom Herbert0a9627f2010-03-16 08:03:29 +00001967 queue_index, dev->real_num_tx_queues);
Eric Dumazeted046422009-11-13 21:54:04 +00001968 }
1969 return 0;
1970 }
1971 return queue_index;
1972}
1973
David S. Millere8a04642008-07-17 00:34:19 -07001974static struct netdev_queue *dev_pick_tx(struct net_device *dev,
1975 struct sk_buff *skb)
1976{
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00001977 u16 queue_index;
1978 struct sock *sk = skb->sk;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001979
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00001980 if (sk_tx_queue_recorded(sk)) {
1981 queue_index = sk_tx_queue_get(sk);
1982 } else {
1983 const struct net_device_ops *ops = dev->netdev_ops;
1984
1985 if (ops->ndo_select_queue) {
1986 queue_index = ops->ndo_select_queue(dev, skb);
Eric Dumazeted046422009-11-13 21:54:04 +00001987 queue_index = dev_cap_txqueue(dev, queue_index);
Krishna Kumara4ee3ce2009-10-19 23:50:07 +00001988 } else {
1989 queue_index = 0;
1990 if (dev->real_num_tx_queues > 1)
1991 queue_index = skb_tx_hash(dev, skb);
1992
1993 if (sk && sk->sk_dst_cache)
1994 sk_tx_queue_set(sk, queue_index);
1995 }
1996 }
David S. Millereae792b2008-07-15 03:03:33 -07001997
David S. Millerfd2ea0a2008-07-17 01:56:23 -07001998 skb_set_queue_mapping(skb, queue_index);
1999 return netdev_get_tx_queue(dev, queue_index);
David S. Millere8a04642008-07-17 00:34:19 -07002000}
2001
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002002static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2003 struct net_device *dev,
2004 struct netdev_queue *txq)
2005{
2006 spinlock_t *root_lock = qdisc_lock(q);
2007 int rc;
2008
2009 spin_lock(root_lock);
2010 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2011 kfree_skb(skb);
2012 rc = NET_XMIT_DROP;
2013 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2014 !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) {
2015 /*
2016 * This is a work-conserving queue; there are no old skbs
2017 * waiting to be sent out; and the qdisc is not running -
2018 * xmit the skb directly.
2019 */
2020 __qdisc_update_bstats(q, skb->len);
2021 if (sch_direct_xmit(skb, q, dev, txq, root_lock))
2022 __qdisc_run(q);
2023 else
2024 clear_bit(__QDISC_STATE_RUNNING, &q->state);
2025
2026 rc = NET_XMIT_SUCCESS;
2027 } else {
2028 rc = qdisc_enqueue_root(skb, q);
2029 qdisc_run(q);
2030 }
2031 spin_unlock(root_lock);
2032
2033 return rc;
2034}
2035
Krishna Kumar4b258462010-01-21 01:26:29 -08002036/*
2037 * Returns true if either:
2038 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2039 * 2. skb is fragmented and the device does not support SG, or if
2040 * at least one of fragments is in highmem and device does not
2041 * support DMA from it.
2042 */
2043static inline int skb_needs_linearize(struct sk_buff *skb,
2044 struct net_device *dev)
2045{
2046 return (skb_has_frags(skb) && !(dev->features & NETIF_F_FRAGLIST)) ||
2047 (skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) ||
2048 illegal_highdma(dev, skb)));
2049}
2050
Dave Jonesd29f7492008-07-22 14:09:06 -07002051/**
2052 * dev_queue_xmit - transmit a buffer
2053 * @skb: buffer to transmit
2054 *
2055 * Queue a buffer for transmission to a network device. The caller must
2056 * have set the device and priority and built the buffer before calling
2057 * this function. The function can be called from an interrupt.
2058 *
2059 * A negative errno code is returned on a failure. A success does not
2060 * guarantee the frame will be transmitted as it may be dropped due
2061 * to congestion or traffic shaping.
2062 *
2063 * -----------------------------------------------------------------------------------
2064 * I notice this method can also return errors from the queue disciplines,
2065 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2066 * be positive.
2067 *
2068 * Regardless of the return value, the skb is consumed, so it is currently
2069 * difficult to retry a send to this method. (You can bump the ref count
2070 * before sending to hold a reference for retry if you are careful.)
2071 *
2072 * When calling this method, interrupts MUST be enabled. This is because
2073 * the BH enable code must have IRQs enabled so that it will not deadlock.
2074 * --BLG
2075 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076int dev_queue_xmit(struct sk_buff *skb)
2077{
2078 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07002079 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 struct Qdisc *q;
2081 int rc = -ENOMEM;
2082
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002083 /* GSO will handle the following emulations directly. */
2084 if (netif_needs_gso(dev, skb))
2085 goto gso;
2086
Krishna Kumar4b258462010-01-21 01:26:29 -08002087 /* Convert a paged skb to linear, if required */
2088 if (skb_needs_linearize(skb, dev) && __skb_linearize(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089 goto out_kfree_skb;
2090
2091 /* If packet is not checksummed and device does not support
2092 * checksumming for this protocol, complete checksumming here.
2093 */
Herbert Xu663ead32007-04-09 11:59:07 -07002094 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2095 skb_set_transport_header(skb, skb->csum_start -
2096 skb_headroom(skb));
Ben Hutchings6de329e2008-06-16 17:02:28 -07002097 if (!dev_can_checksum(dev, skb) && skb_checksum_help(skb))
2098 goto out_kfree_skb;
Herbert Xu663ead32007-04-09 11:59:07 -07002099 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100
Herbert Xuf6a78bf2006-06-22 02:57:17 -07002101gso:
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002102 /* Disable soft irqs for various locks below. Also
2103 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002104 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002105 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106
David S. Millereae792b2008-07-15 03:03:33 -07002107 txq = dev_pick_tx(dev, skb);
Paul E. McKenneya898def2010-02-22 17:04:49 -08002108 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07002109
Linus Torvalds1da177e2005-04-16 15:20:36 -07002110#ifdef CONFIG_NET_CLS_ACT
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002111 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002112#endif
2113 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00002114 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07002115 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116 }
2117
2118 /* The device has no queue. Common case for software devices:
2119 loopback, all the sorts of tunnels...
2120
Herbert Xu932ff272006-06-09 12:20:56 -07002121 Really, it is unlikely that netif_tx_lock protection is necessary
2122 here. (f.e. loopback and IP tunnels are clean ignoring statistics
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 counters.)
2124 However, it is possible, that they rely on protection
2125 made by us here.
2126
2127 Check this and shot the lock. It is not prone from deadlocks.
2128 Either shot noqueue qdisc, it is even simpler 8)
2129 */
2130 if (dev->flags & IFF_UP) {
2131 int cpu = smp_processor_id(); /* ok because BHs are off */
2132
David S. Millerc773e842008-07-08 23:13:53 -07002133 if (txq->xmit_lock_owner != cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002134
David S. Millerc773e842008-07-08 23:13:53 -07002135 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002136
David S. Millerfd2ea0a2008-07-17 01:56:23 -07002137 if (!netif_tx_queue_stopped(txq)) {
Patrick McHardy572a9d72009-11-10 06:14:14 +00002138 rc = dev_hard_start_xmit(skb, dev, txq);
2139 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07002140 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002141 goto out;
2142 }
2143 }
David S. Millerc773e842008-07-08 23:13:53 -07002144 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145 if (net_ratelimit())
2146 printk(KERN_CRIT "Virtual device %s asks to "
2147 "queue packet!\n", dev->name);
2148 } else {
2149 /* Recursion is detected! It is possible,
2150 * unfortunately */
2151 if (net_ratelimit())
2152 printk(KERN_CRIT "Dead loop on virtual device "
2153 "%s, fix it urgently!\n", dev->name);
2154 }
2155 }
2156
2157 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07002158 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002159
2160out_kfree_skb:
2161 kfree_skb(skb);
2162 return rc;
2163out:
Herbert Xud4828d82006-06-22 02:28:18 -07002164 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002165 return rc;
2166}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002167EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002168
2169
2170/*=======================================================================
2171 Receiver routines
2172 =======================================================================*/
2173
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07002174int netdev_max_backlog __read_mostly = 1000;
2175int netdev_budget __read_mostly = 300;
2176int weight_p __read_mostly = 64; /* old backlog weight */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002177
2178DEFINE_PER_CPU(struct netif_rx_stats, netdev_rx_stat) = { 0, };
2179
Tom Herbert1e94d722010-03-18 17:45:44 -07002180#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00002181/*
2182 * get_rps_cpu is called from netif_receive_skb and returns the target
2183 * CPU from the RPS map of the receiving queue for a given skb.
2184 */
2185static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb)
2186{
2187 struct ipv6hdr *ip6;
2188 struct iphdr *ip;
2189 struct netdev_rx_queue *rxqueue;
2190 struct rps_map *map;
2191 int cpu = -1;
2192 u8 ip_proto;
2193 u32 addr1, addr2, ports, ihl;
2194
2195 rcu_read_lock();
2196
2197 if (skb_rx_queue_recorded(skb)) {
2198 u16 index = skb_get_rx_queue(skb);
2199 if (unlikely(index >= dev->num_rx_queues)) {
2200 if (net_ratelimit()) {
2201 netdev_warn(dev, "received packet on queue "
2202 "%u, but number of RX queues is %u\n",
2203 index, dev->num_rx_queues);
2204 }
2205 goto done;
2206 }
2207 rxqueue = dev->_rx + index;
2208 } else
2209 rxqueue = dev->_rx;
2210
2211 if (!rxqueue->rps_map)
2212 goto done;
2213
2214 if (skb->rxhash)
2215 goto got_hash; /* Skip hash computation on packet header */
2216
2217 switch (skb->protocol) {
2218 case __constant_htons(ETH_P_IP):
2219 if (!pskb_may_pull(skb, sizeof(*ip)))
2220 goto done;
2221
2222 ip = (struct iphdr *) skb->data;
2223 ip_proto = ip->protocol;
2224 addr1 = ip->saddr;
2225 addr2 = ip->daddr;
2226 ihl = ip->ihl;
2227 break;
2228 case __constant_htons(ETH_P_IPV6):
2229 if (!pskb_may_pull(skb, sizeof(*ip6)))
2230 goto done;
2231
2232 ip6 = (struct ipv6hdr *) skb->data;
2233 ip_proto = ip6->nexthdr;
2234 addr1 = ip6->saddr.s6_addr32[3];
2235 addr2 = ip6->daddr.s6_addr32[3];
2236 ihl = (40 >> 2);
2237 break;
2238 default:
2239 goto done;
2240 }
2241 ports = 0;
2242 switch (ip_proto) {
2243 case IPPROTO_TCP:
2244 case IPPROTO_UDP:
2245 case IPPROTO_DCCP:
2246 case IPPROTO_ESP:
2247 case IPPROTO_AH:
2248 case IPPROTO_SCTP:
2249 case IPPROTO_UDPLITE:
2250 if (pskb_may_pull(skb, (ihl * 4) + 4))
2251 ports = *((u32 *) (skb->data + (ihl * 4)));
2252 break;
2253
2254 default:
2255 break;
2256 }
2257
2258 skb->rxhash = jhash_3words(addr1, addr2, ports, hashrnd);
2259 if (!skb->rxhash)
2260 skb->rxhash = 1;
2261
2262got_hash:
2263 map = rcu_dereference(rxqueue->rps_map);
2264 if (map) {
2265 u16 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
2266
2267 if (cpu_online(tcpu)) {
2268 cpu = tcpu;
2269 goto done;
2270 }
2271 }
2272
2273done:
2274 rcu_read_unlock();
2275 return cpu;
2276}
2277
2278/*
2279 * This structure holds the per-CPU mask of CPUs for which IPIs are scheduled
2280 * to be sent to kick remote softirq processing. There are two masks since
2281 * the sending of IPIs must be done with interrupts enabled. The select field
2282 * indicates the current mask that enqueue_backlog uses to schedule IPIs.
2283 * select is flipped before net_rps_action is called while still under lock,
2284 * net_rps_action then uses the non-selected mask to send the IPIs and clears
2285 * it without conflicting with enqueue_backlog operation.
2286 */
2287struct rps_remote_softirq_cpus {
2288 cpumask_t mask[2];
2289 int select;
2290};
2291static DEFINE_PER_CPU(struct rps_remote_softirq_cpus, rps_remote_softirq_cpus);
2292
2293/* Called from hardirq (IPI) context */
2294static void trigger_softirq(void *data)
2295{
2296 struct softnet_data *queue = data;
2297 __napi_schedule(&queue->backlog);
2298 __get_cpu_var(netdev_rx_stat).received_rps++;
2299}
Tom Herbert1e94d722010-03-18 17:45:44 -07002300#endif /* CONFIG_SMP */
Tom Herbert0a9627f2010-03-16 08:03:29 +00002301
2302/*
2303 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2304 * queue (may be a remote CPU queue).
2305 */
2306static int enqueue_to_backlog(struct sk_buff *skb, int cpu)
2307{
2308 struct softnet_data *queue;
2309 unsigned long flags;
2310
2311 queue = &per_cpu(softnet_data, cpu);
2312
2313 local_irq_save(flags);
2314 __get_cpu_var(netdev_rx_stat).total++;
2315
2316 spin_lock(&queue->input_pkt_queue.lock);
2317 if (queue->input_pkt_queue.qlen <= netdev_max_backlog) {
2318 if (queue->input_pkt_queue.qlen) {
2319enqueue:
2320 __skb_queue_tail(&queue->input_pkt_queue, skb);
2321 spin_unlock_irqrestore(&queue->input_pkt_queue.lock,
2322 flags);
2323 return NET_RX_SUCCESS;
2324 }
2325
2326 /* Schedule NAPI for backlog device */
2327 if (napi_schedule_prep(&queue->backlog)) {
Tom Herbert1e94d722010-03-18 17:45:44 -07002328#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00002329 if (cpu != smp_processor_id()) {
2330 struct rps_remote_softirq_cpus *rcpus =
2331 &__get_cpu_var(rps_remote_softirq_cpus);
2332
2333 cpu_set(cpu, rcpus->mask[rcpus->select]);
2334 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2335 } else
2336 __napi_schedule(&queue->backlog);
Tom Herbert1e94d722010-03-18 17:45:44 -07002337#else
2338 __napi_schedule(&queue->backlog);
2339#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00002340 }
2341 goto enqueue;
2342 }
2343
2344 spin_unlock(&queue->input_pkt_queue.lock);
2345
2346 __get_cpu_var(netdev_rx_stat).dropped++;
2347 local_irq_restore(flags);
2348
2349 kfree_skb(skb);
2350 return NET_RX_DROP;
2351}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352
Linus Torvalds1da177e2005-04-16 15:20:36 -07002353/**
2354 * netif_rx - post buffer to the network code
2355 * @skb: buffer to post
2356 *
2357 * This function receives a packet from a device driver and queues it for
2358 * the upper (protocol) levels to process. It always succeeds. The buffer
2359 * may be dropped during processing for congestion control or by the
2360 * protocol layers.
2361 *
2362 * return values:
2363 * NET_RX_SUCCESS (no congestion)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364 * NET_RX_DROP (packet was dropped)
2365 *
2366 */
2367
2368int netif_rx(struct sk_buff *skb)
2369{
Tom Herbert0a9627f2010-03-16 08:03:29 +00002370 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002371
2372 /* if netpoll wants it, pretend we never saw it */
2373 if (netpoll_rx(skb))
2374 return NET_RX_DROP;
2375
Eric Dumazetb7aa0bf2007-04-19 16:16:32 -07002376 if (!skb->tstamp.tv64)
Patrick McHardya61bbcf2005-08-14 17:24:31 -07002377 net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002378
Tom Herbert1e94d722010-03-18 17:45:44 -07002379#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00002380 cpu = get_rps_cpu(skb->dev, skb);
2381 if (cpu < 0)
2382 cpu = smp_processor_id();
Tom Herbert1e94d722010-03-18 17:45:44 -07002383#else
2384 cpu = smp_processor_id();
2385#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002386
Tom Herbert0a9627f2010-03-16 08:03:29 +00002387 return enqueue_to_backlog(skb, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002388}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002389EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002390
2391int netif_rx_ni(struct sk_buff *skb)
2392{
2393 int err;
2394
2395 preempt_disable();
2396 err = netif_rx(skb);
2397 if (local_softirq_pending())
2398 do_softirq();
2399 preempt_enable();
2400
2401 return err;
2402}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403EXPORT_SYMBOL(netif_rx_ni);
2404
Linus Torvalds1da177e2005-04-16 15:20:36 -07002405static void net_tx_action(struct softirq_action *h)
2406{
2407 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2408
2409 if (sd->completion_queue) {
2410 struct sk_buff *clist;
2411
2412 local_irq_disable();
2413 clist = sd->completion_queue;
2414 sd->completion_queue = NULL;
2415 local_irq_enable();
2416
2417 while (clist) {
2418 struct sk_buff *skb = clist;
2419 clist = clist->next;
2420
Ilpo Järvinen547b7922008-07-25 21:43:18 -07002421 WARN_ON(atomic_read(&skb->users));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002422 __kfree_skb(skb);
2423 }
2424 }
2425
2426 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07002427 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002428
2429 local_irq_disable();
2430 head = sd->output_queue;
2431 sd->output_queue = NULL;
2432 local_irq_enable();
2433
2434 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07002435 struct Qdisc *q = head;
2436 spinlock_t *root_lock;
2437
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438 head = head->next_sched;
2439
David S. Miller5fb66222008-08-02 20:02:43 -07002440 root_lock = qdisc_lock(q);
David S. Miller37437bb2008-07-16 02:15:04 -07002441 if (spin_trylock(root_lock)) {
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002442 smp_mb__before_clear_bit();
2443 clear_bit(__QDISC_STATE_SCHED,
2444 &q->state);
David S. Miller37437bb2008-07-16 02:15:04 -07002445 qdisc_run(q);
2446 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002447 } else {
David S. Miller195648b2008-08-19 04:00:36 -07002448 if (!test_bit(__QDISC_STATE_DEACTIVATED,
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002449 &q->state)) {
David S. Miller195648b2008-08-19 04:00:36 -07002450 __netif_reschedule(q);
Jarek Poplawskie8a83e12008-09-07 18:41:21 -07002451 } else {
2452 smp_mb__before_clear_bit();
2453 clear_bit(__QDISC_STATE_SCHED,
2454 &q->state);
2455 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456 }
2457 }
2458 }
2459}
2460
Stephen Hemminger6f05f622007-03-08 20:46:03 -08002461static inline int deliver_skb(struct sk_buff *skb,
2462 struct packet_type *pt_prev,
2463 struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002464{
2465 atomic_inc(&skb->users);
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002466 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467}
2468
2469#if defined(CONFIG_BRIDGE) || defined (CONFIG_BRIDGE_MODULE)
Michał Mirosławda678292009-06-05 05:35:28 +00002470
2471#if defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)
2472/* This hook is defined here for ATM LANE */
2473int (*br_fdb_test_addr_hook)(struct net_device *dev,
2474 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07002475EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00002476#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002477
Stephen Hemminger6229e362007-03-21 13:38:47 -07002478/*
2479 * If bridge module is loaded call bridging hook.
2480 * returns NULL if packet was consumed.
2481 */
2482struct sk_buff *(*br_handle_frame_hook)(struct net_bridge_port *p,
2483 struct sk_buff *skb) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07002484EXPORT_SYMBOL_GPL(br_handle_frame_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00002485
Stephen Hemminger6229e362007-03-21 13:38:47 -07002486static inline struct sk_buff *handle_bridge(struct sk_buff *skb,
2487 struct packet_type **pt_prev, int *ret,
2488 struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002489{
2490 struct net_bridge_port *port;
2491
Stephen Hemminger6229e362007-03-21 13:38:47 -07002492 if (skb->pkt_type == PACKET_LOOPBACK ||
2493 (port = rcu_dereference(skb->dev->br_port)) == NULL)
2494 return skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002495
2496 if (*pt_prev) {
Stephen Hemminger6229e362007-03-21 13:38:47 -07002497 *ret = deliver_skb(skb, *pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498 *pt_prev = NULL;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002499 }
2500
Stephen Hemminger6229e362007-03-21 13:38:47 -07002501 return br_handle_frame_hook(port, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002502}
2503#else
Stephen Hemminger6229e362007-03-21 13:38:47 -07002504#define handle_bridge(skb, pt_prev, ret, orig_dev) (skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505#endif
2506
Patrick McHardyb863ceb2007-07-14 18:55:06 -07002507#if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE)
2508struct sk_buff *(*macvlan_handle_frame_hook)(struct sk_buff *skb) __read_mostly;
2509EXPORT_SYMBOL_GPL(macvlan_handle_frame_hook);
2510
2511static inline struct sk_buff *handle_macvlan(struct sk_buff *skb,
2512 struct packet_type **pt_prev,
2513 int *ret,
2514 struct net_device *orig_dev)
2515{
2516 if (skb->dev->macvlan_port == NULL)
2517 return skb;
2518
2519 if (*pt_prev) {
2520 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2521 *pt_prev = NULL;
2522 }
2523 return macvlan_handle_frame_hook(skb);
2524}
2525#else
2526#define handle_macvlan(skb, pt_prev, ret, orig_dev) (skb)
2527#endif
2528
Linus Torvalds1da177e2005-04-16 15:20:36 -07002529#ifdef CONFIG_NET_CLS_ACT
2530/* TODO: Maybe we should just force sch_ingress to be compiled in
2531 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2532 * a compare and 2 stores extra right now if we dont have it on
2533 * but have CONFIG_NET_CLS_ACT
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002534 * NOTE: This doesnt stop any functionality; if you dont have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535 * the ingress scheduler, you just cant add policies on ingress.
2536 *
2537 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002538static int ing_filter(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002539{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002540 struct net_device *dev = skb->dev;
Herbert Xuf697c3e2007-10-14 00:38:47 -07002541 u32 ttl = G_TC_RTTL(skb->tc_verd);
David S. Miller555353c2008-07-08 17:33:13 -07002542 struct netdev_queue *rxq;
2543 int result = TC_ACT_OK;
2544 struct Qdisc *q;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002545
Herbert Xuf697c3e2007-10-14 00:38:47 -07002546 if (MAX_RED_LOOP < ttl++) {
2547 printk(KERN_WARNING
2548 "Redir loop detected Dropping packet (%d->%d)\n",
Eric Dumazet8964be42009-11-20 15:35:04 -08002549 skb->skb_iif, dev->ifindex);
Herbert Xuf697c3e2007-10-14 00:38:47 -07002550 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002551 }
2552
Herbert Xuf697c3e2007-10-14 00:38:47 -07002553 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2554 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2555
David S. Miller555353c2008-07-08 17:33:13 -07002556 rxq = &dev->rx_queue;
2557
David S. Miller83874002008-07-17 00:53:03 -07002558 q = rxq->qdisc;
David S. Miller8d50b532008-07-30 02:37:46 -07002559 if (q != &noop_qdisc) {
David S. Miller83874002008-07-17 00:53:03 -07002560 spin_lock(qdisc_lock(q));
David S. Millera9312ae2008-08-17 21:51:03 -07002561 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2562 result = qdisc_enqueue_root(skb, q);
David S. Miller83874002008-07-17 00:53:03 -07002563 spin_unlock(qdisc_lock(q));
2564 }
Herbert Xuf697c3e2007-10-14 00:38:47 -07002565
Linus Torvalds1da177e2005-04-16 15:20:36 -07002566 return result;
2567}
Herbert Xuf697c3e2007-10-14 00:38:47 -07002568
2569static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2570 struct packet_type **pt_prev,
2571 int *ret, struct net_device *orig_dev)
2572{
David S. Miller8d50b532008-07-30 02:37:46 -07002573 if (skb->dev->rx_queue.qdisc == &noop_qdisc)
Herbert Xuf697c3e2007-10-14 00:38:47 -07002574 goto out;
2575
2576 if (*pt_prev) {
2577 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2578 *pt_prev = NULL;
2579 } else {
2580 /* Huh? Why does turning on AF_PACKET affect this? */
2581 skb->tc_verd = SET_TC_OK2MUNGE(skb->tc_verd);
2582 }
2583
2584 switch (ing_filter(skb)) {
2585 case TC_ACT_SHOT:
2586 case TC_ACT_STOLEN:
2587 kfree_skb(skb);
2588 return NULL;
2589 }
2590
2591out:
2592 skb->tc_verd = 0;
2593 return skb;
2594}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002595#endif
2596
Patrick McHardybc1d0412008-07-14 22:49:30 -07002597/*
2598 * netif_nit_deliver - deliver received packets to network taps
2599 * @skb: buffer
2600 *
2601 * This function is used to deliver incoming packets to network
2602 * taps. It should be used when the normal netif_receive_skb path
2603 * is bypassed, for example because of VLAN acceleration.
2604 */
2605void netif_nit_deliver(struct sk_buff *skb)
2606{
2607 struct packet_type *ptype;
2608
2609 if (list_empty(&ptype_all))
2610 return;
2611
2612 skb_reset_network_header(skb);
2613 skb_reset_transport_header(skb);
2614 skb->mac_len = skb->network_header - skb->mac_header;
2615
2616 rcu_read_lock();
2617 list_for_each_entry_rcu(ptype, &ptype_all, list) {
2618 if (!ptype->dev || ptype->dev == skb->dev)
2619 deliver_skb(skb, ptype, skb->dev);
2620 }
2621 rcu_read_unlock();
2622}
2623
Tom Herbert0a9627f2010-03-16 08:03:29 +00002624int __netif_receive_skb(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002625{
2626 struct packet_type *ptype, *pt_prev;
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002627 struct net_device *orig_dev;
Eric Dumazet0641e4f2010-03-18 21:16:45 -07002628 struct net_device *master;
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002629 struct net_device *null_or_orig;
Andy Gospodarekca8d9ea2010-01-06 12:56:37 +00002630 struct net_device *null_or_bond;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002631 int ret = NET_RX_DROP;
Al Viro252e3342006-11-14 20:48:11 -08002632 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002633
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07002634 if (!skb->tstamp.tv64)
2635 net_timestamp(skb);
2636
Eric Dumazet05423b22009-10-26 18:40:35 -07002637 if (vlan_tx_tag_present(skb) && vlan_hwaccel_do_receive(skb))
Patrick McHardy9b22ea52008-11-04 14:49:57 -08002638 return NET_RX_SUCCESS;
2639
Linus Torvalds1da177e2005-04-16 15:20:36 -07002640 /* if we've gotten here through NAPI, check netpoll */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002641 if (netpoll_receive_skb(skb))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002642 return NET_RX_DROP;
2643
Eric Dumazet8964be42009-11-20 15:35:04 -08002644 if (!skb->skb_iif)
2645 skb->skb_iif = skb->dev->ifindex;
David S. Miller86e65da2005-08-09 19:36:29 -07002646
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002647 null_or_orig = NULL;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07002648 orig_dev = skb->dev;
Eric Dumazet0641e4f2010-03-18 21:16:45 -07002649 master = ACCESS_ONCE(orig_dev->master);
2650 if (master) {
2651 if (skb_bond_should_drop(skb, master))
Joe Eykholt0d7a3682008-07-02 18:22:01 -07002652 null_or_orig = orig_dev; /* deliver only exact match */
2653 else
Eric Dumazet0641e4f2010-03-18 21:16:45 -07002654 skb->dev = master;
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07002655 }
Jay Vosburgh8f903c72006-02-21 16:36:44 -08002656
Linus Torvalds1da177e2005-04-16 15:20:36 -07002657 __get_cpu_var(netdev_rx_stat).total++;
2658
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07002659 skb_reset_network_header(skb);
Arnaldo Carvalho de Melobadff6d2007-03-13 13:06:52 -03002660 skb_reset_transport_header(skb);
Arnaldo Carvalho de Melob0e380b2007-04-10 21:21:55 -07002661 skb->mac_len = skb->network_header - skb->mac_header;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002662
2663 pt_prev = NULL;
2664
2665 rcu_read_lock();
2666
2667#ifdef CONFIG_NET_CLS_ACT
2668 if (skb->tc_verd & TC_NCLS) {
2669 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
2670 goto ncls;
2671 }
2672#endif
2673
2674 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Joe Eykholtf9823072008-07-02 18:22:02 -07002675 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2676 ptype->dev == orig_dev) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002677 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002678 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679 pt_prev = ptype;
2680 }
2681 }
2682
2683#ifdef CONFIG_NET_CLS_ACT
Herbert Xuf697c3e2007-10-14 00:38:47 -07002684 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
2685 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002686 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002687ncls:
2688#endif
2689
Stephen Hemminger6229e362007-03-21 13:38:47 -07002690 skb = handle_bridge(skb, &pt_prev, &ret, orig_dev);
2691 if (!skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002692 goto out;
Patrick McHardyb863ceb2007-07-14 18:55:06 -07002693 skb = handle_macvlan(skb, &pt_prev, &ret, orig_dev);
2694 if (!skb)
2695 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002696
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002697 /*
2698 * Make sure frames received on VLAN interfaces stacked on
2699 * bonding interfaces still make their way to any base bonding
2700 * device that may have registered for a specific ptype. The
2701 * handler may have to adjust skb->dev and orig_dev.
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002702 */
Andy Gospodarekca8d9ea2010-01-06 12:56:37 +00002703 null_or_bond = NULL;
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002704 if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
2705 (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
Andy Gospodarekca8d9ea2010-01-06 12:56:37 +00002706 null_or_bond = vlan_dev_real_dev(skb->dev);
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002707 }
2708
Linus Torvalds1da177e2005-04-16 15:20:36 -07002709 type = skb->protocol;
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08002710 list_for_each_entry_rcu(ptype,
2711 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
Andy Gospodarek1f3c8802009-12-14 10:48:58 +00002712 if (ptype->type == type && (ptype->dev == null_or_orig ||
Andy Gospodarekca8d9ea2010-01-06 12:56:37 +00002713 ptype->dev == skb->dev || ptype->dev == orig_dev ||
2714 ptype->dev == null_or_bond)) {
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002715 if (pt_prev)
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002716 ret = deliver_skb(skb, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002717 pt_prev = ptype;
2718 }
2719 }
2720
2721 if (pt_prev) {
David S. Millerf2ccd8f2005-08-09 19:34:12 -07002722 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002723 } else {
2724 kfree_skb(skb);
2725 /* Jamal, now you will not able to escape explaining
2726 * me how you were going to use this. :-)
2727 */
2728 ret = NET_RX_DROP;
2729 }
2730
2731out:
2732 rcu_read_unlock();
2733 return ret;
2734}
Tom Herbert0a9627f2010-03-16 08:03:29 +00002735
2736/**
2737 * netif_receive_skb - process receive buffer from network
2738 * @skb: buffer to process
2739 *
2740 * netif_receive_skb() is the main receive data processing function.
2741 * It always succeeds. The buffer may be dropped during processing
2742 * for congestion control or by the protocol layers.
2743 *
2744 * This function may only be called from softirq context and interrupts
2745 * should be enabled.
2746 *
2747 * Return values (usually ignored):
2748 * NET_RX_SUCCESS: no congestion
2749 * NET_RX_DROP: packet was dropped
2750 */
2751int netif_receive_skb(struct sk_buff *skb)
2752{
Tom Herbert1e94d722010-03-18 17:45:44 -07002753#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00002754 int cpu;
2755
2756 cpu = get_rps_cpu(skb->dev, skb);
2757
2758 if (cpu < 0)
2759 return __netif_receive_skb(skb);
2760 else
2761 return enqueue_to_backlog(skb, cpu);
Tom Herbert1e94d722010-03-18 17:45:44 -07002762#else
2763 return __netif_receive_skb(skb);
2764#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00002765}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002766EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002767
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07002768/* Network device is going away, flush any packets still pending */
2769static void flush_backlog(void *arg)
2770{
2771 struct net_device *dev = arg;
2772 struct softnet_data *queue = &__get_cpu_var(softnet_data);
2773 struct sk_buff *skb, *tmp;
2774
2775 skb_queue_walk_safe(&queue->input_pkt_queue, skb, tmp)
2776 if (skb->dev == dev) {
2777 __skb_unlink(skb, &queue->input_pkt_queue);
2778 kfree_skb(skb);
2779 }
2780}
2781
Herbert Xud565b0a2008-12-15 23:38:52 -08002782static int napi_gro_complete(struct sk_buff *skb)
2783{
2784 struct packet_type *ptype;
2785 __be16 type = skb->protocol;
2786 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
2787 int err = -ENOENT;
2788
Herbert Xufc59f9a2009-04-14 15:11:06 -07002789 if (NAPI_GRO_CB(skb)->count == 1) {
2790 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002791 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07002792 }
Herbert Xud565b0a2008-12-15 23:38:52 -08002793
2794 rcu_read_lock();
2795 list_for_each_entry_rcu(ptype, head, list) {
2796 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
2797 continue;
2798
2799 err = ptype->gro_complete(skb);
2800 break;
2801 }
2802 rcu_read_unlock();
2803
2804 if (err) {
2805 WARN_ON(&ptype->list == head);
2806 kfree_skb(skb);
2807 return NET_RX_SUCCESS;
2808 }
2809
2810out:
Herbert Xud565b0a2008-12-15 23:38:52 -08002811 return netif_receive_skb(skb);
2812}
2813
David S. Miller11380a42010-01-19 13:46:10 -08002814static void napi_gro_flush(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08002815{
2816 struct sk_buff *skb, *next;
2817
2818 for (skb = napi->gro_list; skb; skb = next) {
2819 next = skb->next;
2820 skb->next = NULL;
2821 napi_gro_complete(skb);
2822 }
2823
Herbert Xu4ae55442009-02-08 18:00:36 +00002824 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002825 napi->gro_list = NULL;
2826}
Herbert Xud565b0a2008-12-15 23:38:52 -08002827
Ben Hutchings5b252f02009-10-29 07:17:09 +00002828enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08002829{
2830 struct sk_buff **pp = NULL;
2831 struct packet_type *ptype;
2832 __be16 type = skb->protocol;
2833 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
Herbert Xu0da2afd52008-12-26 14:57:42 -08002834 int same_flow;
Herbert Xud565b0a2008-12-15 23:38:52 -08002835 int mac_len;
Ben Hutchings5b252f02009-10-29 07:17:09 +00002836 enum gro_result ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08002837
2838 if (!(skb->dev->features & NETIF_F_GRO))
2839 goto normal;
2840
David S. Miller4cf704f2009-06-09 00:18:51 -07002841 if (skb_is_gso(skb) || skb_has_frags(skb))
Herbert Xuf17f5c92009-01-14 14:36:12 -08002842 goto normal;
2843
Herbert Xud565b0a2008-12-15 23:38:52 -08002844 rcu_read_lock();
2845 list_for_each_entry_rcu(ptype, head, list) {
Herbert Xud565b0a2008-12-15 23:38:52 -08002846 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
2847 continue;
2848
Herbert Xu86911732009-01-29 14:19:50 +00002849 skb_set_network_header(skb, skb_gro_offset(skb));
Herbert Xud565b0a2008-12-15 23:38:52 -08002850 mac_len = skb->network_header - skb->mac_header;
2851 skb->mac_len = mac_len;
2852 NAPI_GRO_CB(skb)->same_flow = 0;
2853 NAPI_GRO_CB(skb)->flush = 0;
Herbert Xu5d38a072009-01-04 16:13:40 -08002854 NAPI_GRO_CB(skb)->free = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08002855
Herbert Xud565b0a2008-12-15 23:38:52 -08002856 pp = ptype->gro_receive(&napi->gro_list, skb);
2857 break;
2858 }
2859 rcu_read_unlock();
2860
2861 if (&ptype->list == head)
2862 goto normal;
2863
Herbert Xu0da2afd52008-12-26 14:57:42 -08002864 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002865 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08002866
Herbert Xud565b0a2008-12-15 23:38:52 -08002867 if (pp) {
2868 struct sk_buff *nskb = *pp;
2869
2870 *pp = nskb->next;
2871 nskb->next = NULL;
2872 napi_gro_complete(nskb);
Herbert Xu4ae55442009-02-08 18:00:36 +00002873 napi->gro_count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08002874 }
2875
Herbert Xu0da2afd52008-12-26 14:57:42 -08002876 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08002877 goto ok;
2878
Herbert Xu4ae55442009-02-08 18:00:36 +00002879 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
Herbert Xud565b0a2008-12-15 23:38:52 -08002880 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08002881
Herbert Xu4ae55442009-02-08 18:00:36 +00002882 napi->gro_count++;
Herbert Xud565b0a2008-12-15 23:38:52 -08002883 NAPI_GRO_CB(skb)->count = 1;
Herbert Xu86911732009-01-29 14:19:50 +00002884 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08002885 skb->next = napi->gro_list;
2886 napi->gro_list = skb;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002887 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08002888
Herbert Xuad0f9902009-02-01 01:24:55 -08002889pull:
Herbert Xucb189782009-05-26 18:50:31 +00002890 if (skb_headlen(skb) < skb_gro_offset(skb)) {
2891 int grow = skb_gro_offset(skb) - skb_headlen(skb);
2892
2893 BUG_ON(skb->end - skb->tail < grow);
2894
2895 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
2896
2897 skb->tail += grow;
2898 skb->data_len -= grow;
2899
2900 skb_shinfo(skb)->frags[0].page_offset += grow;
2901 skb_shinfo(skb)->frags[0].size -= grow;
2902
2903 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
2904 put_page(skb_shinfo(skb)->frags[0].page);
2905 memmove(skb_shinfo(skb)->frags,
2906 skb_shinfo(skb)->frags + 1,
2907 --skb_shinfo(skb)->nr_frags);
2908 }
Herbert Xuad0f9902009-02-01 01:24:55 -08002909 }
2910
Herbert Xud565b0a2008-12-15 23:38:52 -08002911ok:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002912 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08002913
2914normal:
Herbert Xuad0f9902009-02-01 01:24:55 -08002915 ret = GRO_NORMAL;
2916 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08002917}
Herbert Xu96e93ea2009-01-06 10:49:34 -08002918EXPORT_SYMBOL(dev_gro_receive);
2919
Ben Hutchings5b252f02009-10-29 07:17:09 +00002920static gro_result_t
2921__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08002922{
2923 struct sk_buff *p;
2924
Herbert Xud1c76af2009-03-16 10:50:02 -07002925 if (netpoll_rx_on(skb))
2926 return GRO_NORMAL;
2927
Herbert Xu96e93ea2009-01-06 10:49:34 -08002928 for (p = napi->gro_list; p; p = p->next) {
Joe Perchesf64f9e72009-11-29 16:55:45 -08002929 NAPI_GRO_CB(p)->same_flow =
2930 (p->dev == skb->dev) &&
2931 !compare_ether_header(skb_mac_header(p),
2932 skb_gro_mac_header(skb));
Herbert Xu96e93ea2009-01-06 10:49:34 -08002933 NAPI_GRO_CB(p)->flush = 0;
2934 }
2935
2936 return dev_gro_receive(napi, skb);
2937}
Herbert Xu5d38a072009-01-04 16:13:40 -08002938
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07002939gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08002940{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002941 switch (ret) {
2942 case GRO_NORMAL:
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07002943 if (netif_receive_skb(skb))
2944 ret = GRO_DROP;
2945 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08002946
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002947 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002948 case GRO_MERGED_FREE:
Herbert Xu5d38a072009-01-04 16:13:40 -08002949 kfree_skb(skb);
2950 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00002951
2952 case GRO_HELD:
2953 case GRO_MERGED:
2954 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08002955 }
2956
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07002957 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002958}
2959EXPORT_SYMBOL(napi_skb_finish);
2960
Herbert Xu78a478d2009-05-26 18:50:21 +00002961void skb_gro_reset_offset(struct sk_buff *skb)
2962{
2963 NAPI_GRO_CB(skb)->data_offset = 0;
2964 NAPI_GRO_CB(skb)->frag0 = NULL;
Herbert Xu74895942009-05-26 18:50:27 +00002965 NAPI_GRO_CB(skb)->frag0_len = 0;
Herbert Xu78a478d2009-05-26 18:50:21 +00002966
Herbert Xu78d3fd02009-05-26 18:50:23 +00002967 if (skb->mac_header == skb->tail &&
Herbert Xu74895942009-05-26 18:50:27 +00002968 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
Herbert Xu78a478d2009-05-26 18:50:21 +00002969 NAPI_GRO_CB(skb)->frag0 =
2970 page_address(skb_shinfo(skb)->frags[0].page) +
2971 skb_shinfo(skb)->frags[0].page_offset;
Herbert Xu74895942009-05-26 18:50:27 +00002972 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
2973 }
Herbert Xu78a478d2009-05-26 18:50:21 +00002974}
2975EXPORT_SYMBOL(skb_gro_reset_offset);
2976
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07002977gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002978{
Herbert Xu86911732009-01-29 14:19:50 +00002979 skb_gro_reset_offset(skb);
2980
Herbert Xu5d0d9be2009-01-29 14:19:48 +00002981 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08002982}
2983EXPORT_SYMBOL(napi_gro_receive);
2984
Herbert Xu96e93ea2009-01-06 10:49:34 -08002985void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
2986{
Herbert Xu96e93ea2009-01-06 10:49:34 -08002987 __skb_pull(skb, skb_headlen(skb));
2988 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
2989
2990 napi->skb = skb;
2991}
2992EXPORT_SYMBOL(napi_reuse_skb);
2993
Herbert Xu76620aa2009-04-16 02:02:07 -07002994struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08002995{
Herbert Xu5d38a072009-01-04 16:13:40 -08002996 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08002997
2998 if (!skb) {
Eric Dumazet89d71a62009-10-13 05:34:20 +00002999 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3000 if (skb)
3001 napi->skb = skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08003002 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08003003 return skb;
3004}
Herbert Xu76620aa2009-04-16 02:02:07 -07003005EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003006
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003007gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3008 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003009{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003010 switch (ret) {
3011 case GRO_NORMAL:
Herbert Xu86911732009-01-29 14:19:50 +00003012 case GRO_HELD:
Ajit Khapardee76b69c2010-02-16 20:25:43 +00003013 skb->protocol = eth_type_trans(skb, skb->dev);
Herbert Xu86911732009-01-29 14:19:50 +00003014
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003015 if (ret == GRO_HELD)
3016 skb_gro_pull(skb, -ETH_HLEN);
3017 else if (netif_receive_skb(skb))
3018 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00003019 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003020
3021 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003022 case GRO_MERGED_FREE:
3023 napi_reuse_skb(napi, skb);
3024 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00003025
3026 case GRO_MERGED:
3027 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003028 }
3029
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003030 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003031}
3032EXPORT_SYMBOL(napi_frags_finish);
3033
Herbert Xu76620aa2009-04-16 02:02:07 -07003034struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08003035{
Herbert Xu76620aa2009-04-16 02:02:07 -07003036 struct sk_buff *skb = napi->skb;
3037 struct ethhdr *eth;
Herbert Xua5b1cf22009-05-26 18:50:28 +00003038 unsigned int hlen;
3039 unsigned int off;
Herbert Xu76620aa2009-04-16 02:02:07 -07003040
3041 napi->skb = NULL;
3042
3043 skb_reset_mac_header(skb);
3044 skb_gro_reset_offset(skb);
3045
Herbert Xua5b1cf22009-05-26 18:50:28 +00003046 off = skb_gro_offset(skb);
3047 hlen = off + sizeof(*eth);
3048 eth = skb_gro_header_fast(skb, off);
3049 if (skb_gro_header_hard(skb, hlen)) {
3050 eth = skb_gro_header_slow(skb, hlen, off);
3051 if (unlikely(!eth)) {
3052 napi_reuse_skb(napi, skb);
3053 skb = NULL;
3054 goto out;
3055 }
Herbert Xu76620aa2009-04-16 02:02:07 -07003056 }
3057
3058 skb_gro_pull(skb, sizeof(*eth));
3059
3060 /*
3061 * This works because the only protocols we care about don't require
3062 * special handling. We'll fix it up properly at the end.
3063 */
3064 skb->protocol = eth->h_proto;
3065
3066out:
3067 return skb;
3068}
3069EXPORT_SYMBOL(napi_frags_skb);
3070
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003071gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07003072{
3073 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08003074
3075 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07003076 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08003077
Herbert Xu5d0d9be2009-01-29 14:19:48 +00003078 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
Herbert Xu5d38a072009-01-04 16:13:40 -08003079}
3080EXPORT_SYMBOL(napi_gro_frags);
3081
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003082static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003083{
3084 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085 struct softnet_data *queue = &__get_cpu_var(softnet_data);
3086 unsigned long start_time = jiffies;
3087
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003088 napi->weight = weight_p;
3089 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003090 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003091
Tom Herbert0a9627f2010-03-16 08:03:29 +00003092 spin_lock_irq(&queue->input_pkt_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003093 skb = __skb_dequeue(&queue->input_pkt_queue);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003094 if (!skb) {
Herbert Xu8f1ead22009-03-26 00:59:10 -07003095 __napi_complete(napi);
Tom Herbert0a9627f2010-03-16 08:03:29 +00003096 spin_unlock_irq(&queue->input_pkt_queue.lock);
Herbert Xu8f1ead22009-03-26 00:59:10 -07003097 break;
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003098 }
Tom Herbert0a9627f2010-03-16 08:03:29 +00003099 spin_unlock_irq(&queue->input_pkt_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003100
Tom Herbert0a9627f2010-03-16 08:03:29 +00003101 __netif_receive_skb(skb);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003102 } while (++work < quota && jiffies == start_time);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003103
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003104 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003105}
3106
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003107/**
3108 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07003109 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003110 *
3111 * The entry's receive function will be scheduled to run
3112 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08003113void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003114{
3115 unsigned long flags;
3116
3117 local_irq_save(flags);
3118 list_add_tail(&n->poll_list, &__get_cpu_var(softnet_data).poll_list);
3119 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3120 local_irq_restore(flags);
3121}
3122EXPORT_SYMBOL(__napi_schedule);
3123
Herbert Xud565b0a2008-12-15 23:38:52 -08003124void __napi_complete(struct napi_struct *n)
3125{
3126 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3127 BUG_ON(n->gro_list);
3128
3129 list_del(&n->poll_list);
3130 smp_mb__before_clear_bit();
3131 clear_bit(NAPI_STATE_SCHED, &n->state);
3132}
3133EXPORT_SYMBOL(__napi_complete);
3134
3135void napi_complete(struct napi_struct *n)
3136{
3137 unsigned long flags;
3138
3139 /*
3140 * don't let napi dequeue from the cpu poll list
3141 * just in case its running on a different cpu
3142 */
3143 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3144 return;
3145
3146 napi_gro_flush(n);
3147 local_irq_save(flags);
3148 __napi_complete(n);
3149 local_irq_restore(flags);
3150}
3151EXPORT_SYMBOL(napi_complete);
3152
3153void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3154 int (*poll)(struct napi_struct *, int), int weight)
3155{
3156 INIT_LIST_HEAD(&napi->poll_list);
Herbert Xu4ae55442009-02-08 18:00:36 +00003157 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003158 napi->gro_list = NULL;
Herbert Xu5d38a072009-01-04 16:13:40 -08003159 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08003160 napi->poll = poll;
3161 napi->weight = weight;
3162 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08003163 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08003164#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08003165 spin_lock_init(&napi->poll_lock);
3166 napi->poll_owner = -1;
3167#endif
3168 set_bit(NAPI_STATE_SCHED, &napi->state);
3169}
3170EXPORT_SYMBOL(netif_napi_add);
3171
3172void netif_napi_del(struct napi_struct *napi)
3173{
3174 struct sk_buff *skb, *next;
3175
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08003176 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07003177 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08003178
3179 for (skb = napi->gro_list; skb; skb = next) {
3180 next = skb->next;
3181 skb->next = NULL;
3182 kfree_skb(skb);
3183 }
3184
3185 napi->gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00003186 napi->gro_count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08003187}
3188EXPORT_SYMBOL(netif_napi_del);
3189
Tom Herbert1e94d722010-03-18 17:45:44 -07003190#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00003191/*
3192 * net_rps_action sends any pending IPI's for rps. This is only called from
3193 * softirq and interrupts must be enabled.
3194 */
3195static void net_rps_action(cpumask_t *mask)
3196{
3197 int cpu;
3198
3199 /* Send pending IPI's to kick RPS processing on remote cpus. */
3200 for_each_cpu_mask_nr(cpu, *mask) {
3201 struct softnet_data *queue = &per_cpu(softnet_data, cpu);
3202 if (cpu_online(cpu))
3203 __smp_call_function_single(cpu, &queue->csd, 0);
3204 }
3205 cpus_clear(*mask);
3206}
Tom Herbert1e94d722010-03-18 17:45:44 -07003207#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003208
Linus Torvalds1da177e2005-04-16 15:20:36 -07003209static void net_rx_action(struct softirq_action *h)
3210{
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003211 struct list_head *list = &__get_cpu_var(softnet_data).poll_list;
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003212 unsigned long time_limit = jiffies + 2;
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07003213 int budget = netdev_budget;
Matt Mackall53fb95d2005-08-11 19:27:43 -07003214 void *have;
Tom Herbert1e94d722010-03-18 17:45:44 -07003215#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00003216 int select;
3217 struct rps_remote_softirq_cpus *rcpus;
Tom Herbert1e94d722010-03-18 17:45:44 -07003218#endif
Matt Mackall53fb95d2005-08-11 19:27:43 -07003219
Linus Torvalds1da177e2005-04-16 15:20:36 -07003220 local_irq_disable();
3221
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003222 while (!list_empty(list)) {
3223 struct napi_struct *n;
3224 int work, weight;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003225
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003226 /* If softirq window is exhuasted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003227 * Allow this to run for 2 jiffies since which will allow
3228 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003229 */
Stephen Hemminger24f8b232008-11-03 17:14:38 -08003230 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003231 goto softnet_break;
3232
3233 local_irq_enable();
3234
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003235 /* Even though interrupts have been re-enabled, this
3236 * access is safe because interrupts can only add new
3237 * entries to the tail of this list, and only ->poll()
3238 * calls can remove this head entry from the list.
3239 */
stephen hemmingere5e26d72010-02-24 14:01:38 +00003240 n = list_first_entry(list, struct napi_struct, poll_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003241
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003242 have = netpoll_poll_lock(n);
3243
3244 weight = n->weight;
3245
David S. Miller0a7606c2007-10-29 21:28:47 -07003246 /* This NAPI_STATE_SCHED test is for avoiding a race
3247 * with netpoll's poll_napi(). Only the entity which
3248 * obtains the lock and sees NAPI_STATE_SCHED set will
3249 * actually make the ->poll() call. Therefore we avoid
3250 * accidently calling ->poll() when NAPI is not scheduled.
3251 */
3252 work = 0;
Neil Horman4ea7e382009-05-21 07:36:08 +00003253 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
David S. Miller0a7606c2007-10-29 21:28:47 -07003254 work = n->poll(n, weight);
Neil Horman4ea7e382009-05-21 07:36:08 +00003255 trace_napi_poll(n);
3256 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003257
3258 WARN_ON_ONCE(work > weight);
3259
3260 budget -= work;
3261
3262 local_irq_disable();
3263
3264 /* Drivers must not modify the NAPI state if they
3265 * consume the entire weight. In such cases this code
3266 * still "owns" the NAPI instance and therefore can
3267 * move the instance around on the list at-will.
3268 */
David S. Millerfed17f32008-01-07 21:00:40 -08003269 if (unlikely(work == weight)) {
Herbert Xuff780cd2009-06-26 19:27:04 -07003270 if (unlikely(napi_disable_pending(n))) {
3271 local_irq_enable();
3272 napi_complete(n);
3273 local_irq_disable();
3274 } else
David S. Millerfed17f32008-01-07 21:00:40 -08003275 list_move_tail(&n->poll_list, list);
3276 }
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003277
3278 netpoll_poll_unlock(have);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003279 }
3280out:
Tom Herbert1e94d722010-03-18 17:45:44 -07003281#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00003282 rcpus = &__get_cpu_var(rps_remote_softirq_cpus);
3283 select = rcpus->select;
3284 rcpus->select ^= 1;
3285
Shannon Nelson515e06c2007-06-23 23:09:23 -07003286 local_irq_enable();
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003287
Tom Herbert0a9627f2010-03-16 08:03:29 +00003288 net_rps_action(&rcpus->mask[select]);
Tom Herbert1e94d722010-03-18 17:45:44 -07003289#else
3290 local_irq_enable();
3291#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00003292
Chris Leechdb217332006-06-17 21:24:58 -07003293#ifdef CONFIG_NET_DMA
3294 /*
3295 * There may not be any more sk_buffs coming right now, so push
3296 * any pending DMA copies to hardware
3297 */
Dan Williams2ba05622009-01-06 11:38:14 -07003298 dma_issue_pending_all();
Chris Leechdb217332006-06-17 21:24:58 -07003299#endif
Stephen Hemmingerbea33482007-10-03 16:41:36 -07003300
Linus Torvalds1da177e2005-04-16 15:20:36 -07003301 return;
3302
3303softnet_break:
3304 __get_cpu_var(netdev_rx_stat).time_squeeze++;
3305 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3306 goto out;
3307}
3308
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003309static gifconf_func_t *gifconf_list[NPROTO];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003310
3311/**
3312 * register_gifconf - register a SIOCGIF handler
3313 * @family: Address family
3314 * @gifconf: Function handler
3315 *
3316 * Register protocol dependent address dumping routines. The handler
3317 * that is passed must not be freed or reused until it has been replaced
3318 * by another handler.
3319 */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003320int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003321{
3322 if (family >= NPROTO)
3323 return -EINVAL;
3324 gifconf_list[family] = gifconf;
3325 return 0;
3326}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003327EXPORT_SYMBOL(register_gifconf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003328
3329
3330/*
3331 * Map an interface index to its name (SIOCGIFNAME)
3332 */
3333
3334/*
3335 * We need this ioctl for efficient implementation of the
3336 * if_indextoname() function required by the IPv6 API. Without
3337 * it, we would have to search all the interfaces to find a
3338 * match. --pb
3339 */
3340
Eric W. Biederman881d9662007-09-17 11:56:21 -07003341static int dev_ifname(struct net *net, struct ifreq __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003342{
3343 struct net_device *dev;
3344 struct ifreq ifr;
3345
3346 /*
3347 * Fetch the caller's info block.
3348 */
3349
3350 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3351 return -EFAULT;
3352
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003353 rcu_read_lock();
3354 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003355 if (!dev) {
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003356 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003357 return -ENODEV;
3358 }
3359
3360 strcpy(ifr.ifr_name, dev->name);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +00003361 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003362
3363 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3364 return -EFAULT;
3365 return 0;
3366}
3367
3368/*
3369 * Perform a SIOCGIFCONF call. This structure will change
3370 * size eventually, and there is nothing I can do about it.
3371 * Thus we will need a 'compatibility mode'.
3372 */
3373
Eric W. Biederman881d9662007-09-17 11:56:21 -07003374static int dev_ifconf(struct net *net, char __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003375{
3376 struct ifconf ifc;
3377 struct net_device *dev;
3378 char __user *pos;
3379 int len;
3380 int total;
3381 int i;
3382
3383 /*
3384 * Fetch the caller's info block.
3385 */
3386
3387 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3388 return -EFAULT;
3389
3390 pos = ifc.ifc_buf;
3391 len = ifc.ifc_len;
3392
3393 /*
3394 * Loop over the interfaces, and write an info block for each.
3395 */
3396
3397 total = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003398 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003399 for (i = 0; i < NPROTO; i++) {
3400 if (gifconf_list[i]) {
3401 int done;
3402 if (!pos)
3403 done = gifconf_list[i](dev, NULL, 0);
3404 else
3405 done = gifconf_list[i](dev, pos + total,
3406 len - total);
3407 if (done < 0)
3408 return -EFAULT;
3409 total += done;
3410 }
3411 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003412 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003413
3414 /*
3415 * All done. Write the updated control block back to the caller.
3416 */
3417 ifc.ifc_len = total;
3418
3419 /*
3420 * Both BSD and Solaris return 0 here, so we do too.
3421 */
3422 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3423}
3424
3425#ifdef CONFIG_PROC_FS
3426/*
3427 * This is invoked by the /proc filesystem handler to display a device
3428 * in detail.
3429 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003430void *dev_seq_start(struct seq_file *seq, loff_t *pos)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003431 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003432{
Denis V. Luneve372c412007-11-19 22:31:54 -08003433 struct net *net = seq_file_net(seq);
Pavel Emelianov7562f872007-05-03 15:13:45 -07003434 loff_t off;
3435 struct net_device *dev;
3436
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003437 rcu_read_lock();
Pavel Emelianov7562f872007-05-03 15:13:45 -07003438 if (!*pos)
3439 return SEQ_START_TOKEN;
3440
3441 off = 1;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003442 for_each_netdev_rcu(net, dev)
Pavel Emelianov7562f872007-05-03 15:13:45 -07003443 if (off++ == *pos)
3444 return dev;
3445
3446 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003447}
3448
3449void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3450{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003451 struct net_device *dev = (v == SEQ_START_TOKEN) ?
3452 first_net_device(seq_file_net(seq)) :
3453 next_net_device((struct net_device *)v);
3454
Linus Torvalds1da177e2005-04-16 15:20:36 -07003455 ++*pos;
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003456 return rcu_dereference(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003457}
3458
3459void dev_seq_stop(struct seq_file *seq, void *v)
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003460 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003461{
Eric Dumazetc6d14c82009-11-04 05:43:23 -08003462 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003463}
3464
3465static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
3466{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08003467 const struct net_device_stats *stats = dev_get_stats(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003468
Jesper Dangaard Brouer2d13baf2010-01-05 05:50:52 +00003469 seq_printf(seq, "%6s: %7lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu "
Rusty Russell5a1b5892007-04-28 21:04:03 -07003470 "%8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n",
3471 dev->name, stats->rx_bytes, stats->rx_packets,
3472 stats->rx_errors,
3473 stats->rx_dropped + stats->rx_missed_errors,
3474 stats->rx_fifo_errors,
3475 stats->rx_length_errors + stats->rx_over_errors +
3476 stats->rx_crc_errors + stats->rx_frame_errors,
3477 stats->rx_compressed, stats->multicast,
3478 stats->tx_bytes, stats->tx_packets,
3479 stats->tx_errors, stats->tx_dropped,
3480 stats->tx_fifo_errors, stats->collisions,
3481 stats->tx_carrier_errors +
3482 stats->tx_aborted_errors +
3483 stats->tx_window_errors +
3484 stats->tx_heartbeat_errors,
3485 stats->tx_compressed);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003486}
3487
3488/*
3489 * Called from the PROCfs module. This now uses the new arbitrary sized
3490 * /proc/net interface to create /proc/net/dev
3491 */
3492static int dev_seq_show(struct seq_file *seq, void *v)
3493{
3494 if (v == SEQ_START_TOKEN)
3495 seq_puts(seq, "Inter-| Receive "
3496 " | Transmit\n"
3497 " face |bytes packets errs drop fifo frame "
3498 "compressed multicast|bytes packets errs "
3499 "drop fifo colls carrier compressed\n");
3500 else
3501 dev_seq_printf_stats(seq, v);
3502 return 0;
3503}
3504
3505static struct netif_rx_stats *softnet_get_online(loff_t *pos)
3506{
3507 struct netif_rx_stats *rc = NULL;
3508
Mike Travis0c0b0ac2008-05-02 16:43:08 -07003509 while (*pos < nr_cpu_ids)
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003510 if (cpu_online(*pos)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003511 rc = &per_cpu(netdev_rx_stat, *pos);
3512 break;
3513 } else
3514 ++*pos;
3515 return rc;
3516}
3517
3518static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
3519{
3520 return softnet_get_online(pos);
3521}
3522
3523static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3524{
3525 ++*pos;
3526 return softnet_get_online(pos);
3527}
3528
3529static void softnet_seq_stop(struct seq_file *seq, void *v)
3530{
3531}
3532
3533static int softnet_seq_show(struct seq_file *seq, void *v)
3534{
3535 struct netif_rx_stats *s = v;
3536
Tom Herbert0a9627f2010-03-16 08:03:29 +00003537 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
Stephen Hemminger31aa02c2005-06-23 20:12:48 -07003538 s->total, s->dropped, s->time_squeeze, 0,
Stephen Hemmingerc1ebcdb2005-06-23 20:08:59 -07003539 0, 0, 0, 0, /* was fastroute */
Tom Herbert0a9627f2010-03-16 08:03:29 +00003540 s->cpu_collision, s->received_rps);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003541 return 0;
3542}
3543
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003544static const struct seq_operations dev_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003545 .start = dev_seq_start,
3546 .next = dev_seq_next,
3547 .stop = dev_seq_stop,
3548 .show = dev_seq_show,
3549};
3550
3551static int dev_seq_open(struct inode *inode, struct file *file)
3552{
Denis V. Luneve372c412007-11-19 22:31:54 -08003553 return seq_open_net(inode, file, &dev_seq_ops,
3554 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003555}
3556
Arjan van de Ven9a321442007-02-12 00:55:35 -08003557static const struct file_operations dev_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003558 .owner = THIS_MODULE,
3559 .open = dev_seq_open,
3560 .read = seq_read,
3561 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08003562 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003563};
3564
Stephen Hemmingerf6908082007-03-12 14:34:29 -07003565static const struct seq_operations softnet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003566 .start = softnet_seq_start,
3567 .next = softnet_seq_next,
3568 .stop = softnet_seq_stop,
3569 .show = softnet_seq_show,
3570};
3571
3572static int softnet_seq_open(struct inode *inode, struct file *file)
3573{
3574 return seq_open(file, &softnet_seq_ops);
3575}
3576
Arjan van de Ven9a321442007-02-12 00:55:35 -08003577static const struct file_operations softnet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003578 .owner = THIS_MODULE,
3579 .open = softnet_seq_open,
3580 .read = seq_read,
3581 .llseek = seq_lseek,
3582 .release = seq_release,
3583};
3584
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003585static void *ptype_get_idx(loff_t pos)
3586{
3587 struct packet_type *pt = NULL;
3588 loff_t i = 0;
3589 int t;
3590
3591 list_for_each_entry_rcu(pt, &ptype_all, list) {
3592 if (i == pos)
3593 return pt;
3594 ++i;
3595 }
3596
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003597 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003598 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
3599 if (i == pos)
3600 return pt;
3601 ++i;
3602 }
3603 }
3604 return NULL;
3605}
3606
3607static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
Stephen Hemminger72348a42008-01-21 02:27:29 -08003608 __acquires(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003609{
3610 rcu_read_lock();
3611 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
3612}
3613
3614static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3615{
3616 struct packet_type *pt;
3617 struct list_head *nxt;
3618 int hash;
3619
3620 ++*pos;
3621 if (v == SEQ_START_TOKEN)
3622 return ptype_get_idx(0);
3623
3624 pt = v;
3625 nxt = pt->list.next;
3626 if (pt->type == htons(ETH_P_ALL)) {
3627 if (nxt != &ptype_all)
3628 goto found;
3629 hash = 0;
3630 nxt = ptype_base[0].next;
3631 } else
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003632 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003633
3634 while (nxt == &ptype_base[hash]) {
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08003635 if (++hash >= PTYPE_HASH_SIZE)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003636 return NULL;
3637 nxt = ptype_base[hash].next;
3638 }
3639found:
3640 return list_entry(nxt, struct packet_type, list);
3641}
3642
3643static void ptype_seq_stop(struct seq_file *seq, void *v)
Stephen Hemminger72348a42008-01-21 02:27:29 -08003644 __releases(RCU)
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003645{
3646 rcu_read_unlock();
3647}
3648
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003649static int ptype_seq_show(struct seq_file *seq, void *v)
3650{
3651 struct packet_type *pt = v;
3652
3653 if (v == SEQ_START_TOKEN)
3654 seq_puts(seq, "Type Device Function\n");
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003655 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003656 if (pt->type == htons(ETH_P_ALL))
3657 seq_puts(seq, "ALL ");
3658 else
3659 seq_printf(seq, "%04x", ntohs(pt->type));
3660
Alexey Dobriyan908cd2d2008-11-16 19:50:35 -08003661 seq_printf(seq, " %-8s %pF\n",
3662 pt->dev ? pt->dev->name : "", pt->func);
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003663 }
3664
3665 return 0;
3666}
3667
3668static const struct seq_operations ptype_seq_ops = {
3669 .start = ptype_seq_start,
3670 .next = ptype_seq_next,
3671 .stop = ptype_seq_stop,
3672 .show = ptype_seq_show,
3673};
3674
3675static int ptype_seq_open(struct inode *inode, struct file *file)
3676{
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07003677 return seq_open_net(inode, file, &ptype_seq_ops,
3678 sizeof(struct seq_net_private));
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003679}
3680
3681static const struct file_operations ptype_seq_fops = {
3682 .owner = THIS_MODULE,
3683 .open = ptype_seq_open,
3684 .read = seq_read,
3685 .llseek = seq_lseek,
Pavel Emelyanov2feb27d2008-03-24 14:57:45 -07003686 .release = seq_release_net,
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003687};
3688
3689
Pavel Emelyanov46650792007-10-08 20:38:39 -07003690static int __net_init dev_proc_net_init(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003691{
3692 int rc = -ENOMEM;
3693
Eric W. Biederman881d9662007-09-17 11:56:21 -07003694 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003695 goto out;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003696 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003697 goto out_dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07003698 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02003699 goto out_softnet;
Stephen Hemminger0e1256f2007-03-12 14:35:37 -07003700
Eric W. Biederman881d9662007-09-17 11:56:21 -07003701 if (wext_proc_init(net))
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02003702 goto out_ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003703 rc = 0;
3704out:
3705 return rc;
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02003706out_ptype:
Eric W. Biederman881d9662007-09-17 11:56:21 -07003707 proc_net_remove(net, "ptype");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003708out_softnet:
Eric W. Biederman881d9662007-09-17 11:56:21 -07003709 proc_net_remove(net, "softnet_stat");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003710out_dev:
Eric W. Biederman881d9662007-09-17 11:56:21 -07003711 proc_net_remove(net, "dev");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003712 goto out;
3713}
Eric W. Biederman881d9662007-09-17 11:56:21 -07003714
Pavel Emelyanov46650792007-10-08 20:38:39 -07003715static void __net_exit dev_proc_net_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07003716{
3717 wext_proc_exit(net);
3718
3719 proc_net_remove(net, "ptype");
3720 proc_net_remove(net, "softnet_stat");
3721 proc_net_remove(net, "dev");
3722}
3723
Denis V. Lunev022cbae2007-11-13 03:23:50 -08003724static struct pernet_operations __net_initdata dev_proc_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07003725 .init = dev_proc_net_init,
3726 .exit = dev_proc_net_exit,
3727};
3728
3729static int __init dev_proc_init(void)
3730{
3731 return register_pernet_subsys(&dev_proc_ops);
3732}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003733#else
3734#define dev_proc_init() 0
3735#endif /* CONFIG_PROC_FS */
3736
3737
3738/**
3739 * netdev_set_master - set up master/slave pair
3740 * @slave: slave device
3741 * @master: new master device
3742 *
3743 * Changes the master device of the slave. Pass %NULL to break the
3744 * bonding. The caller must hold the RTNL semaphore. On a failure
3745 * a negative errno code is returned. On success the reference counts
3746 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
3747 * function returns zero.
3748 */
3749int netdev_set_master(struct net_device *slave, struct net_device *master)
3750{
3751 struct net_device *old = slave->master;
3752
3753 ASSERT_RTNL();
3754
3755 if (master) {
3756 if (old)
3757 return -EBUSY;
3758 dev_hold(master);
3759 }
3760
3761 slave->master = master;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003762
Eric Dumazet283f2fe2010-03-18 13:37:40 +00003763 if (old) {
3764 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003765 dev_put(old);
Eric Dumazet283f2fe2010-03-18 13:37:40 +00003766 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003767 if (master)
3768 slave->flags |= IFF_SLAVE;
3769 else
3770 slave->flags &= ~IFF_SLAVE;
3771
3772 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
3773 return 0;
3774}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003775EXPORT_SYMBOL(netdev_set_master);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003776
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003777static void dev_change_rx_flags(struct net_device *dev, int flags)
3778{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003779 const struct net_device_ops *ops = dev->netdev_ops;
3780
3781 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
3782 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003783}
3784
Wang Chendad9b332008-06-18 01:48:28 -07003785static int __dev_set_promiscuity(struct net_device *dev, int inc)
Patrick McHardy4417da62007-06-27 01:28:10 -07003786{
3787 unsigned short old_flags = dev->flags;
David Howells8192b0c2008-11-14 10:39:10 +11003788 uid_t uid;
3789 gid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07003790
Patrick McHardy24023452007-07-14 18:51:31 -07003791 ASSERT_RTNL();
3792
Wang Chendad9b332008-06-18 01:48:28 -07003793 dev->flags |= IFF_PROMISC;
3794 dev->promiscuity += inc;
3795 if (dev->promiscuity == 0) {
3796 /*
3797 * Avoid overflow.
3798 * If inc causes overflow, untouch promisc and return error.
3799 */
3800 if (inc < 0)
3801 dev->flags &= ~IFF_PROMISC;
3802 else {
3803 dev->promiscuity -= inc;
3804 printk(KERN_WARNING "%s: promiscuity touches roof, "
3805 "set promiscuity failed, promiscuity feature "
3806 "of device might be broken.\n", dev->name);
3807 return -EOVERFLOW;
3808 }
3809 }
Patrick McHardy4417da62007-06-27 01:28:10 -07003810 if (dev->flags != old_flags) {
3811 printk(KERN_INFO "device %s %s promiscuous mode\n",
3812 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
3813 "left");
David Howells8192b0c2008-11-14 10:39:10 +11003814 if (audit_enabled) {
3815 current_uid_gid(&uid, &gid);
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05003816 audit_log(current->audit_context, GFP_ATOMIC,
3817 AUDIT_ANOM_PROMISCUOUS,
3818 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
3819 dev->name, (dev->flags & IFF_PROMISC),
3820 (old_flags & IFF_PROMISC),
3821 audit_get_loginuid(current),
David Howells8192b0c2008-11-14 10:39:10 +11003822 uid, gid,
Klaus Heinrich Kiwi7759db82008-01-23 22:57:45 -05003823 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11003824 }
Patrick McHardy24023452007-07-14 18:51:31 -07003825
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003826 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07003827 }
Wang Chendad9b332008-06-18 01:48:28 -07003828 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07003829}
3830
Linus Torvalds1da177e2005-04-16 15:20:36 -07003831/**
3832 * dev_set_promiscuity - update promiscuity count on a device
3833 * @dev: device
3834 * @inc: modifier
3835 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07003836 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07003837 * remains above zero the interface remains promiscuous. Once it hits zero
3838 * the device reverts back to normal filtering operation. A negative inc
3839 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07003840 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003841 */
Wang Chendad9b332008-06-18 01:48:28 -07003842int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003843{
3844 unsigned short old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07003845 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003846
Wang Chendad9b332008-06-18 01:48:28 -07003847 err = __dev_set_promiscuity(dev, inc);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07003848 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07003849 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07003850 if (dev->flags != old_flags)
3851 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07003852 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003853}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003854EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003855
3856/**
3857 * dev_set_allmulti - update allmulti count on a device
3858 * @dev: device
3859 * @inc: modifier
3860 *
3861 * Add or remove reception of all multicast frames to a device. While the
3862 * count in the device remains above zero the interface remains listening
3863 * to all interfaces. Once it hits zero the device reverts back to normal
3864 * filtering operation. A negative @inc value is used to drop the counter
3865 * when releasing a resource needing all multicasts.
Wang Chendad9b332008-06-18 01:48:28 -07003866 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003867 */
3868
Wang Chendad9b332008-06-18 01:48:28 -07003869int dev_set_allmulti(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003870{
3871 unsigned short old_flags = dev->flags;
3872
Patrick McHardy24023452007-07-14 18:51:31 -07003873 ASSERT_RTNL();
3874
Linus Torvalds1da177e2005-04-16 15:20:36 -07003875 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07003876 dev->allmulti += inc;
3877 if (dev->allmulti == 0) {
3878 /*
3879 * Avoid overflow.
3880 * If inc causes overflow, untouch allmulti and return error.
3881 */
3882 if (inc < 0)
3883 dev->flags &= ~IFF_ALLMULTI;
3884 else {
3885 dev->allmulti -= inc;
3886 printk(KERN_WARNING "%s: allmulti touches roof, "
3887 "set allmulti failed, allmulti feature of "
3888 "device might be broken.\n", dev->name);
3889 return -EOVERFLOW;
3890 }
3891 }
Patrick McHardy24023452007-07-14 18:51:31 -07003892 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07003893 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07003894 dev_set_rx_mode(dev);
Patrick McHardy24023452007-07-14 18:51:31 -07003895 }
Wang Chendad9b332008-06-18 01:48:28 -07003896 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07003897}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07003898EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07003899
3900/*
3901 * Upload unicast and multicast address lists to device and
3902 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08003903 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07003904 * are present.
3905 */
3906void __dev_set_rx_mode(struct net_device *dev)
3907{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003908 const struct net_device_ops *ops = dev->netdev_ops;
3909
Patrick McHardy4417da62007-06-27 01:28:10 -07003910 /* dev_open will call this function so the list will stay sane. */
3911 if (!(dev->flags&IFF_UP))
3912 return;
3913
3914 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09003915 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07003916
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003917 if (ops->ndo_set_rx_mode)
3918 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003919 else {
3920 /* Unicast addresses changes may only happen under the rtnl,
3921 * therefore calling __dev_set_promiscuity here is safe.
3922 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08003923 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07003924 __dev_set_promiscuity(dev, 1);
3925 dev->uc_promisc = 1;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08003926 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Patrick McHardy4417da62007-06-27 01:28:10 -07003927 __dev_set_promiscuity(dev, -1);
3928 dev->uc_promisc = 0;
3929 }
3930
Stephen Hemmingerd3147742008-11-19 21:32:24 -08003931 if (ops->ndo_set_multicast_list)
3932 ops->ndo_set_multicast_list(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003933 }
3934}
3935
3936void dev_set_rx_mode(struct net_device *dev)
3937{
David S. Millerb9e40852008-07-15 00:15:08 -07003938 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07003939 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07003940 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003941}
3942
Jiri Pirkof001fde2009-05-05 02:48:28 +00003943/* hw addresses list handling functions */
3944
Jiri Pirko31278e72009-06-17 01:12:19 +00003945static int __hw_addr_add(struct netdev_hw_addr_list *list, unsigned char *addr,
3946 int addr_len, unsigned char addr_type)
Jiri Pirkof001fde2009-05-05 02:48:28 +00003947{
3948 struct netdev_hw_addr *ha;
3949 int alloc_size;
3950
3951 if (addr_len > MAX_ADDR_LEN)
3952 return -EINVAL;
3953
Jiri Pirko31278e72009-06-17 01:12:19 +00003954 list_for_each_entry(ha, &list->list, list) {
Jiri Pirkoccffad252009-05-22 23:22:17 +00003955 if (!memcmp(ha->addr, addr, addr_len) &&
3956 ha->type == addr_type) {
3957 ha->refcount++;
3958 return 0;
3959 }
3960 }
3961
3962
Jiri Pirkof001fde2009-05-05 02:48:28 +00003963 alloc_size = sizeof(*ha);
3964 if (alloc_size < L1_CACHE_BYTES)
3965 alloc_size = L1_CACHE_BYTES;
3966 ha = kmalloc(alloc_size, GFP_ATOMIC);
3967 if (!ha)
3968 return -ENOMEM;
3969 memcpy(ha->addr, addr, addr_len);
3970 ha->type = addr_type;
Jiri Pirkoccffad252009-05-22 23:22:17 +00003971 ha->refcount = 1;
3972 ha->synced = false;
Jiri Pirko31278e72009-06-17 01:12:19 +00003973 list_add_tail_rcu(&ha->list, &list->list);
3974 list->count++;
Jiri Pirkof001fde2009-05-05 02:48:28 +00003975 return 0;
3976}
3977
3978static void ha_rcu_free(struct rcu_head *head)
3979{
3980 struct netdev_hw_addr *ha;
3981
3982 ha = container_of(head, struct netdev_hw_addr, rcu_head);
3983 kfree(ha);
3984}
3985
Jiri Pirko31278e72009-06-17 01:12:19 +00003986static int __hw_addr_del(struct netdev_hw_addr_list *list, unsigned char *addr,
3987 int addr_len, unsigned char addr_type)
Jiri Pirkof001fde2009-05-05 02:48:28 +00003988{
3989 struct netdev_hw_addr *ha;
Jiri Pirkof001fde2009-05-05 02:48:28 +00003990
Jiri Pirko31278e72009-06-17 01:12:19 +00003991 list_for_each_entry(ha, &list->list, list) {
Jiri Pirkoccffad252009-05-22 23:22:17 +00003992 if (!memcmp(ha->addr, addr, addr_len) &&
Jiri Pirkof001fde2009-05-05 02:48:28 +00003993 (ha->type == addr_type || !addr_type)) {
Jiri Pirkoccffad252009-05-22 23:22:17 +00003994 if (--ha->refcount)
3995 return 0;
Jiri Pirkof001fde2009-05-05 02:48:28 +00003996 list_del_rcu(&ha->list);
3997 call_rcu(&ha->rcu_head, ha_rcu_free);
Jiri Pirko31278e72009-06-17 01:12:19 +00003998 list->count--;
Jiri Pirkof001fde2009-05-05 02:48:28 +00003999 return 0;
4000 }
4001 }
4002 return -ENOENT;
4003}
4004
Jiri Pirko31278e72009-06-17 01:12:19 +00004005static int __hw_addr_add_multiple(struct netdev_hw_addr_list *to_list,
4006 struct netdev_hw_addr_list *from_list,
4007 int addr_len,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004008 unsigned char addr_type)
Jiri Pirkof001fde2009-05-05 02:48:28 +00004009{
4010 int err;
4011 struct netdev_hw_addr *ha, *ha2;
4012 unsigned char type;
4013
Jiri Pirko31278e72009-06-17 01:12:19 +00004014 list_for_each_entry(ha, &from_list->list, list) {
Jiri Pirkof001fde2009-05-05 02:48:28 +00004015 type = addr_type ? addr_type : ha->type;
Jiri Pirko31278e72009-06-17 01:12:19 +00004016 err = __hw_addr_add(to_list, ha->addr, addr_len, type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004017 if (err)
4018 goto unroll;
4019 }
4020 return 0;
4021
4022unroll:
Jiri Pirko31278e72009-06-17 01:12:19 +00004023 list_for_each_entry(ha2, &from_list->list, list) {
Jiri Pirkof001fde2009-05-05 02:48:28 +00004024 if (ha2 == ha)
4025 break;
4026 type = addr_type ? addr_type : ha2->type;
Jiri Pirko31278e72009-06-17 01:12:19 +00004027 __hw_addr_del(to_list, ha2->addr, addr_len, type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004028 }
4029 return err;
4030}
4031
Jiri Pirko31278e72009-06-17 01:12:19 +00004032static void __hw_addr_del_multiple(struct netdev_hw_addr_list *to_list,
4033 struct netdev_hw_addr_list *from_list,
4034 int addr_len,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004035 unsigned char addr_type)
Jiri Pirkof001fde2009-05-05 02:48:28 +00004036{
4037 struct netdev_hw_addr *ha;
4038 unsigned char type;
4039
Jiri Pirko31278e72009-06-17 01:12:19 +00004040 list_for_each_entry(ha, &from_list->list, list) {
Jiri Pirkof001fde2009-05-05 02:48:28 +00004041 type = addr_type ? addr_type : ha->type;
Jiri Pirko31278e72009-06-17 01:12:19 +00004042 __hw_addr_del(to_list, ha->addr, addr_len, addr_type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004043 }
4044}
4045
Jiri Pirko31278e72009-06-17 01:12:19 +00004046static int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
4047 struct netdev_hw_addr_list *from_list,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004048 int addr_len)
4049{
4050 int err = 0;
4051 struct netdev_hw_addr *ha, *tmp;
4052
Jiri Pirko31278e72009-06-17 01:12:19 +00004053 list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
Jiri Pirkoccffad252009-05-22 23:22:17 +00004054 if (!ha->synced) {
Jiri Pirko31278e72009-06-17 01:12:19 +00004055 err = __hw_addr_add(to_list, ha->addr,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004056 addr_len, ha->type);
4057 if (err)
4058 break;
4059 ha->synced = true;
4060 ha->refcount++;
4061 } else if (ha->refcount == 1) {
Jiri Pirko31278e72009-06-17 01:12:19 +00004062 __hw_addr_del(to_list, ha->addr, addr_len, ha->type);
4063 __hw_addr_del(from_list, ha->addr, addr_len, ha->type);
Jiri Pirkoccffad252009-05-22 23:22:17 +00004064 }
4065 }
4066 return err;
4067}
4068
Jiri Pirko31278e72009-06-17 01:12:19 +00004069static void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
4070 struct netdev_hw_addr_list *from_list,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004071 int addr_len)
4072{
4073 struct netdev_hw_addr *ha, *tmp;
4074
Jiri Pirko31278e72009-06-17 01:12:19 +00004075 list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
Jiri Pirkoccffad252009-05-22 23:22:17 +00004076 if (ha->synced) {
Jiri Pirko31278e72009-06-17 01:12:19 +00004077 __hw_addr_del(to_list, ha->addr,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004078 addr_len, ha->type);
4079 ha->synced = false;
Jiri Pirko31278e72009-06-17 01:12:19 +00004080 __hw_addr_del(from_list, ha->addr,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004081 addr_len, ha->type);
4082 }
4083 }
4084}
4085
Jiri Pirko31278e72009-06-17 01:12:19 +00004086static void __hw_addr_flush(struct netdev_hw_addr_list *list)
Jiri Pirkof001fde2009-05-05 02:48:28 +00004087{
4088 struct netdev_hw_addr *ha, *tmp;
4089
Jiri Pirko31278e72009-06-17 01:12:19 +00004090 list_for_each_entry_safe(ha, tmp, &list->list, list) {
Jiri Pirkof001fde2009-05-05 02:48:28 +00004091 list_del_rcu(&ha->list);
4092 call_rcu(&ha->rcu_head, ha_rcu_free);
4093 }
Jiri Pirko31278e72009-06-17 01:12:19 +00004094 list->count = 0;
4095}
4096
4097static void __hw_addr_init(struct netdev_hw_addr_list *list)
4098{
4099 INIT_LIST_HEAD(&list->list);
4100 list->count = 0;
Jiri Pirkof001fde2009-05-05 02:48:28 +00004101}
4102
4103/* Device addresses handling functions */
4104
4105static void dev_addr_flush(struct net_device *dev)
4106{
4107 /* rtnl_mutex must be held here */
4108
Jiri Pirko31278e72009-06-17 01:12:19 +00004109 __hw_addr_flush(&dev->dev_addrs);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004110 dev->dev_addr = NULL;
4111}
4112
4113static int dev_addr_init(struct net_device *dev)
4114{
4115 unsigned char addr[MAX_ADDR_LEN];
4116 struct netdev_hw_addr *ha;
4117 int err;
4118
4119 /* rtnl_mutex must be held here */
4120
Jiri Pirko31278e72009-06-17 01:12:19 +00004121 __hw_addr_init(&dev->dev_addrs);
Eric Dumazet0c279222009-06-08 03:49:24 +00004122 memset(addr, 0, sizeof(addr));
Jiri Pirko31278e72009-06-17 01:12:19 +00004123 err = __hw_addr_add(&dev->dev_addrs, addr, sizeof(addr),
Jiri Pirkof001fde2009-05-05 02:48:28 +00004124 NETDEV_HW_ADDR_T_LAN);
4125 if (!err) {
4126 /*
4127 * Get the first (previously created) address from the list
4128 * and set dev_addr pointer to this location.
4129 */
Jiri Pirko31278e72009-06-17 01:12:19 +00004130 ha = list_first_entry(&dev->dev_addrs.list,
Jiri Pirkof001fde2009-05-05 02:48:28 +00004131 struct netdev_hw_addr, list);
4132 dev->dev_addr = ha->addr;
4133 }
4134 return err;
4135}
4136
4137/**
4138 * dev_addr_add - Add a device address
4139 * @dev: device
4140 * @addr: address to add
4141 * @addr_type: address type
4142 *
4143 * Add a device address to the device or increase the reference count if
4144 * it already exists.
4145 *
4146 * The caller must hold the rtnl_mutex.
4147 */
4148int dev_addr_add(struct net_device *dev, unsigned char *addr,
4149 unsigned char addr_type)
4150{
4151 int err;
4152
4153 ASSERT_RTNL();
4154
Jiri Pirko31278e72009-06-17 01:12:19 +00004155 err = __hw_addr_add(&dev->dev_addrs, addr, dev->addr_len, addr_type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004156 if (!err)
4157 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4158 return err;
4159}
4160EXPORT_SYMBOL(dev_addr_add);
4161
4162/**
4163 * dev_addr_del - Release a device address.
4164 * @dev: device
4165 * @addr: address to delete
4166 * @addr_type: address type
4167 *
4168 * Release reference to a device address and remove it from the device
4169 * if the reference count drops to zero.
4170 *
4171 * The caller must hold the rtnl_mutex.
4172 */
4173int dev_addr_del(struct net_device *dev, unsigned char *addr,
4174 unsigned char addr_type)
4175{
4176 int err;
Jiri Pirkoccffad252009-05-22 23:22:17 +00004177 struct netdev_hw_addr *ha;
Jiri Pirkof001fde2009-05-05 02:48:28 +00004178
4179 ASSERT_RTNL();
4180
Jiri Pirkoccffad252009-05-22 23:22:17 +00004181 /*
4182 * We can not remove the first address from the list because
4183 * dev->dev_addr points to that.
4184 */
Jiri Pirko31278e72009-06-17 01:12:19 +00004185 ha = list_first_entry(&dev->dev_addrs.list,
4186 struct netdev_hw_addr, list);
Jiri Pirkoccffad252009-05-22 23:22:17 +00004187 if (ha->addr == dev->dev_addr && ha->refcount == 1)
4188 return -ENOENT;
4189
Jiri Pirko31278e72009-06-17 01:12:19 +00004190 err = __hw_addr_del(&dev->dev_addrs, addr, dev->addr_len,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004191 addr_type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004192 if (!err)
4193 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4194 return err;
4195}
4196EXPORT_SYMBOL(dev_addr_del);
4197
4198/**
4199 * dev_addr_add_multiple - Add device addresses from another device
4200 * @to_dev: device to which addresses will be added
4201 * @from_dev: device from which addresses will be added
4202 * @addr_type: address type - 0 means type will be used from from_dev
4203 *
4204 * Add device addresses of the one device to another.
4205 **
4206 * The caller must hold the rtnl_mutex.
4207 */
4208int dev_addr_add_multiple(struct net_device *to_dev,
4209 struct net_device *from_dev,
4210 unsigned char addr_type)
4211{
4212 int err;
4213
4214 ASSERT_RTNL();
4215
4216 if (from_dev->addr_len != to_dev->addr_len)
4217 return -EINVAL;
Jiri Pirko31278e72009-06-17 01:12:19 +00004218 err = __hw_addr_add_multiple(&to_dev->dev_addrs, &from_dev->dev_addrs,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004219 to_dev->addr_len, addr_type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004220 if (!err)
4221 call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev);
4222 return err;
4223}
4224EXPORT_SYMBOL(dev_addr_add_multiple);
4225
4226/**
4227 * dev_addr_del_multiple - Delete device addresses by another device
4228 * @to_dev: device where the addresses will be deleted
4229 * @from_dev: device by which addresses the addresses will be deleted
4230 * @addr_type: address type - 0 means type will used from from_dev
4231 *
4232 * Deletes addresses in to device by the list of addresses in from device.
4233 *
4234 * The caller must hold the rtnl_mutex.
4235 */
4236int dev_addr_del_multiple(struct net_device *to_dev,
4237 struct net_device *from_dev,
4238 unsigned char addr_type)
4239{
4240 ASSERT_RTNL();
4241
4242 if (from_dev->addr_len != to_dev->addr_len)
4243 return -EINVAL;
Jiri Pirko31278e72009-06-17 01:12:19 +00004244 __hw_addr_del_multiple(&to_dev->dev_addrs, &from_dev->dev_addrs,
Jiri Pirkoccffad252009-05-22 23:22:17 +00004245 to_dev->addr_len, addr_type);
Jiri Pirkof001fde2009-05-05 02:48:28 +00004246 call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev);
4247 return 0;
4248}
4249EXPORT_SYMBOL(dev_addr_del_multiple);
4250
Jiri Pirko31278e72009-06-17 01:12:19 +00004251/* multicast addresses handling functions */
Jiri Pirkof001fde2009-05-05 02:48:28 +00004252
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07004253int __dev_addr_delete(struct dev_addr_list **list, int *count,
4254 void *addr, int alen, int glbl)
Patrick McHardybf742482007-06-27 01:26:19 -07004255{
4256 struct dev_addr_list *da;
4257
4258 for (; (da = *list) != NULL; list = &da->next) {
4259 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 &&
4260 alen == da->da_addrlen) {
4261 if (glbl) {
4262 int old_glbl = da->da_gusers;
4263 da->da_gusers = 0;
4264 if (old_glbl == 0)
4265 break;
4266 }
4267 if (--da->da_users)
4268 return 0;
4269
4270 *list = da->next;
4271 kfree(da);
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07004272 (*count)--;
Patrick McHardybf742482007-06-27 01:26:19 -07004273 return 0;
4274 }
4275 }
4276 return -ENOENT;
4277}
4278
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07004279int __dev_addr_add(struct dev_addr_list **list, int *count,
4280 void *addr, int alen, int glbl)
Patrick McHardybf742482007-06-27 01:26:19 -07004281{
4282 struct dev_addr_list *da;
4283
4284 for (da = *list; da != NULL; da = da->next) {
4285 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 &&
4286 da->da_addrlen == alen) {
4287 if (glbl) {
4288 int old_glbl = da->da_gusers;
4289 da->da_gusers = 1;
4290 if (old_glbl)
4291 return 0;
4292 }
4293 da->da_users++;
4294 return 0;
4295 }
4296 }
4297
Jorge Boncompte [DTI2]12aa3432008-02-19 14:17:04 -08004298 da = kzalloc(sizeof(*da), GFP_ATOMIC);
Patrick McHardybf742482007-06-27 01:26:19 -07004299 if (da == NULL)
4300 return -ENOMEM;
4301 memcpy(da->da_addr, addr, alen);
4302 da->da_addrlen = alen;
4303 da->da_users = 1;
4304 da->da_gusers = glbl ? 1 : 0;
4305 da->next = *list;
4306 *list = da;
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07004307 (*count)++;
Patrick McHardybf742482007-06-27 01:26:19 -07004308 return 0;
4309}
4310
Patrick McHardy4417da62007-06-27 01:28:10 -07004311/**
4312 * dev_unicast_delete - Release secondary unicast address.
4313 * @dev: device
Randy Dunlap0ed72ec2007-07-26 00:03:29 -07004314 * @addr: address to delete
Patrick McHardy4417da62007-06-27 01:28:10 -07004315 *
4316 * Release reference to a secondary unicast address and remove it
Randy Dunlap0ed72ec2007-07-26 00:03:29 -07004317 * from the device if the reference count drops to zero.
Patrick McHardy4417da62007-06-27 01:28:10 -07004318 *
4319 * The caller must hold the rtnl_mutex.
4320 */
Jiri Pirkoccffad252009-05-22 23:22:17 +00004321int dev_unicast_delete(struct net_device *dev, void *addr)
Patrick McHardy4417da62007-06-27 01:28:10 -07004322{
4323 int err;
4324
4325 ASSERT_RTNL();
4326
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004327 netif_addr_lock_bh(dev);
Jiri Pirko31278e72009-06-17 01:12:19 +00004328 err = __hw_addr_del(&dev->uc, addr, dev->addr_len,
4329 NETDEV_HW_ADDR_T_UNICAST);
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07004330 if (!err)
Patrick McHardy4417da62007-06-27 01:28:10 -07004331 __dev_set_rx_mode(dev);
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004332 netif_addr_unlock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004333 return err;
4334}
4335EXPORT_SYMBOL(dev_unicast_delete);
4336
4337/**
4338 * dev_unicast_add - add a secondary unicast address
4339 * @dev: device
Wang Chen5dbaec52008-06-27 19:35:16 -07004340 * @addr: address to add
Patrick McHardy4417da62007-06-27 01:28:10 -07004341 *
4342 * Add a secondary unicast address to the device or increase
4343 * the reference count if it already exists.
4344 *
4345 * The caller must hold the rtnl_mutex.
4346 */
Jiri Pirkoccffad252009-05-22 23:22:17 +00004347int dev_unicast_add(struct net_device *dev, void *addr)
Patrick McHardy4417da62007-06-27 01:28:10 -07004348{
4349 int err;
4350
4351 ASSERT_RTNL();
4352
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004353 netif_addr_lock_bh(dev);
Jiri Pirko31278e72009-06-17 01:12:19 +00004354 err = __hw_addr_add(&dev->uc, addr, dev->addr_len,
4355 NETDEV_HW_ADDR_T_UNICAST);
Patrick McHardy61cbc2f2007-06-30 13:35:52 -07004356 if (!err)
Patrick McHardy4417da62007-06-27 01:28:10 -07004357 __dev_set_rx_mode(dev);
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004358 netif_addr_unlock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07004359 return err;
4360}
4361EXPORT_SYMBOL(dev_unicast_add);
4362
Chris Leeche83a2ea2008-01-31 16:53:23 -08004363int __dev_addr_sync(struct dev_addr_list **to, int *to_count,
4364 struct dev_addr_list **from, int *from_count)
4365{
4366 struct dev_addr_list *da, *next;
4367 int err = 0;
4368
4369 da = *from;
4370 while (da != NULL) {
4371 next = da->next;
4372 if (!da->da_synced) {
4373 err = __dev_addr_add(to, to_count,
4374 da->da_addr, da->da_addrlen, 0);
4375 if (err < 0)
4376 break;
4377 da->da_synced = 1;
4378 da->da_users++;
4379 } else if (da->da_users == 1) {
4380 __dev_addr_delete(to, to_count,
4381 da->da_addr, da->da_addrlen, 0);
4382 __dev_addr_delete(from, from_count,
4383 da->da_addr, da->da_addrlen, 0);
4384 }
4385 da = next;
4386 }
4387 return err;
4388}
Johannes Bergc4029082009-06-17 17:43:30 +02004389EXPORT_SYMBOL_GPL(__dev_addr_sync);
Chris Leeche83a2ea2008-01-31 16:53:23 -08004390
4391void __dev_addr_unsync(struct dev_addr_list **to, int *to_count,
4392 struct dev_addr_list **from, int *from_count)
4393{
4394 struct dev_addr_list *da, *next;
4395
4396 da = *from;
4397 while (da != NULL) {
4398 next = da->next;
4399 if (da->da_synced) {
4400 __dev_addr_delete(to, to_count,
4401 da->da_addr, da->da_addrlen, 0);
4402 da->da_synced = 0;
4403 __dev_addr_delete(from, from_count,
4404 da->da_addr, da->da_addrlen, 0);
4405 }
4406 da = next;
4407 }
4408}
Johannes Bergc4029082009-06-17 17:43:30 +02004409EXPORT_SYMBOL_GPL(__dev_addr_unsync);
Chris Leeche83a2ea2008-01-31 16:53:23 -08004410
4411/**
4412 * dev_unicast_sync - Synchronize device's unicast list to another device
4413 * @to: destination device
4414 * @from: source device
4415 *
4416 * Add newly added addresses to the destination device and release
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004417 * addresses that have no users left. The source device must be
4418 * locked by netif_tx_lock_bh.
Chris Leeche83a2ea2008-01-31 16:53:23 -08004419 *
4420 * This function is intended to be called from the dev->set_rx_mode
4421 * function of layered software devices.
4422 */
4423int dev_unicast_sync(struct net_device *to, struct net_device *from)
4424{
4425 int err = 0;
4426
Jiri Pirkoccffad252009-05-22 23:22:17 +00004427 if (to->addr_len != from->addr_len)
4428 return -EINVAL;
4429
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004430 netif_addr_lock_bh(to);
Jiri Pirko31278e72009-06-17 01:12:19 +00004431 err = __hw_addr_sync(&to->uc, &from->uc, to->addr_len);
Chris Leeche83a2ea2008-01-31 16:53:23 -08004432 if (!err)
4433 __dev_set_rx_mode(to);
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004434 netif_addr_unlock_bh(to);
Chris Leeche83a2ea2008-01-31 16:53:23 -08004435 return err;
4436}
4437EXPORT_SYMBOL(dev_unicast_sync);
4438
4439/**
Randy Dunlapbc2cda12008-02-13 15:03:25 -08004440 * dev_unicast_unsync - Remove synchronized addresses from the destination device
Chris Leeche83a2ea2008-01-31 16:53:23 -08004441 * @to: destination device
4442 * @from: source device
4443 *
4444 * Remove all addresses that were added to the destination device by
4445 * dev_unicast_sync(). This function is intended to be called from the
4446 * dev->stop function of layered software devices.
4447 */
4448void dev_unicast_unsync(struct net_device *to, struct net_device *from)
4449{
Jiri Pirkoccffad252009-05-22 23:22:17 +00004450 if (to->addr_len != from->addr_len)
4451 return;
4452
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004453 netif_addr_lock_bh(from);
4454 netif_addr_lock(to);
Jiri Pirko31278e72009-06-17 01:12:19 +00004455 __hw_addr_unsync(&to->uc, &from->uc, to->addr_len);
Chris Leeche83a2ea2008-01-31 16:53:23 -08004456 __dev_set_rx_mode(to);
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004457 netif_addr_unlock(to);
4458 netif_addr_unlock_bh(from);
Chris Leeche83a2ea2008-01-31 16:53:23 -08004459}
4460EXPORT_SYMBOL(dev_unicast_unsync);
4461
Jiri Pirko32a806c2010-03-19 04:00:23 +00004462void dev_unicast_flush(struct net_device *dev)
Jiri Pirkoccffad252009-05-22 23:22:17 +00004463{
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004464 netif_addr_lock_bh(dev);
Jiri Pirko31278e72009-06-17 01:12:19 +00004465 __hw_addr_flush(&dev->uc);
Jiri Pirkoa6ac65d2009-07-30 01:06:12 +00004466 netif_addr_unlock_bh(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00004467}
Jiri Pirko32a806c2010-03-19 04:00:23 +00004468EXPORT_SYMBOL(dev_unicast_flush);
Jiri Pirkoccffad252009-05-22 23:22:17 +00004469
4470static void dev_unicast_init(struct net_device *dev)
4471{
Jiri Pirko31278e72009-06-17 01:12:19 +00004472 __hw_addr_init(&dev->uc);
Jiri Pirkoccffad252009-05-22 23:22:17 +00004473}
4474
4475
Denis Cheng12972622007-07-18 02:12:56 -07004476static void __dev_addr_discard(struct dev_addr_list **list)
4477{
4478 struct dev_addr_list *tmp;
4479
4480 while (*list != NULL) {
4481 tmp = *list;
4482 *list = tmp->next;
4483 if (tmp->da_users > tmp->da_gusers)
4484 printk("__dev_addr_discard: address leakage! "
4485 "da_users=%d\n", tmp->da_users);
4486 kfree(tmp);
4487 }
4488}
4489
Jiri Pirko32a806c2010-03-19 04:00:23 +00004490void dev_addr_discard(struct net_device *dev)
Patrick McHardy4417da62007-06-27 01:28:10 -07004491{
David S. Millerb9e40852008-07-15 00:15:08 -07004492 netif_addr_lock_bh(dev);
Denis Cheng26cc2522007-07-18 02:12:03 -07004493
Denis Cheng456ad752007-07-18 02:10:54 -07004494 __dev_addr_discard(&dev->mc_list);
Jiri Pirko4cd24ea2010-02-08 04:30:35 +00004495 netdev_mc_count(dev) = 0;
Denis Cheng26cc2522007-07-18 02:12:03 -07004496
David S. Millerb9e40852008-07-15 00:15:08 -07004497 netif_addr_unlock_bh(dev);
Denis Cheng456ad752007-07-18 02:10:54 -07004498}
Jiri Pirko32a806c2010-03-19 04:00:23 +00004499EXPORT_SYMBOL(dev_addr_discard);
Denis Cheng456ad752007-07-18 02:10:54 -07004500
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004501/**
4502 * dev_get_flags - get flags reported to userspace
4503 * @dev: device
4504 *
4505 * Get the combination of flag bits exported through APIs to userspace.
4506 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004507unsigned dev_get_flags(const struct net_device *dev)
4508{
4509 unsigned flags;
4510
4511 flags = (dev->flags & ~(IFF_PROMISC |
4512 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08004513 IFF_RUNNING |
4514 IFF_LOWER_UP |
4515 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07004516 (dev->gflags & (IFF_PROMISC |
4517 IFF_ALLMULTI));
4518
Stefan Rompfb00055a2006-03-20 17:09:11 -08004519 if (netif_running(dev)) {
4520 if (netif_oper_up(dev))
4521 flags |= IFF_RUNNING;
4522 if (netif_carrier_ok(dev))
4523 flags |= IFF_LOWER_UP;
4524 if (netif_dormant(dev))
4525 flags |= IFF_DORMANT;
4526 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004527
4528 return flags;
4529}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004530EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004531
Patrick McHardybd380812010-02-26 06:34:53 +00004532int __dev_change_flags(struct net_device *dev, unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004533{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004534 int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00004535 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004536
Patrick McHardy24023452007-07-14 18:51:31 -07004537 ASSERT_RTNL();
4538
Linus Torvalds1da177e2005-04-16 15:20:36 -07004539 /*
4540 * Set the flags on our device.
4541 */
4542
4543 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4544 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4545 IFF_AUTOMEDIA)) |
4546 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4547 IFF_ALLMULTI));
4548
4549 /*
4550 * Load in the correct multicast list now the flags have changed.
4551 */
4552
Patrick McHardyb6c40d62008-10-07 15:26:48 -07004553 if ((old_flags ^ flags) & IFF_MULTICAST)
4554 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07004555
Patrick McHardy4417da62007-06-27 01:28:10 -07004556 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004557
4558 /*
4559 * Have we downed the interface. We handle IFF_UP ourselves
4560 * according to user attempts to set it, rather than blindly
4561 * setting it.
4562 */
4563
4564 ret = 0;
4565 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
Patrick McHardybd380812010-02-26 06:34:53 +00004566 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004567
4568 if (!ret)
Patrick McHardy4417da62007-06-27 01:28:10 -07004569 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004570 }
4571
Linus Torvalds1da177e2005-04-16 15:20:36 -07004572 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004573 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4574
Linus Torvalds1da177e2005-04-16 15:20:36 -07004575 dev->gflags ^= IFF_PROMISC;
4576 dev_set_promiscuity(dev, inc);
4577 }
4578
4579 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4580 is important. Some (broken) drivers set IFF_PROMISC, when
4581 IFF_ALLMULTI is requested not asking us and not reporting.
4582 */
4583 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004584 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4585
Linus Torvalds1da177e2005-04-16 15:20:36 -07004586 dev->gflags ^= IFF_ALLMULTI;
4587 dev_set_allmulti(dev, inc);
4588 }
4589
Patrick McHardybd380812010-02-26 06:34:53 +00004590 return ret;
4591}
4592
4593void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4594{
4595 unsigned int changes = dev->flags ^ old_flags;
4596
4597 if (changes & IFF_UP) {
4598 if (dev->flags & IFF_UP)
4599 call_netdevice_notifiers(NETDEV_UP, dev);
4600 else
4601 call_netdevice_notifiers(NETDEV_DOWN, dev);
4602 }
4603
4604 if (dev->flags & IFF_UP &&
4605 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4606 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4607}
4608
4609/**
4610 * dev_change_flags - change device settings
4611 * @dev: device
4612 * @flags: device state flags
4613 *
4614 * Change settings on device based state flags. The flags are
4615 * in the userspace exported format.
4616 */
4617int dev_change_flags(struct net_device *dev, unsigned flags)
4618{
4619 int ret, changes;
4620 int old_flags = dev->flags;
4621
4622 ret = __dev_change_flags(dev, flags);
4623 if (ret < 0)
4624 return ret;
4625
4626 changes = old_flags ^ dev->flags;
Thomas Graf7c355f52007-06-05 16:03:03 -07004627 if (changes)
4628 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004629
Patrick McHardybd380812010-02-26 06:34:53 +00004630 __dev_notify_flags(dev, old_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004631 return ret;
4632}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004633EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004634
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004635/**
4636 * dev_set_mtu - Change maximum transfer unit
4637 * @dev: device
4638 * @new_mtu: new transfer unit
4639 *
4640 * Change the maximum transfer size of the network device.
4641 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004642int dev_set_mtu(struct net_device *dev, int new_mtu)
4643{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004644 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004645 int err;
4646
4647 if (new_mtu == dev->mtu)
4648 return 0;
4649
4650 /* MTU must be positive. */
4651 if (new_mtu < 0)
4652 return -EINVAL;
4653
4654 if (!netif_device_present(dev))
4655 return -ENODEV;
4656
4657 err = 0;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004658 if (ops->ndo_change_mtu)
4659 err = ops->ndo_change_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004660 else
4661 dev->mtu = new_mtu;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004662
Linus Torvalds1da177e2005-04-16 15:20:36 -07004663 if (!err && dev->flags & IFF_UP)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004664 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004665 return err;
4666}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004667EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004668
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07004669/**
4670 * dev_set_mac_address - Change Media Access Control Address
4671 * @dev: device
4672 * @sa: new address
4673 *
4674 * Change the hardware (MAC) address of the device
4675 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004676int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4677{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004678 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004679 int err;
4680
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004681 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004682 return -EOPNOTSUPP;
4683 if (sa->sa_family != dev->type)
4684 return -EINVAL;
4685 if (!netif_device_present(dev))
4686 return -ENODEV;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08004687 err = ops->ndo_set_mac_address(dev, sa);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004688 if (!err)
Pavel Emelyanov056925a2007-09-16 15:42:43 -07004689 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004690 return err;
4691}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004692EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004693
4694/*
Eric Dumazet3710bec2009-11-01 19:42:09 +00004695 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -07004696 */
Jeff Garzik14e3e072007-10-08 00:06:32 -07004697static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004698{
4699 int err;
Eric Dumazet3710bec2009-11-01 19:42:09 +00004700 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004701
4702 if (!dev)
4703 return -ENODEV;
4704
4705 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004706 case SIOCGIFFLAGS: /* Get interface flags */
4707 ifr->ifr_flags = (short) dev_get_flags(dev);
4708 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004709
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004710 case SIOCGIFMETRIC: /* Get the metric on the interface
4711 (currently unused) */
4712 ifr->ifr_metric = 0;
4713 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004714
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004715 case SIOCGIFMTU: /* Get the MTU of a device */
4716 ifr->ifr_mtu = dev->mtu;
4717 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004718
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004719 case SIOCGIFHWADDR:
4720 if (!dev->addr_len)
4721 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4722 else
4723 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4724 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4725 ifr->ifr_hwaddr.sa_family = dev->type;
4726 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004727
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004728 case SIOCGIFSLAVE:
4729 err = -EINVAL;
4730 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004731
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004732 case SIOCGIFMAP:
4733 ifr->ifr_map.mem_start = dev->mem_start;
4734 ifr->ifr_map.mem_end = dev->mem_end;
4735 ifr->ifr_map.base_addr = dev->base_addr;
4736 ifr->ifr_map.irq = dev->irq;
4737 ifr->ifr_map.dma = dev->dma;
4738 ifr->ifr_map.port = dev->if_port;
4739 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004740
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004741 case SIOCGIFINDEX:
4742 ifr->ifr_ifindex = dev->ifindex;
4743 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004744
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004745 case SIOCGIFTXQLEN:
4746 ifr->ifr_qlen = dev->tx_queue_len;
4747 return 0;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004748
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004749 default:
4750 /* dev_ioctl() should ensure this case
4751 * is never reached
4752 */
4753 WARN_ON(1);
4754 err = -EINVAL;
4755 break;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004756
4757 }
4758 return err;
4759}
4760
4761/*
4762 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4763 */
4764static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4765{
4766 int err;
4767 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004768 const struct net_device_ops *ops;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004769
4770 if (!dev)
4771 return -ENODEV;
4772
Jarek Poplawski5f2f6da2008-12-22 19:35:28 -08004773 ops = dev->netdev_ops;
4774
Jeff Garzik14e3e072007-10-08 00:06:32 -07004775 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004776 case SIOCSIFFLAGS: /* Set interface flags */
4777 return dev_change_flags(dev, ifr->ifr_flags);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004778
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004779 case SIOCSIFMETRIC: /* Set the metric on the interface
4780 (currently unused) */
4781 return -EOPNOTSUPP;
Jeff Garzik14e3e072007-10-08 00:06:32 -07004782
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004783 case SIOCSIFMTU: /* Set the MTU of a device */
4784 return dev_set_mtu(dev, ifr->ifr_mtu);
Jeff Garzik14e3e072007-10-08 00:06:32 -07004785
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004786 case SIOCSIFHWADDR:
4787 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004788
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004789 case SIOCSIFHWBROADCAST:
4790 if (ifr->ifr_hwaddr.sa_family != dev->type)
4791 return -EINVAL;
4792 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4793 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4794 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4795 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004796
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004797 case SIOCSIFMAP:
4798 if (ops->ndo_set_config) {
4799 if (!netif_device_present(dev))
4800 return -ENODEV;
4801 return ops->ndo_set_config(dev, &ifr->ifr_map);
4802 }
4803 return -EOPNOTSUPP;
4804
4805 case SIOCADDMULTI:
4806 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4807 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4808 return -EINVAL;
4809 if (!netif_device_present(dev))
4810 return -ENODEV;
4811 return dev_mc_add(dev, ifr->ifr_hwaddr.sa_data,
4812 dev->addr_len, 1);
4813
4814 case SIOCDELMULTI:
4815 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4816 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4817 return -EINVAL;
4818 if (!netif_device_present(dev))
4819 return -ENODEV;
4820 return dev_mc_delete(dev, ifr->ifr_hwaddr.sa_data,
4821 dev->addr_len, 1);
4822
4823 case SIOCSIFTXQLEN:
4824 if (ifr->ifr_qlen < 0)
4825 return -EINVAL;
4826 dev->tx_queue_len = ifr->ifr_qlen;
4827 return 0;
4828
4829 case SIOCSIFNAME:
4830 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4831 return dev_change_name(dev, ifr->ifr_newname);
4832
4833 /*
4834 * Unknown or private ioctl
4835 */
4836 default:
4837 if ((cmd >= SIOCDEVPRIVATE &&
4838 cmd <= SIOCDEVPRIVATE + 15) ||
4839 cmd == SIOCBONDENSLAVE ||
4840 cmd == SIOCBONDRELEASE ||
4841 cmd == SIOCBONDSETHWADDR ||
4842 cmd == SIOCBONDSLAVEINFOQUERY ||
4843 cmd == SIOCBONDINFOQUERY ||
4844 cmd == SIOCBONDCHANGEACTIVE ||
4845 cmd == SIOCGMIIPHY ||
4846 cmd == SIOCGMIIREG ||
4847 cmd == SIOCSMIIREG ||
4848 cmd == SIOCBRADDIF ||
4849 cmd == SIOCBRDELIF ||
4850 cmd == SIOCSHWTSTAMP ||
4851 cmd == SIOCWANDEV) {
4852 err = -EOPNOTSUPP;
4853 if (ops->ndo_do_ioctl) {
4854 if (netif_device_present(dev))
4855 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4856 else
4857 err = -ENODEV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004858 }
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004859 } else
4860 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004861
4862 }
4863 return err;
4864}
4865
4866/*
4867 * This function handles all "interface"-type I/O control requests. The actual
4868 * 'doing' part of this is dev_ifsioc above.
4869 */
4870
4871/**
4872 * dev_ioctl - network device ioctl
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07004873 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07004874 * @cmd: command to issue
4875 * @arg: pointer to a struct ifreq in user space
4876 *
4877 * Issue ioctl functions to devices. This is normally called by the
4878 * user space syscall interfaces but can sometimes be useful for
4879 * other purposes. The return value is the return from the syscall if
4880 * positive or a negative errno code on error.
4881 */
4882
Eric W. Biederman881d9662007-09-17 11:56:21 -07004883int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004884{
4885 struct ifreq ifr;
4886 int ret;
4887 char *colon;
4888
4889 /* One special case: SIOCGIFCONF takes ifconf argument
4890 and requires shared lock, because it sleeps writing
4891 to user space.
4892 */
4893
4894 if (cmd == SIOCGIFCONF) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004895 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07004896 ret = dev_ifconf(net, (char __user *) arg);
Stephen Hemminger6756ae42006-03-20 22:23:58 -08004897 rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004898 return ret;
4899 }
4900 if (cmd == SIOCGIFNAME)
Eric W. Biederman881d9662007-09-17 11:56:21 -07004901 return dev_ifname(net, (struct ifreq __user *)arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004902
4903 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4904 return -EFAULT;
4905
4906 ifr.ifr_name[IFNAMSIZ-1] = 0;
4907
4908 colon = strchr(ifr.ifr_name, ':');
4909 if (colon)
4910 *colon = 0;
4911
4912 /*
4913 * See which interface the caller is talking about.
4914 */
4915
4916 switch (cmd) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004917 /*
4918 * These ioctl calls:
4919 * - can be done by all.
4920 * - atomic and do not require locking.
4921 * - return a value
4922 */
4923 case SIOCGIFFLAGS:
4924 case SIOCGIFMETRIC:
4925 case SIOCGIFMTU:
4926 case SIOCGIFHWADDR:
4927 case SIOCGIFSLAVE:
4928 case SIOCGIFMAP:
4929 case SIOCGIFINDEX:
4930 case SIOCGIFTXQLEN:
4931 dev_load(net, ifr.ifr_name);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004932 rcu_read_lock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004933 ret = dev_ifsioc_locked(net, &ifr, cmd);
Eric Dumazet3710bec2009-11-01 19:42:09 +00004934 rcu_read_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004935 if (!ret) {
4936 if (colon)
4937 *colon = ':';
4938 if (copy_to_user(arg, &ifr,
4939 sizeof(struct ifreq)))
4940 ret = -EFAULT;
4941 }
4942 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004943
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004944 case SIOCETHTOOL:
4945 dev_load(net, ifr.ifr_name);
4946 rtnl_lock();
4947 ret = dev_ethtool(net, &ifr);
4948 rtnl_unlock();
4949 if (!ret) {
4950 if (colon)
4951 *colon = ':';
4952 if (copy_to_user(arg, &ifr,
4953 sizeof(struct ifreq)))
4954 ret = -EFAULT;
4955 }
4956 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004957
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004958 /*
4959 * These ioctl calls:
4960 * - require superuser power.
4961 * - require strict serialization.
4962 * - return a value
4963 */
4964 case SIOCGMIIPHY:
4965 case SIOCGMIIREG:
4966 case SIOCSIFNAME:
4967 if (!capable(CAP_NET_ADMIN))
4968 return -EPERM;
4969 dev_load(net, ifr.ifr_name);
4970 rtnl_lock();
4971 ret = dev_ifsioc(net, &ifr, cmd);
4972 rtnl_unlock();
4973 if (!ret) {
4974 if (colon)
4975 *colon = ':';
4976 if (copy_to_user(arg, &ifr,
4977 sizeof(struct ifreq)))
4978 ret = -EFAULT;
4979 }
4980 return ret;
4981
4982 /*
4983 * These ioctl calls:
4984 * - require superuser power.
4985 * - require strict serialization.
4986 * - do not return a value
4987 */
4988 case SIOCSIFFLAGS:
4989 case SIOCSIFMETRIC:
4990 case SIOCSIFMTU:
4991 case SIOCSIFMAP:
4992 case SIOCSIFHWADDR:
4993 case SIOCSIFSLAVE:
4994 case SIOCADDMULTI:
4995 case SIOCDELMULTI:
4996 case SIOCSIFHWBROADCAST:
4997 case SIOCSIFTXQLEN:
4998 case SIOCSMIIREG:
4999 case SIOCBONDENSLAVE:
5000 case SIOCBONDRELEASE:
5001 case SIOCBONDSETHWADDR:
5002 case SIOCBONDCHANGEACTIVE:
5003 case SIOCBRADDIF:
5004 case SIOCBRDELIF:
5005 case SIOCSHWTSTAMP:
5006 if (!capable(CAP_NET_ADMIN))
5007 return -EPERM;
5008 /* fall through */
5009 case SIOCBONDSLAVEINFOQUERY:
5010 case SIOCBONDINFOQUERY:
5011 dev_load(net, ifr.ifr_name);
5012 rtnl_lock();
5013 ret = dev_ifsioc(net, &ifr, cmd);
5014 rtnl_unlock();
5015 return ret;
5016
5017 case SIOCGIFMEM:
5018 /* Get the per device memory space. We can add this but
5019 * currently do not support it */
5020 case SIOCSIFMEM:
5021 /* Set the per device memory buffer space.
5022 * Not applicable in our case */
5023 case SIOCSIFLINK:
5024 return -EINVAL;
5025
5026 /*
5027 * Unknown or private ioctl.
5028 */
5029 default:
5030 if (cmd == SIOCWANDEV ||
5031 (cmd >= SIOCDEVPRIVATE &&
5032 cmd <= SIOCDEVPRIVATE + 15)) {
Eric W. Biederman881d9662007-09-17 11:56:21 -07005033 dev_load(net, ifr.ifr_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005034 rtnl_lock();
Eric W. Biederman881d9662007-09-17 11:56:21 -07005035 ret = dev_ifsioc(net, &ifr, cmd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005036 rtnl_unlock();
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005037 if (!ret && copy_to_user(arg, &ifr,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005038 sizeof(struct ifreq)))
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005039 ret = -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005040 return ret;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005041 }
5042 /* Take care of Wireless Extensions */
5043 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5044 return wext_handle_ioctl(net, &ifr, cmd, arg);
5045 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005046 }
5047}
5048
5049
5050/**
5051 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005052 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07005053 *
5054 * Returns a suitable unique value for a new device interface
5055 * number. The caller must hold the rtnl semaphore or the
5056 * dev_base_lock to be sure it remains unique.
5057 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07005058static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005059{
5060 static int ifindex;
5061 for (;;) {
5062 if (++ifindex <= 0)
5063 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07005064 if (!__dev_get_by_index(net, ifindex))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005065 return ifindex;
5066 }
5067}
5068
Linus Torvalds1da177e2005-04-16 15:20:36 -07005069/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08005070static LIST_HEAD(net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005071
Stephen Hemminger6f05f622007-03-08 20:46:03 -08005072static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005073{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005074 list_add_tail(&dev->todo_list, &net_todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005075}
5076
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005077static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005078{
Krishna Kumare93737b2009-12-08 22:26:02 +00005079 struct net_device *dev, *tmp;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005080
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005081 BUG_ON(dev_boot_phase);
5082 ASSERT_RTNL();
5083
Krishna Kumare93737b2009-12-08 22:26:02 +00005084 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005085 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00005086 * for initialization unwind. Remove those
5087 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005088 */
5089 if (dev->reg_state == NETREG_UNINITIALIZED) {
5090 pr_debug("unregister_netdevice: device %s/%p never "
5091 "was registered\n", dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005092
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005093 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00005094 list_del(&dev->unreg_list);
5095 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005096 }
5097
5098 BUG_ON(dev->reg_state != NETREG_REGISTERED);
5099
5100 /* If device is running, close it first. */
5101 dev_close(dev);
5102
5103 /* And unlink it from device chain. */
5104 unlist_netdevice(dev);
5105
5106 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005107 }
5108
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005109 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005110
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005111 list_for_each_entry(dev, head, unreg_list) {
5112 /* Shutdown queueing discipline. */
5113 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005114
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005115
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005116 /* Notify protocols, that we are about to destroy
5117 this device. They should clean all the things.
5118 */
5119 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5120
Patrick McHardya2835762010-02-26 06:34:51 +00005121 if (!dev->rtnl_link_ops ||
5122 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5123 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5124
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005125 /*
5126 * Flush the unicast and multicast chains
5127 */
5128 dev_unicast_flush(dev);
5129 dev_addr_discard(dev);
5130
5131 if (dev->netdev_ops->ndo_uninit)
5132 dev->netdev_ops->ndo_uninit(dev);
5133
5134 /* Notifier chain MUST detach us from master device. */
5135 WARN_ON(dev->master);
5136
5137 /* Remove entries from kobject tree */
5138 netdev_unregister_kobject(dev);
5139 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005140
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005141 /* Process any work delayed until the end of the batch */
stephen hemmingere5e26d72010-02-24 14:01:38 +00005142 dev = list_first_entry(head, struct net_device, unreg_list);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005143 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5144
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005145 synchronize_net();
5146
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005147 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005148 dev_put(dev);
5149}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005150
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005151static void rollback_registered(struct net_device *dev)
5152{
5153 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005154
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005155 list_add(&dev->unreg_list, &single);
5156 rollback_registered_many(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005157}
5158
David S. Millere8a04642008-07-17 00:34:19 -07005159static void __netdev_init_queue_locks_one(struct net_device *dev,
5160 struct netdev_queue *dev_queue,
5161 void *_unused)
David S. Millerc773e842008-07-08 23:13:53 -07005162{
5163 spin_lock_init(&dev_queue->_xmit_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005164 netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);
David S. Millerc773e842008-07-08 23:13:53 -07005165 dev_queue->xmit_lock_owner = -1;
5166}
5167
5168static void netdev_init_queue_locks(struct net_device *dev)
5169{
David S. Millere8a04642008-07-17 00:34:19 -07005170 netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);
5171 __netdev_init_queue_locks_one(dev, &dev->rx_queue, NULL);
David S. Millerc773e842008-07-08 23:13:53 -07005172}
5173
Herbert Xub63365a2008-10-23 01:11:29 -07005174unsigned long netdev_fix_features(unsigned long features, const char *name)
5175{
5176 /* Fix illegal SG+CSUM combinations. */
5177 if ((features & NETIF_F_SG) &&
5178 !(features & NETIF_F_ALL_CSUM)) {
5179 if (name)
5180 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
5181 "checksum feature.\n", name);
5182 features &= ~NETIF_F_SG;
5183 }
5184
5185 /* TSO requires that SG is present as well. */
5186 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
5187 if (name)
5188 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
5189 "SG feature.\n", name);
5190 features &= ~NETIF_F_TSO;
5191 }
5192
5193 if (features & NETIF_F_UFO) {
5194 if (!(features & NETIF_F_GEN_CSUM)) {
5195 if (name)
5196 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
5197 "since no NETIF_F_HW_CSUM feature.\n",
5198 name);
5199 features &= ~NETIF_F_UFO;
5200 }
5201
5202 if (!(features & NETIF_F_SG)) {
5203 if (name)
5204 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
5205 "since no NETIF_F_SG feature.\n", name);
5206 features &= ~NETIF_F_UFO;
5207 }
5208 }
5209
5210 return features;
5211}
5212EXPORT_SYMBOL(netdev_fix_features);
5213
Linus Torvalds1da177e2005-04-16 15:20:36 -07005214/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08005215 * netif_stacked_transfer_operstate - transfer operstate
5216 * @rootdev: the root or lower level device to transfer state from
5217 * @dev: the device to transfer operstate to
5218 *
5219 * Transfer operational state from root to device. This is normally
5220 * called when a stacking relationship exists between the root
5221 * device and the device(a leaf device).
5222 */
5223void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5224 struct net_device *dev)
5225{
5226 if (rootdev->operstate == IF_OPER_DORMANT)
5227 netif_dormant_on(dev);
5228 else
5229 netif_dormant_off(dev);
5230
5231 if (netif_carrier_ok(rootdev)) {
5232 if (!netif_carrier_ok(dev))
5233 netif_carrier_on(dev);
5234 } else {
5235 if (netif_carrier_ok(dev))
5236 netif_carrier_off(dev);
5237 }
5238}
5239EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5240
5241/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005242 * register_netdevice - register a network device
5243 * @dev: device to register
5244 *
5245 * Take a completed network device structure and add it to the kernel
5246 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5247 * chain. 0 is returned on success. A negative errno code is returned
5248 * on a failure to set up the device, or if the name is a duplicate.
5249 *
5250 * Callers must hold the rtnl semaphore. You may want
5251 * register_netdev() instead of this.
5252 *
5253 * BUGS:
5254 * The locking appears insufficient to guarantee two parallel registers
5255 * will not get the same name.
5256 */
5257
5258int register_netdevice(struct net_device *dev)
5259{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005260 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005261 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005262
5263 BUG_ON(dev_boot_phase);
5264 ASSERT_RTNL();
5265
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005266 might_sleep();
5267
Linus Torvalds1da177e2005-04-16 15:20:36 -07005268 /* When net_device's are persistent, this will be fatal. */
5269 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005270 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005271
David S. Millerf1f28aa2008-07-15 00:08:33 -07005272 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07005273 netdev_set_addr_lockdep_class(dev);
David S. Millerc773e842008-07-08 23:13:53 -07005274 netdev_init_queue_locks(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005275
Linus Torvalds1da177e2005-04-16 15:20:36 -07005276 dev->iflink = -1;
5277
Tom Herbert0a9627f2010-03-16 08:03:29 +00005278 if (!dev->num_rx_queues) {
5279 /*
5280 * Allocate a single RX queue if driver never called
5281 * alloc_netdev_mq
5282 */
5283
5284 dev->_rx = kzalloc(sizeof(struct netdev_rx_queue), GFP_KERNEL);
5285 if (!dev->_rx) {
5286 ret = -ENOMEM;
5287 goto out;
5288 }
5289
5290 dev->_rx->first = dev->_rx;
5291 atomic_set(&dev->_rx->count, 1);
5292 dev->num_rx_queues = 1;
5293 }
5294
Linus Torvalds1da177e2005-04-16 15:20:36 -07005295 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005296 if (dev->netdev_ops->ndo_init) {
5297 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005298 if (ret) {
5299 if (ret > 0)
5300 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08005301 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005302 }
5303 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005304
Octavian Purdilad9031022009-11-18 02:36:59 +00005305 ret = dev_get_valid_name(net, dev->name, dev->name, 0);
5306 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005307 goto err_uninit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005308
Eric W. Biederman881d9662007-09-17 11:56:21 -07005309 dev->ifindex = dev_new_index(net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005310 if (dev->iflink == -1)
5311 dev->iflink = dev->ifindex;
5312
Stephen Hemmingerd212f872007-06-27 00:47:37 -07005313 /* Fix illegal checksum combinations */
5314 if ((dev->features & NETIF_F_HW_CSUM) &&
5315 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5316 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
5317 dev->name);
5318 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5319 }
5320
5321 if ((dev->features & NETIF_F_NO_CSUM) &&
5322 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5323 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
5324 dev->name);
5325 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5326 }
5327
Herbert Xub63365a2008-10-23 01:11:29 -07005328 dev->features = netdev_fix_features(dev->features, dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005329
Lennert Buytenheke5a4a722008-08-03 01:23:10 -07005330 /* Enable software GSO if SG is supported. */
5331 if (dev->features & NETIF_F_SG)
5332 dev->features |= NETIF_F_GSO;
5333
Daniel Lezcanoaaf8cdc2008-05-02 17:00:58 -07005334 netdev_initialize_kobject(dev);
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00005335
5336 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5337 ret = notifier_to_errno(ret);
5338 if (ret)
5339 goto err_uninit;
5340
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005341 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005342 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005343 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005344 dev->reg_state = NETREG_REGISTERED;
5345
Linus Torvalds1da177e2005-04-16 15:20:36 -07005346 /*
5347 * Default initial state at registry is that the
5348 * device is present.
5349 */
5350
5351 set_bit(__LINK_STATE_PRESENT, &dev->state);
5352
Linus Torvalds1da177e2005-04-16 15:20:36 -07005353 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005354 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005355 list_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005356
5357 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005358 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07005359 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07005360 if (ret) {
5361 rollback_registered(dev);
5362 dev->reg_state = NETREG_UNREGISTERED;
5363 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005364 /*
5365 * Prevent userspace races by waiting until the network
5366 * device is fully setup before sending notifications.
5367 */
Patrick McHardya2835762010-02-26 06:34:51 +00005368 if (!dev->rtnl_link_ops ||
5369 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5370 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005371
5372out:
5373 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005374
5375err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08005376 if (dev->netdev_ops->ndo_uninit)
5377 dev->netdev_ops->ndo_uninit(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07005378 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005379}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005380EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005381
5382/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08005383 * init_dummy_netdev - init a dummy network device for NAPI
5384 * @dev: device to init
5385 *
5386 * This takes a network device structure and initialize the minimum
5387 * amount of fields so it can be used to schedule NAPI polls without
5388 * registering a full blown interface. This is to be used by drivers
5389 * that need to tie several hardware interfaces to a single NAPI
5390 * poll scheduler due to HW limitations.
5391 */
5392int init_dummy_netdev(struct net_device *dev)
5393{
5394 /* Clear everything. Note we don't initialize spinlocks
5395 * are they aren't supposed to be taken by any of the
5396 * NAPI code and this dummy netdev is supposed to be
5397 * only ever used for NAPI polls
5398 */
5399 memset(dev, 0, sizeof(struct net_device));
5400
5401 /* make sure we BUG if trying to hit standard
5402 * register/unregister code path
5403 */
5404 dev->reg_state = NETREG_DUMMY;
5405
5406 /* initialize the ref count */
5407 atomic_set(&dev->refcnt, 1);
5408
5409 /* NAPI wants this */
5410 INIT_LIST_HEAD(&dev->napi_list);
5411
5412 /* a dummy interface is started by default */
5413 set_bit(__LINK_STATE_PRESENT, &dev->state);
5414 set_bit(__LINK_STATE_START, &dev->state);
5415
5416 return 0;
5417}
5418EXPORT_SYMBOL_GPL(init_dummy_netdev);
5419
5420
5421/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005422 * register_netdev - register a network device
5423 * @dev: device to register
5424 *
5425 * Take a completed network device structure and add it to the kernel
5426 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5427 * chain. 0 is returned on success. A negative errno code is returned
5428 * on a failure to set up the device, or if the name is a duplicate.
5429 *
Borislav Petkov38b4da32007-04-20 22:14:10 -07005430 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07005431 * and expands the device name if you passed a format string to
5432 * alloc_netdev.
5433 */
5434int register_netdev(struct net_device *dev)
5435{
5436 int err;
5437
5438 rtnl_lock();
5439
5440 /*
5441 * If the name is a format string the caller wants us to do a
5442 * name allocation.
5443 */
5444 if (strchr(dev->name, '%')) {
5445 err = dev_alloc_name(dev, dev->name);
5446 if (err < 0)
5447 goto out;
5448 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005449
Linus Torvalds1da177e2005-04-16 15:20:36 -07005450 err = register_netdevice(dev);
5451out:
5452 rtnl_unlock();
5453 return err;
5454}
5455EXPORT_SYMBOL(register_netdev);
5456
5457/*
5458 * netdev_wait_allrefs - wait until all references are gone.
5459 *
5460 * This is called when unregistering network devices.
5461 *
5462 * Any protocol or device that holds a reference should register
5463 * for netdevice notification, and cleanup and put back the
5464 * reference if they receive an UNREGISTER event.
5465 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005466 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005467 */
5468static void netdev_wait_allrefs(struct net_device *dev)
5469{
5470 unsigned long rebroadcast_time, warning_time;
5471
Eric Dumazete014deb2009-11-17 05:59:21 +00005472 linkwatch_forget_dev(dev);
5473
Linus Torvalds1da177e2005-04-16 15:20:36 -07005474 rebroadcast_time = warning_time = jiffies;
5475 while (atomic_read(&dev->refcnt) != 0) {
5476 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005477 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005478
5479 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07005480 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005481 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
Octavian Purdila395264d2009-11-16 13:49:35 +00005482 * should have already handle it the first time */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005483
5484 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5485 &dev->state)) {
5486 /* We must not have linkwatch events
5487 * pending on unregister. If this
5488 * happens, we simply run the queue
5489 * unscheduled, resulting in a noop
5490 * for this device.
5491 */
5492 linkwatch_run_queue();
5493 }
5494
Stephen Hemminger6756ae42006-03-20 22:23:58 -08005495 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005496
5497 rebroadcast_time = jiffies;
5498 }
5499
5500 msleep(250);
5501
5502 if (time_after(jiffies, warning_time + 10 * HZ)) {
5503 printk(KERN_EMERG "unregister_netdevice: "
5504 "waiting for %s to become free. Usage "
5505 "count = %d\n",
5506 dev->name, atomic_read(&dev->refcnt));
5507 warning_time = jiffies;
5508 }
5509 }
5510}
5511
5512/* The sequence is:
5513 *
5514 * rtnl_lock();
5515 * ...
5516 * register_netdevice(x1);
5517 * register_netdevice(x2);
5518 * ...
5519 * unregister_netdevice(y1);
5520 * unregister_netdevice(y2);
5521 * ...
5522 * rtnl_unlock();
5523 * free_netdev(y1);
5524 * free_netdev(y2);
5525 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07005526 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07005527 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005528 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07005529 * without deadlocking with linkwatch via keventd.
5530 * 2) Since we run with the RTNL semaphore not held, we can sleep
5531 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07005532 *
5533 * We must not return until all unregister events added during
5534 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005535 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005536void netdev_run_todo(void)
5537{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005538 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005539
Linus Torvalds1da177e2005-04-16 15:20:36 -07005540 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005541 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07005542
5543 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07005544
Linus Torvalds1da177e2005-04-16 15:20:36 -07005545 while (!list_empty(&list)) {
5546 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00005547 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005548 list_del(&dev->todo_list);
5549
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005550 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005551 printk(KERN_ERR "network todo '%s' but state %d\n",
5552 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005553 dump_stack();
5554 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005555 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005556
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005557 dev->reg_state = NETREG_UNREGISTERED;
5558
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005559 on_each_cpu(flush_backlog, dev, 1);
5560
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005561 netdev_wait_allrefs(dev);
5562
5563 /* paranoia */
5564 BUG_ON(atomic_read(&dev->refcnt));
Ilpo Järvinen547b7922008-07-25 21:43:18 -07005565 WARN_ON(dev->ip_ptr);
5566 WARN_ON(dev->ip6_ptr);
5567 WARN_ON(dev->dn_ptr);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005568
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07005569 if (dev->destructor)
5570 dev->destructor(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07005571
5572 /* Free network device */
5573 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005574 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005575}
5576
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005577/**
Eric Dumazetd83345a2009-11-16 03:36:51 +00005578 * dev_txq_stats_fold - fold tx_queues stats
5579 * @dev: device to get statistics from
5580 * @stats: struct net_device_stats to hold results
5581 */
5582void dev_txq_stats_fold(const struct net_device *dev,
5583 struct net_device_stats *stats)
5584{
5585 unsigned long tx_bytes = 0, tx_packets = 0, tx_dropped = 0;
5586 unsigned int i;
5587 struct netdev_queue *txq;
5588
5589 for (i = 0; i < dev->num_tx_queues; i++) {
5590 txq = netdev_get_tx_queue(dev, i);
5591 tx_bytes += txq->tx_bytes;
5592 tx_packets += txq->tx_packets;
5593 tx_dropped += txq->tx_dropped;
5594 }
5595 if (tx_bytes || tx_packets || tx_dropped) {
5596 stats->tx_bytes = tx_bytes;
5597 stats->tx_packets = tx_packets;
5598 stats->tx_dropped = tx_dropped;
5599 }
5600}
5601EXPORT_SYMBOL(dev_txq_stats_fold);
5602
5603/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005604 * dev_get_stats - get network device statistics
5605 * @dev: device to get statistics from
5606 *
5607 * Get network statistics from device. The device driver may provide
5608 * its own method by setting dev->netdev_ops->get_stats; otherwise
5609 * the internal statistics structure is used.
5610 */
5611const struct net_device_stats *dev_get_stats(struct net_device *dev)
Eric Dumazet7004bf22009-05-18 00:34:33 +00005612{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005613 const struct net_device_ops *ops = dev->netdev_ops;
5614
5615 if (ops->ndo_get_stats)
5616 return ops->ndo_get_stats(dev);
Eric Dumazet7004bf22009-05-18 00:34:33 +00005617
Eric Dumazetd83345a2009-11-16 03:36:51 +00005618 dev_txq_stats_fold(dev, &dev->stats);
5619 return &dev->stats;
Rusty Russellc45d2862007-03-28 14:29:08 -07005620}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08005621EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07005622
David S. Millerdc2b4842008-07-08 17:18:23 -07005623static void netdev_init_one_queue(struct net_device *dev,
David S. Millere8a04642008-07-17 00:34:19 -07005624 struct netdev_queue *queue,
5625 void *_unused)
David S. Millerdc2b4842008-07-08 17:18:23 -07005626{
David S. Millerdc2b4842008-07-08 17:18:23 -07005627 queue->dev = dev;
5628}
5629
David S. Millerbb949fb2008-07-08 16:55:56 -07005630static void netdev_init_queues(struct net_device *dev)
5631{
David S. Millere8a04642008-07-17 00:34:19 -07005632 netdev_init_one_queue(dev, &dev->rx_queue, NULL);
5633 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
David S. Millerc3f26a22008-07-31 16:58:50 -07005634 spin_lock_init(&dev->tx_global_lock);
David S. Millerbb949fb2008-07-08 16:55:56 -07005635}
5636
Linus Torvalds1da177e2005-04-16 15:20:36 -07005637/**
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005638 * alloc_netdev_mq - allocate network device
Linus Torvalds1da177e2005-04-16 15:20:36 -07005639 * @sizeof_priv: size of private data to allocate space for
5640 * @name: device name format string
5641 * @setup: callback to initialize device
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005642 * @queue_count: the number of subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07005643 *
5644 * Allocates a struct net_device with private data area for driver use
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005645 * and performs basic initialization. Also allocates subquue structs
5646 * for each queue on the device at the end of the netdevice.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005647 */
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005648struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
5649 void (*setup)(struct net_device *), unsigned int queue_count)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005650{
David S. Millere8a04642008-07-17 00:34:19 -07005651 struct netdev_queue *tx;
Tom Herbert0a9627f2010-03-16 08:03:29 +00005652 struct netdev_rx_queue *rx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005653 struct net_device *dev;
Stephen Hemminger79439862008-07-21 13:28:44 -07005654 size_t alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005655 struct net_device *p;
Tom Herbert0a9627f2010-03-16 08:03:29 +00005656 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005657
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005658 BUG_ON(strlen(name) >= sizeof(dev->name));
5659
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005660 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005661 if (sizeof_priv) {
5662 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005663 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07005664 alloc_size += sizeof_priv;
5665 }
5666 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005667 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005668
Paolo 'Blaisorblade' Giarrusso31380de2006-04-06 22:38:28 -07005669 p = kzalloc(alloc_size, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005670 if (!p) {
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07005671 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005672 return NULL;
5673 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005674
Stephen Hemminger79439862008-07-21 13:28:44 -07005675 tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);
David S. Millere8a04642008-07-17 00:34:19 -07005676 if (!tx) {
5677 printk(KERN_ERR "alloc_netdev: Unable to allocate "
5678 "tx qdiscs.\n");
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005679 goto free_p;
David S. Millere8a04642008-07-17 00:34:19 -07005680 }
5681
Tom Herbert0a9627f2010-03-16 08:03:29 +00005682 rx = kcalloc(queue_count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5683 if (!rx) {
5684 printk(KERN_ERR "alloc_netdev: Unable to allocate "
5685 "rx queues.\n");
5686 goto free_tx;
5687 }
5688
5689 atomic_set(&rx->count, queue_count);
5690
5691 /*
5692 * Set a pointer to first element in the array which holds the
5693 * reference count.
5694 */
5695 for (i = 0; i < queue_count; i++)
5696 rx[i].first = rx;
5697
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00005698 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005699 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005700
5701 if (dev_addr_init(dev))
Tom Herbert0a9627f2010-03-16 08:03:29 +00005702 goto free_rx;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005703
Jiri Pirkoccffad252009-05-22 23:22:17 +00005704 dev_unicast_init(dev);
5705
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005706 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005707
David S. Millere8a04642008-07-17 00:34:19 -07005708 dev->_tx = tx;
5709 dev->num_tx_queues = queue_count;
David S. Millerfd2ea0a2008-07-17 01:56:23 -07005710 dev->real_num_tx_queues = queue_count;
David S. Millere8a04642008-07-17 00:34:19 -07005711
Tom Herbert0a9627f2010-03-16 08:03:29 +00005712 dev->_rx = rx;
5713 dev->num_rx_queues = queue_count;
5714
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07005715 dev->gso_max_size = GSO_MAX_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005716
David S. Millerbb949fb2008-07-08 16:55:56 -07005717 netdev_init_queues(dev);
5718
Peter P Waskiewicz Jr15682bc2010-02-10 20:03:05 -08005719 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5720 dev->ethtool_ntuple_list.count = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005721 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005722 INIT_LIST_HEAD(&dev->unreg_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00005723 INIT_LIST_HEAD(&dev->link_watch_list);
Eric Dumazet93f154b2009-05-18 22:19:19 -07005724 dev->priv_flags = IFF_XMIT_DST_RELEASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005725 setup(dev);
5726 strcpy(dev->name, name);
5727 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005728
Tom Herbert0a9627f2010-03-16 08:03:29 +00005729free_rx:
5730 kfree(rx);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005731free_tx:
5732 kfree(tx);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00005733free_p:
5734 kfree(p);
5735 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005736}
Peter P Waskiewicz Jrf25f4e42007-07-06 13:36:20 -07005737EXPORT_SYMBOL(alloc_netdev_mq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005738
5739/**
5740 * free_netdev - free network device
5741 * @dev: device
5742 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005743 * This function does the last stage of destroying an allocated device
5744 * interface. The reference to the device object is released.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005745 * If this is the last reference then it will be freed.
5746 */
5747void free_netdev(struct net_device *dev)
5748{
Herbert Xud565b0a2008-12-15 23:38:52 -08005749 struct napi_struct *p, *n;
5750
Denis V. Lunevf3005d72008-04-16 02:02:18 -07005751 release_net(dev_net(dev));
5752
David S. Millere8a04642008-07-17 00:34:19 -07005753 kfree(dev->_tx);
5754
Jiri Pirkof001fde2009-05-05 02:48:28 +00005755 /* Flush device addresses */
5756 dev_addr_flush(dev);
5757
Peter P Waskiewicz Jr15682bc2010-02-10 20:03:05 -08005758 /* Clear ethtool n-tuple list */
5759 ethtool_ntuple_flush(dev);
5760
Herbert Xud565b0a2008-12-15 23:38:52 -08005761 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5762 netif_napi_del(p);
5763
Stephen Hemminger3041a062006-05-26 13:25:24 -07005764 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005765 if (dev->reg_state == NETREG_UNINITIALIZED) {
5766 kfree((char *)dev - dev->padded);
5767 return;
5768 }
5769
5770 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5771 dev->reg_state = NETREG_RELEASED;
5772
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07005773 /* will free via device release */
5774 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005775}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005776EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005777
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07005778/**
5779 * synchronize_net - Synchronize with packet receive processing
5780 *
5781 * Wait for packets currently being received to be done.
5782 * Does not block later packets from starting.
5783 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09005784void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005785{
5786 might_sleep();
Paul E. McKenneyfbd568a3e2005-05-01 08:59:04 -07005787 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005788}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07005789EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005790
5791/**
Eric Dumazet44a08732009-10-27 07:03:04 +00005792 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07005793 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00005794 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08005795 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07005796 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005797 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00005798 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005799 *
5800 * Callers must hold the rtnl semaphore. You may want
5801 * unregister_netdev() instead of this.
5802 */
5803
Eric Dumazet44a08732009-10-27 07:03:04 +00005804void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005805{
Herbert Xua6620712007-12-12 19:21:56 -08005806 ASSERT_RTNL();
5807
Eric Dumazet44a08732009-10-27 07:03:04 +00005808 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00005809 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00005810 } else {
5811 rollback_registered(dev);
5812 /* Finish processing unregister after unlock */
5813 net_set_todo(dev);
5814 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005815}
Eric Dumazet44a08732009-10-27 07:03:04 +00005816EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005817
5818/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005819 * unregister_netdevice_many - unregister many devices
5820 * @head: list of devices
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005821 */
5822void unregister_netdevice_many(struct list_head *head)
5823{
5824 struct net_device *dev;
5825
5826 if (!list_empty(head)) {
5827 rollback_registered_many(head);
5828 list_for_each_entry(dev, head, unreg_list)
5829 net_set_todo(dev);
5830 }
5831}
Eric Dumazet63c80992009-10-27 07:06:49 +00005832EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00005833
5834/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07005835 * unregister_netdev - remove device from the kernel
5836 * @dev: device
5837 *
5838 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08005839 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005840 *
5841 * This is just a wrapper for unregister_netdevice that takes
5842 * the rtnl semaphore. In general you want to use this and not
5843 * unregister_netdevice.
5844 */
5845void unregister_netdev(struct net_device *dev)
5846{
5847 rtnl_lock();
5848 unregister_netdevice(dev);
5849 rtnl_unlock();
5850}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005851EXPORT_SYMBOL(unregister_netdev);
5852
Eric W. Biedermance286d32007-09-12 13:53:49 +02005853/**
5854 * dev_change_net_namespace - move device to different nethost namespace
5855 * @dev: device
5856 * @net: network namespace
5857 * @pat: If not NULL name pattern to try if the current device name
5858 * is already taken in the destination network namespace.
5859 *
5860 * This function shuts down a device interface and moves it
5861 * to a new network namespace. On success 0 is returned, on
5862 * a failure a netagive errno code is returned.
5863 *
5864 * Callers must hold the rtnl semaphore.
5865 */
5866
5867int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5868{
Eric W. Biedermance286d32007-09-12 13:53:49 +02005869 int err;
5870
5871 ASSERT_RTNL();
5872
5873 /* Don't allow namespace local devices to be moved. */
5874 err = -EINVAL;
5875 if (dev->features & NETIF_F_NETNS_LOCAL)
5876 goto out;
5877
Eric W. Biederman38918452008-10-27 17:51:47 -07005878#ifdef CONFIG_SYSFS
5879 /* Don't allow real devices to be moved when sysfs
5880 * is enabled.
5881 */
5882 err = -EINVAL;
5883 if (dev->dev.parent)
5884 goto out;
5885#endif
5886
Eric W. Biedermance286d32007-09-12 13:53:49 +02005887 /* Ensure the device has been registrered */
5888 err = -EINVAL;
5889 if (dev->reg_state != NETREG_REGISTERED)
5890 goto out;
5891
5892 /* Get out if there is nothing todo */
5893 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09005894 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02005895 goto out;
5896
5897 /* Pick the destination device name, and ensure
5898 * we can use it in the destination network namespace.
5899 */
5900 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00005901 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02005902 /* We get here if we can't use the current device name */
5903 if (!pat)
5904 goto out;
Octavian Purdilad9031022009-11-18 02:36:59 +00005905 if (dev_get_valid_name(net, pat, dev->name, 1))
Eric W. Biedermance286d32007-09-12 13:53:49 +02005906 goto out;
5907 }
5908
5909 /*
5910 * And now a mini version of register_netdevice unregister_netdevice.
5911 */
5912
5913 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07005914 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005915
5916 /* And unlink it from device chain */
5917 err = -ENODEV;
5918 unlist_netdevice(dev);
5919
5920 synchronize_net();
5921
5922 /* Shutdown queueing discipline. */
5923 dev_shutdown(dev);
5924
5925 /* Notify protocols, that we are about to destroy
5926 this device. They should clean all the things.
5927 */
5928 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00005929 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005930
5931 /*
5932 * Flush the unicast and multicast chains
5933 */
Jiri Pirkoccffad252009-05-22 23:22:17 +00005934 dev_unicast_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005935 dev_addr_discard(dev);
5936
Eric W. Biederman38918452008-10-27 17:51:47 -07005937 netdev_unregister_kobject(dev);
5938
Eric W. Biedermance286d32007-09-12 13:53:49 +02005939 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09005940 dev_net_set(dev, net);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005941
Eric W. Biedermance286d32007-09-12 13:53:49 +02005942 /* If there is an ifindex conflict assign a new one */
5943 if (__dev_get_by_index(net, dev->ifindex)) {
5944 int iflink = (dev->iflink == dev->ifindex);
5945 dev->ifindex = dev_new_index(net);
5946 if (iflink)
5947 dev->iflink = dev->ifindex;
5948 }
5949
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005950 /* Fixup kobjects */
Daniel Lezcanoaaf8cdc2008-05-02 17:00:58 -07005951 err = netdev_register_kobject(dev);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07005952 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005953
5954 /* Add the device back in the hashes */
5955 list_netdevice(dev);
5956
5957 /* Notify protocols, that a new device appeared. */
5958 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5959
Eric W. Biedermand90a9092009-12-12 22:11:15 +00005960 /*
5961 * Prevent userspace races by waiting until the network
5962 * device is fully setup before sending notifications.
5963 */
5964 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5965
Eric W. Biedermance286d32007-09-12 13:53:49 +02005966 synchronize_net();
5967 err = 0;
5968out:
5969 return err;
5970}
Johannes Berg463d0182009-07-14 00:33:35 +02005971EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02005972
Linus Torvalds1da177e2005-04-16 15:20:36 -07005973static int dev_cpu_callback(struct notifier_block *nfb,
5974 unsigned long action,
5975 void *ocpu)
5976{
5977 struct sk_buff **list_skb;
David S. Miller37437bb2008-07-16 02:15:04 -07005978 struct Qdisc **list_net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005979 struct sk_buff *skb;
5980 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5981 struct softnet_data *sd, *oldsd;
5982
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005983 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005984 return NOTIFY_OK;
5985
5986 local_irq_disable();
5987 cpu = smp_processor_id();
5988 sd = &per_cpu(softnet_data, cpu);
5989 oldsd = &per_cpu(softnet_data, oldcpu);
5990
5991 /* Find end of our completion_queue. */
5992 list_skb = &sd->completion_queue;
5993 while (*list_skb)
5994 list_skb = &(*list_skb)->next;
5995 /* Append completion queue from offline CPU. */
5996 *list_skb = oldsd->completion_queue;
5997 oldsd->completion_queue = NULL;
5998
5999 /* Find end of our output_queue. */
6000 list_net = &sd->output_queue;
6001 while (*list_net)
6002 list_net = &(*list_net)->next_sched;
6003 /* Append output queue from offline CPU. */
6004 *list_net = oldsd->output_queue;
6005 oldsd->output_queue = NULL;
6006
6007 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6008 local_irq_enable();
6009
6010 /* Process offline CPU's input_pkt_queue */
6011 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue)))
6012 netif_rx(skb);
6013
6014 return NOTIFY_OK;
6015}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006016
6017
Herbert Xu7f353bf2007-08-10 15:47:58 -07006018/**
Herbert Xub63365a2008-10-23 01:11:29 -07006019 * netdev_increment_features - increment feature set by one
6020 * @all: current feature set
6021 * @one: new feature set
6022 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07006023 *
6024 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07006025 * @one to the master device with current feature set @all. Will not
6026 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07006027 */
Herbert Xub63365a2008-10-23 01:11:29 -07006028unsigned long netdev_increment_features(unsigned long all, unsigned long one,
6029 unsigned long mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07006030{
Herbert Xub63365a2008-10-23 01:11:29 -07006031 /* If device needs checksumming, downgrade to it. */
Eric Dumazetd1b19df2009-09-03 01:29:39 -07006032 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
Herbert Xub63365a2008-10-23 01:11:29 -07006033 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
6034 else if (mask & NETIF_F_ALL_CSUM) {
6035 /* If one device supports v4/v6 checksumming, set for all. */
6036 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
6037 !(all & NETIF_F_GEN_CSUM)) {
6038 all &= ~NETIF_F_ALL_CSUM;
6039 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
6040 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07006041
Herbert Xub63365a2008-10-23 01:11:29 -07006042 /* If one device supports hw checksumming, set for all. */
6043 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
6044 all &= ~NETIF_F_ALL_CSUM;
6045 all |= NETIF_F_HW_CSUM;
6046 }
6047 }
Herbert Xu7f353bf2007-08-10 15:47:58 -07006048
Herbert Xub63365a2008-10-23 01:11:29 -07006049 one |= NETIF_F_ALL_CSUM;
Herbert Xu7f353bf2007-08-10 15:47:58 -07006050
Herbert Xub63365a2008-10-23 01:11:29 -07006051 one |= all & NETIF_F_ONE_FOR_ALL;
Sridhar Samudralad9f59502009-10-07 12:24:25 +00006052 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
Herbert Xub63365a2008-10-23 01:11:29 -07006053 all |= one & mask & NETIF_F_ONE_FOR_ALL;
Herbert Xu7f353bf2007-08-10 15:47:58 -07006054
6055 return all;
6056}
Herbert Xub63365a2008-10-23 01:11:29 -07006057EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07006058
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006059static struct hlist_head *netdev_create_hash(void)
6060{
6061 int i;
6062 struct hlist_head *hash;
6063
6064 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6065 if (hash != NULL)
6066 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6067 INIT_HLIST_HEAD(&hash[i]);
6068
6069 return hash;
6070}
6071
Eric W. Biederman881d9662007-09-17 11:56:21 -07006072/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07006073static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006074{
Eric W. Biederman881d9662007-09-17 11:56:21 -07006075 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07006076
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006077 net->dev_name_head = netdev_create_hash();
6078 if (net->dev_name_head == NULL)
6079 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006080
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006081 net->dev_index_head = netdev_create_hash();
6082 if (net->dev_index_head == NULL)
6083 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006084
6085 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07006086
6087err_idx:
6088 kfree(net->dev_name_head);
6089err_name:
6090 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07006091}
6092
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07006093/**
6094 * netdev_drivername - network driver for the device
6095 * @dev: network device
6096 * @buffer: buffer for resulting name
6097 * @len: size of buffer
6098 *
6099 * Determine network driver for device.
6100 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006101char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
Arjan van de Ven6579e572008-07-21 13:31:48 -07006102{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07006103 const struct device_driver *driver;
6104 const struct device *parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07006105
6106 if (len <= 0 || !buffer)
6107 return buffer;
6108 buffer[0] = 0;
6109
6110 parent = dev->dev.parent;
6111
6112 if (!parent)
6113 return buffer;
6114
6115 driver = parent->driver;
6116 if (driver && driver->name)
6117 strlcpy(buffer, driver->name, len);
6118 return buffer;
6119}
6120
Pavel Emelyanov46650792007-10-08 20:38:39 -07006121static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07006122{
6123 kfree(net->dev_name_head);
6124 kfree(net->dev_index_head);
6125}
6126
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006127static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07006128 .init = netdev_init,
6129 .exit = netdev_exit,
6130};
6131
Pavel Emelyanov46650792007-10-08 20:38:39 -07006132static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02006133{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006134 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02006135 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006136 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02006137 * initial network namespace
6138 */
6139 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006140 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006141 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006142 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02006143
6144 /* Ignore unmoveable devices (i.e. loopback) */
6145 if (dev->features & NETIF_F_NETNS_LOCAL)
6146 continue;
6147
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00006148 /* Leave virtual devices for the generic cleanup */
6149 if (dev->rtnl_link_ops)
6150 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08006151
Eric W. Biedermance286d32007-09-12 13:53:49 +02006152 /* Push remaing network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006153 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6154 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02006155 if (err) {
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006156 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
Eric W. Biedermance286d32007-09-12 13:53:49 +02006157 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07006158 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02006159 }
6160 }
6161 rtnl_unlock();
6162}
6163
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006164static void __net_exit default_device_exit_batch(struct list_head *net_list)
6165{
6166 /* At exit all network devices most be removed from a network
6167 * namespace. Do this in the reverse order of registeration.
6168 * Do this across as many network namespaces as possible to
6169 * improve batching efficiency.
6170 */
6171 struct net_device *dev;
6172 struct net *net;
6173 LIST_HEAD(dev_kill_list);
6174
6175 rtnl_lock();
6176 list_for_each_entry(net, net_list, exit_list) {
6177 for_each_netdev_reverse(net, dev) {
6178 if (dev->rtnl_link_ops)
6179 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6180 else
6181 unregister_netdevice_queue(dev, &dev_kill_list);
6182 }
6183 }
6184 unregister_netdevice_many(&dev_kill_list);
6185 rtnl_unlock();
6186}
6187
Denis V. Lunev022cbae2007-11-13 03:23:50 -08006188static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02006189 .exit = default_device_exit,
Eric W. Biederman04dc7f62009-12-03 02:29:04 +00006190 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02006191};
6192
Linus Torvalds1da177e2005-04-16 15:20:36 -07006193/*
6194 * Initialize the DEV module. At boot time this walks the device list and
6195 * unhooks any devices that fail to initialise (normally hardware not
6196 * present) and leaves us with a valid list of present and active devices.
6197 *
6198 */
6199
6200/*
6201 * This is called single threaded during boot, so no need
6202 * to take the rtnl semaphore.
6203 */
6204static int __init net_dev_init(void)
6205{
6206 int i, rc = -ENOMEM;
6207
6208 BUG_ON(!dev_boot_phase);
6209
Linus Torvalds1da177e2005-04-16 15:20:36 -07006210 if (dev_proc_init())
6211 goto out;
6212
Eric W. Biederman8b41d182007-09-26 22:02:53 -07006213 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07006214 goto out;
6215
6216 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08006217 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006218 INIT_LIST_HEAD(&ptype_base[i]);
6219
Eric W. Biederman881d9662007-09-17 11:56:21 -07006220 if (register_pernet_subsys(&netdev_net_ops))
6221 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006222
6223 /*
6224 * Initialise the packet receive queues.
6225 */
6226
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07006227 for_each_possible_cpu(i) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006228 struct softnet_data *queue;
6229
6230 queue = &per_cpu(softnet_data, i);
6231 skb_queue_head_init(&queue->input_pkt_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006232 queue->completion_queue = NULL;
6233 INIT_LIST_HEAD(&queue->poll_list);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006234
Tom Herbert1e94d722010-03-18 17:45:44 -07006235#ifdef CONFIG_SMP
Tom Herbert0a9627f2010-03-16 08:03:29 +00006236 queue->csd.func = trigger_softirq;
6237 queue->csd.info = queue;
6238 queue->csd.flags = 0;
Tom Herbert1e94d722010-03-18 17:45:44 -07006239#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00006240
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006241 queue->backlog.poll = process_backlog;
6242 queue->backlog.weight = weight_p;
Herbert Xud565b0a2008-12-15 23:38:52 -08006243 queue->backlog.gro_list = NULL;
Herbert Xu4ae55442009-02-08 18:00:36 +00006244 queue->backlog.gro_count = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006245 }
6246
Linus Torvalds1da177e2005-04-16 15:20:36 -07006247 dev_boot_phase = 0;
6248
Eric W. Biederman505d4f72008-11-07 22:54:20 -08006249 /* The loopback device is special if any other network devices
6250 * is present in a network namespace the loopback device must
6251 * be present. Since we now dynamically allocate and free the
6252 * loopback device ensure this invariant is maintained by
6253 * keeping the loopback device as the first device on the
6254 * list of network devices. Ensuring the loopback devices
6255 * is the first device that appears and the last network device
6256 * that disappears.
6257 */
6258 if (register_pernet_device(&loopback_net_ops))
6259 goto out;
6260
6261 if (register_pernet_device(&default_device_ops))
6262 goto out;
6263
Carlos R. Mafra962cf362008-05-15 11:15:37 -03006264 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6265 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006266
6267 hotcpu_notifier(dev_cpu_callback, 0);
6268 dst_init();
6269 dev_mcast_init();
6270 rc = 0;
6271out:
6272 return rc;
6273}
6274
6275subsys_initcall(net_dev_init);
6276
Krishna Kumare88721f2009-02-18 17:55:02 -08006277static int __init initialize_hashrnd(void)
6278{
Tom Herbert0a9627f2010-03-16 08:03:29 +00006279 get_random_bytes(&hashrnd, sizeof(hashrnd));
Krishna Kumare88721f2009-02-18 17:55:02 -08006280 return 0;
6281}
6282
6283late_initcall_sync(initialize_hashrnd);
6284