blob: 6958fe7c9a775333fbca7f82ddf0bc29bd9e4132 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * net/sched/sch_api.c Packet scheduler API.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10 *
11 * Fixes:
12 *
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
16 */
17
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/module.h>
19#include <linux/types.h>
20#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/string.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/errno.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/skbuff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/init.h>
25#include <linux/proc_fs.h>
26#include <linux/seq_file.h>
27#include <linux/kmod.h>
28#include <linux/list.h>
Patrick McHardy41794772007-03-16 01:19:15 -070029#include <linux/hrtimer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020031#include <net/net_namespace.h>
Denis V. Lunevb8542722007-12-01 00:21:31 +110032#include <net/sock.h>
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -070033#include <net/netlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <net/pkt_sched.h>
35
Linus Torvalds1da177e2005-04-16 15:20:36 -070036static int qdisc_notify(struct sk_buff *oskb, struct nlmsghdr *n, u32 clid,
37 struct Qdisc *old, struct Qdisc *new);
38static int tclass_notify(struct sk_buff *oskb, struct nlmsghdr *n,
39 struct Qdisc *q, unsigned long cl, int event);
40
41/*
42
43 Short review.
44 -------------
45
46 This file consists of two interrelated parts:
47
48 1. queueing disciplines manager frontend.
49 2. traffic classes manager frontend.
50
51 Generally, queueing discipline ("qdisc") is a black box,
52 which is able to enqueue packets and to dequeue them (when
53 device is ready to send something) in order and at times
54 determined by algorithm hidden in it.
55
56 qdisc's are divided to two categories:
57 - "queues", which have no internal structure visible from outside.
58 - "schedulers", which split all the packets to "traffic classes",
59 using "packet classifiers" (look at cls_api.c)
60
61 In turn, classes may have child qdiscs (as rule, queues)
62 attached to them etc. etc. etc.
63
64 The goal of the routines in this file is to translate
65 information supplied by user in the form of handles
66 to more intelligible for kernel form, to make some sanity
67 checks and part of work, which is common to all qdiscs
68 and to provide rtnetlink notifications.
69
70 All real intelligent work is done inside qdisc modules.
71
72
73
74 Every discipline has two major routines: enqueue and dequeue.
75
76 ---dequeue
77
78 dequeue usually returns a skb to send. It is allowed to return NULL,
79 but it does not mean that queue is empty, it just means that
80 discipline does not want to send anything this time.
81 Queue is really empty if q->q.qlen == 0.
82 For complicated disciplines with multiple queues q->q is not
83 real packet queue, but however q->q.qlen must be valid.
84
85 ---enqueue
86
87 enqueue returns 0, if packet was enqueued successfully.
88 If packet (this one or another one) was dropped, it returns
89 not zero error code.
90 NET_XMIT_DROP - this packet dropped
91 Expected action: do not backoff, but wait until queue will clear.
92 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
93 Expected action: backoff or ignore
94 NET_XMIT_POLICED - dropped by police.
95 Expected action: backoff or error to real-time apps.
96
97 Auxiliary routines:
98
99 ---requeue
100
101 requeues once dequeued packet. It is used for non-standard or
David S. Millere65d22e2008-07-08 16:46:01 -0700102 just buggy devices, which can defer output even if netif_queue_stopped()=0.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
104 ---reset
105
106 returns qdisc to initial state: purge all buffers, clear all
107 timers, counters (except for statistics) etc.
108
109 ---init
110
111 initializes newly created qdisc.
112
113 ---destroy
114
115 destroys resources allocated by init and during lifetime of qdisc.
116
117 ---change
118
119 changes qdisc parameters.
120 */
121
122/* Protects list of registered TC modules. It is pure SMP lock. */
123static DEFINE_RWLOCK(qdisc_mod_lock);
124
125
126/************************************************
127 * Queueing disciplines manipulation. *
128 ************************************************/
129
130
131/* The list of all installed queueing disciplines. */
132
133static struct Qdisc_ops *qdisc_base;
134
135/* Register/uregister queueing discipline */
136
137int register_qdisc(struct Qdisc_ops *qops)
138{
139 struct Qdisc_ops *q, **qp;
140 int rc = -EEXIST;
141
142 write_lock(&qdisc_mod_lock);
143 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
144 if (!strcmp(qops->id, q->id))
145 goto out;
146
147 if (qops->enqueue == NULL)
148 qops->enqueue = noop_qdisc_ops.enqueue;
149 if (qops->requeue == NULL)
150 qops->requeue = noop_qdisc_ops.requeue;
151 if (qops->dequeue == NULL)
152 qops->dequeue = noop_qdisc_ops.dequeue;
153
154 qops->next = NULL;
155 *qp = qops;
156 rc = 0;
157out:
158 write_unlock(&qdisc_mod_lock);
159 return rc;
160}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800161EXPORT_SYMBOL(register_qdisc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162
163int unregister_qdisc(struct Qdisc_ops *qops)
164{
165 struct Qdisc_ops *q, **qp;
166 int err = -ENOENT;
167
168 write_lock(&qdisc_mod_lock);
169 for (qp = &qdisc_base; (q=*qp)!=NULL; qp = &q->next)
170 if (q == qops)
171 break;
172 if (q) {
173 *qp = q->next;
174 q->next = NULL;
175 err = 0;
176 }
177 write_unlock(&qdisc_mod_lock);
178 return err;
179}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800180EXPORT_SYMBOL(unregister_qdisc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
182/* We know handle. Find qdisc among all qdisc's attached to device
183 (root qdisc, all its children, children of children etc.)
184 */
185
David S. Milleread81cc2008-07-17 00:50:32 -0700186struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
Patrick McHardy43effa12006-11-29 17:35:48 -0800187{
188 struct Qdisc *q;
189
David S. Milleread81cc2008-07-17 00:50:32 -0700190 list_for_each_entry(q, &dev->qdisc_list, list) {
Patrick McHardy43effa12006-11-29 17:35:48 -0800191 if (q->handle == handle)
192 return q;
193 }
194 return NULL;
195}
196
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
198{
199 unsigned long cl;
200 struct Qdisc *leaf;
Eric Dumazet20fea082007-11-14 01:44:41 -0800201 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202
203 if (cops == NULL)
204 return NULL;
205 cl = cops->get(p, classid);
206
207 if (cl == 0)
208 return NULL;
209 leaf = cops->leaf(p, cl);
210 cops->put(p, cl);
211 return leaf;
212}
213
214/* Find queueing discipline by name */
215
Patrick McHardy1e904742008-01-22 22:11:17 -0800216static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
218 struct Qdisc_ops *q = NULL;
219
220 if (kind) {
221 read_lock(&qdisc_mod_lock);
222 for (q = qdisc_base; q; q = q->next) {
Patrick McHardy1e904742008-01-22 22:11:17 -0800223 if (nla_strcmp(kind, q->id) == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 if (!try_module_get(q->owner))
225 q = NULL;
226 break;
227 }
228 }
229 read_unlock(&qdisc_mod_lock);
230 }
231 return q;
232}
233
234static struct qdisc_rate_table *qdisc_rtab_list;
235
Patrick McHardy1e904742008-01-22 22:11:17 -0800236struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237{
238 struct qdisc_rate_table *rtab;
239
240 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
241 if (memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) == 0) {
242 rtab->refcnt++;
243 return rtab;
244 }
245 }
246
Patrick McHardy5feb5e12008-01-23 20:35:19 -0800247 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
248 nla_len(tab) != TC_RTAB_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 return NULL;
250
251 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
252 if (rtab) {
253 rtab->rate = *r;
254 rtab->refcnt = 1;
Patrick McHardy1e904742008-01-22 22:11:17 -0800255 memcpy(rtab->data, nla_data(tab), 1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 rtab->next = qdisc_rtab_list;
257 qdisc_rtab_list = rtab;
258 }
259 return rtab;
260}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800261EXPORT_SYMBOL(qdisc_get_rtab);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262
263void qdisc_put_rtab(struct qdisc_rate_table *tab)
264{
265 struct qdisc_rate_table *rtab, **rtabp;
266
267 if (!tab || --tab->refcnt)
268 return;
269
270 for (rtabp = &qdisc_rtab_list; (rtab=*rtabp) != NULL; rtabp = &rtab->next) {
271 if (rtab == tab) {
272 *rtabp = rtab->next;
273 kfree(rtab);
274 return;
275 }
276 }
277}
Patrick McHardy62e3ba12008-01-22 22:10:23 -0800278EXPORT_SYMBOL(qdisc_put_rtab);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Patrick McHardy41794772007-03-16 01:19:15 -0700280static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
281{
282 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
283 timer);
284
285 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
Stephen Hemminger11274e52007-03-22 12:17:42 -0700286 smp_wmb();
David S. Miller37437bb2008-07-16 02:15:04 -0700287 __netif_schedule(wd->qdisc);
Stephen Hemminger19365022007-03-22 12:18:35 -0700288
Patrick McHardy41794772007-03-16 01:19:15 -0700289 return HRTIMER_NORESTART;
290}
291
292void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
293{
294 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
295 wd->timer.function = qdisc_watchdog;
296 wd->qdisc = qdisc;
297}
298EXPORT_SYMBOL(qdisc_watchdog_init);
299
300void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, psched_time_t expires)
301{
302 ktime_t time;
303
304 wd->qdisc->flags |= TCQ_F_THROTTLED;
305 time = ktime_set(0, 0);
306 time = ktime_add_ns(time, PSCHED_US2NS(expires));
307 hrtimer_start(&wd->timer, time, HRTIMER_MODE_ABS);
308}
309EXPORT_SYMBOL(qdisc_watchdog_schedule);
310
311void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
312{
313 hrtimer_cancel(&wd->timer);
314 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
315}
316EXPORT_SYMBOL(qdisc_watchdog_cancel);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
Patrick McHardy6fe1c7a2008-07-05 23:21:31 -0700318struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
319{
320 unsigned int size = n * sizeof(struct hlist_head), i;
321 struct hlist_head *h;
322
323 if (size <= PAGE_SIZE)
324 h = kmalloc(size, GFP_KERNEL);
325 else
326 h = (struct hlist_head *)
327 __get_free_pages(GFP_KERNEL, get_order(size));
328
329 if (h != NULL) {
330 for (i = 0; i < n; i++)
331 INIT_HLIST_HEAD(&h[i]);
332 }
333 return h;
334}
335
336static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
337{
338 unsigned int size = n * sizeof(struct hlist_head);
339
340 if (size <= PAGE_SIZE)
341 kfree(h);
342 else
343 free_pages((unsigned long)h, get_order(size));
344}
345
346void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
347{
348 struct Qdisc_class_common *cl;
349 struct hlist_node *n, *next;
350 struct hlist_head *nhash, *ohash;
351 unsigned int nsize, nmask, osize;
352 unsigned int i, h;
353
354 /* Rehash when load factor exceeds 0.75 */
355 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
356 return;
357 nsize = clhash->hashsize * 2;
358 nmask = nsize - 1;
359 nhash = qdisc_class_hash_alloc(nsize);
360 if (nhash == NULL)
361 return;
362
363 ohash = clhash->hash;
364 osize = clhash->hashsize;
365
366 sch_tree_lock(sch);
367 for (i = 0; i < osize; i++) {
368 hlist_for_each_entry_safe(cl, n, next, &ohash[i], hnode) {
369 h = qdisc_class_hash(cl->classid, nmask);
370 hlist_add_head(&cl->hnode, &nhash[h]);
371 }
372 }
373 clhash->hash = nhash;
374 clhash->hashsize = nsize;
375 clhash->hashmask = nmask;
376 sch_tree_unlock(sch);
377
378 qdisc_class_hash_free(ohash, osize);
379}
380EXPORT_SYMBOL(qdisc_class_hash_grow);
381
382int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
383{
384 unsigned int size = 4;
385
386 clhash->hash = qdisc_class_hash_alloc(size);
387 if (clhash->hash == NULL)
388 return -ENOMEM;
389 clhash->hashsize = size;
390 clhash->hashmask = size - 1;
391 clhash->hashelems = 0;
392 return 0;
393}
394EXPORT_SYMBOL(qdisc_class_hash_init);
395
396void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
397{
398 qdisc_class_hash_free(clhash->hash, clhash->hashsize);
399}
400EXPORT_SYMBOL(qdisc_class_hash_destroy);
401
402void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
403 struct Qdisc_class_common *cl)
404{
405 unsigned int h;
406
407 INIT_HLIST_NODE(&cl->hnode);
408 h = qdisc_class_hash(cl->classid, clhash->hashmask);
409 hlist_add_head(&cl->hnode, &clhash->hash[h]);
410 clhash->hashelems++;
411}
412EXPORT_SYMBOL(qdisc_class_hash_insert);
413
414void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
415 struct Qdisc_class_common *cl)
416{
417 hlist_del(&cl->hnode);
418 clhash->hashelems--;
419}
420EXPORT_SYMBOL(qdisc_class_hash_remove);
421
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422/* Allocate an unique handle from space managed by kernel */
423
424static u32 qdisc_alloc_handle(struct net_device *dev)
425{
426 int i = 0x10000;
427 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
428
429 do {
430 autohandle += TC_H_MAKE(0x10000U, 0);
431 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
432 autohandle = TC_H_MAKE(0x80000000U, 0);
433 } while (qdisc_lookup(dev, autohandle) && --i > 0);
434
435 return i>0 ? autohandle : 0;
436}
437
438/* Attach toplevel qdisc to device dev */
439
440static struct Qdisc *
441dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc)
442{
David S. Millerb0e1e642008-07-08 17:42:10 -0700443 struct netdev_queue *dev_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 struct Qdisc *oqdisc;
445
446 if (dev->flags & IFF_UP)
447 dev_deactivate(dev);
448
449 qdisc_lock_tree(dev);
450 if (qdisc && qdisc->flags&TCQ_F_INGRESS) {
David S. Miller816f3252008-07-08 22:49:00 -0700451 dev_queue = &dev->rx_queue;
452 oqdisc = dev_queue->qdisc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 /* Prune old scheduler */
454 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) {
455 /* delete */
456 qdisc_reset(oqdisc);
David S. Miller816f3252008-07-08 22:49:00 -0700457 dev_queue->qdisc = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 } else { /* new */
David S. Miller816f3252008-07-08 22:49:00 -0700459 dev_queue->qdisc = qdisc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 }
461
462 } else {
David S. Millere8a04642008-07-17 00:34:19 -0700463 dev_queue = netdev_get_tx_queue(dev, 0);
David S. Millerb0e1e642008-07-08 17:42:10 -0700464 oqdisc = dev_queue->qdisc_sleeping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
466 /* Prune old scheduler */
467 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
468 qdisc_reset(oqdisc);
469
470 /* ... and graft new one */
471 if (qdisc == NULL)
472 qdisc = &noop_qdisc;
David S. Millerb0e1e642008-07-08 17:42:10 -0700473 dev_queue->qdisc_sleeping = qdisc;
474 dev_queue->qdisc = &noop_qdisc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 }
476
477 qdisc_unlock_tree(dev);
478
479 if (dev->flags & IFF_UP)
480 dev_activate(dev);
481
482 return oqdisc;
483}
484
Patrick McHardy43effa12006-11-29 17:35:48 -0800485void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n)
486{
Eric Dumazet20fea082007-11-14 01:44:41 -0800487 const struct Qdisc_class_ops *cops;
Patrick McHardy43effa12006-11-29 17:35:48 -0800488 unsigned long cl;
489 u32 parentid;
490
491 if (n == 0)
492 return;
493 while ((parentid = sch->parent)) {
Jarek Poplawski066a3b52008-04-14 15:10:42 -0700494 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
495 return;
496
David S. Miller5ce2d482008-07-08 17:06:30 -0700497 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700498 if (sch == NULL) {
499 WARN_ON(parentid != TC_H_ROOT);
500 return;
501 }
Patrick McHardy43effa12006-11-29 17:35:48 -0800502 cops = sch->ops->cl_ops;
503 if (cops->qlen_notify) {
504 cl = cops->get(sch, parentid);
505 cops->qlen_notify(sch, cl);
506 cops->put(sch, cl);
507 }
508 sch->q.qlen -= n;
509 }
510}
511EXPORT_SYMBOL(qdisc_tree_decrease_qlen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512
513/* Graft qdisc "new" to class "classid" of qdisc "parent" or
514 to device "dev".
515
516 Old qdisc is not destroyed but returned in *old.
517 */
518
519static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
520 u32 classid,
521 struct Qdisc *new, struct Qdisc **old)
522{
523 int err = 0;
524 struct Qdisc *q = *old;
525
526
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +0900527 if (parent == NULL) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 if (q && q->flags&TCQ_F_INGRESS) {
529 *old = dev_graft_qdisc(dev, q);
530 } else {
531 *old = dev_graft_qdisc(dev, new);
532 }
533 } else {
Eric Dumazet20fea082007-11-14 01:44:41 -0800534 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535
536 err = -EINVAL;
537
538 if (cops) {
539 unsigned long cl = cops->get(parent, classid);
540 if (cl) {
541 err = cops->graft(parent, cl, new, old);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 cops->put(parent, cl);
543 }
544 }
545 }
546 return err;
547}
548
549/*
550 Allocate and initialize new qdisc.
551
552 Parameters are passed via opt.
553 */
554
555static struct Qdisc *
David S. Millerbb949fb2008-07-08 16:55:56 -0700556qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
557 u32 parent, u32 handle, struct nlattr **tca, int *errp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558{
559 int err;
Patrick McHardy1e904742008-01-22 22:11:17 -0800560 struct nlattr *kind = tca[TCA_KIND];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 struct Qdisc *sch;
562 struct Qdisc_ops *ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
564 ops = qdisc_lookup_ops(kind);
565#ifdef CONFIG_KMOD
566 if (ops == NULL && kind != NULL) {
567 char name[IFNAMSIZ];
Patrick McHardy1e904742008-01-22 22:11:17 -0800568 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 /* We dropped the RTNL semaphore in order to
570 * perform the module load. So, even if we
571 * succeeded in loading the module we have to
572 * tell the caller to replay the request. We
573 * indicate this using -EAGAIN.
574 * We replay the request because the device may
575 * go away in the mean time.
576 */
577 rtnl_unlock();
578 request_module("sch_%s", name);
579 rtnl_lock();
580 ops = qdisc_lookup_ops(kind);
581 if (ops != NULL) {
582 /* We will try again qdisc_lookup_ops,
583 * so don't keep a reference.
584 */
585 module_put(ops->owner);
586 err = -EAGAIN;
587 goto err_out;
588 }
589 }
590 }
591#endif
592
Jamal Hadi Salimb9e2cc02006-08-03 16:36:51 -0700593 err = -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 if (ops == NULL)
595 goto err_out;
596
David S. Miller5ce2d482008-07-08 17:06:30 -0700597 sch = qdisc_alloc(dev_queue, ops);
Thomas Graf3d54b822005-07-05 14:15:09 -0700598 if (IS_ERR(sch)) {
599 err = PTR_ERR(sch);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 goto err_out2;
Thomas Graf3d54b822005-07-05 14:15:09 -0700601 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700603 sch->parent = parent;
604
Thomas Graf3d54b822005-07-05 14:15:09 -0700605 if (handle == TC_H_INGRESS) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 sch->flags |= TCQ_F_INGRESS;
Thomas Graf3d54b822005-07-05 14:15:09 -0700607 handle = TC_H_MAKE(TC_H_INGRESS, 0);
Patrick McHardyfd44de72007-04-16 17:07:08 -0700608 } else {
Patrick McHardyfd44de72007-04-16 17:07:08 -0700609 if (handle == 0) {
610 handle = qdisc_alloc_handle(dev);
611 err = -ENOMEM;
612 if (handle == 0)
613 goto err_out3;
614 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 }
616
Thomas Graf3d54b822005-07-05 14:15:09 -0700617 sch->handle = handle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
Patrick McHardy1e904742008-01-22 22:11:17 -0800619 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
620 if (tca[TCA_RATE]) {
Thomas Graf023e09a2005-07-05 14:15:53 -0700621 err = gen_new_estimator(&sch->bstats, &sch->rate_est,
David S. Miller7698b4f2008-07-16 01:42:40 -0700622 qdisc_root_lock(sch),
Patrick McHardy1e904742008-01-22 22:11:17 -0800623 tca[TCA_RATE]);
Thomas Graf023e09a2005-07-05 14:15:53 -0700624 if (err) {
625 /*
626 * Any broken qdiscs that would require
627 * a ops->reset() here? The qdisc was never
628 * in action so it shouldn't be necessary.
629 */
630 if (ops->destroy)
631 ops->destroy(sch);
632 goto err_out3;
633 }
634 }
David S. Milleread81cc2008-07-17 00:50:32 -0700635 spin_lock_bh(&dev->qdisc_list_lock);
636 list_add_tail(&sch->list, &dev->qdisc_list);
637 spin_unlock_bh(&dev->qdisc_list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 return sch;
640 }
641err_out3:
642 dev_put(dev);
Thomas Graf3d54b822005-07-05 14:15:09 -0700643 kfree((char *) sch - sch->padded);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644err_out2:
645 module_put(ops->owner);
646err_out:
647 *errp = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 return NULL;
649}
650
Patrick McHardy1e904742008-01-22 22:11:17 -0800651static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652{
Patrick McHardy1e904742008-01-22 22:11:17 -0800653 if (tca[TCA_OPTIONS]) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 int err;
655
656 if (sch->ops->change == NULL)
657 return -EINVAL;
Patrick McHardy1e904742008-01-22 22:11:17 -0800658 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 if (err)
660 return err;
661 }
Patrick McHardy1e904742008-01-22 22:11:17 -0800662 if (tca[TCA_RATE])
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 gen_replace_estimator(&sch->bstats, &sch->rate_est,
David S. Miller7698b4f2008-07-16 01:42:40 -0700664 qdisc_root_lock(sch), tca[TCA_RATE]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 return 0;
666}
667
668struct check_loop_arg
669{
670 struct qdisc_walker w;
671 struct Qdisc *p;
672 int depth;
673};
674
675static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
676
677static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
678{
679 struct check_loop_arg arg;
680
681 if (q->ops->cl_ops == NULL)
682 return 0;
683
684 arg.w.stop = arg.w.skip = arg.w.count = 0;
685 arg.w.fn = check_loop_fn;
686 arg.depth = depth;
687 arg.p = p;
688 q->ops->cl_ops->walk(q, &arg.w);
689 return arg.w.stop ? -ELOOP : 0;
690}
691
692static int
693check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
694{
695 struct Qdisc *leaf;
Eric Dumazet20fea082007-11-14 01:44:41 -0800696 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 struct check_loop_arg *arg = (struct check_loop_arg *)w;
698
699 leaf = cops->leaf(q, cl);
700 if (leaf) {
701 if (leaf == arg->p || arg->depth > 7)
702 return -ELOOP;
703 return check_loop(leaf, arg->p, arg->depth + 1);
704 }
705 return 0;
706}
707
708/*
709 * Delete/get qdisc.
710 */
711
712static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
713{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900714 struct net *net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 struct tcmsg *tcm = NLMSG_DATA(n);
Patrick McHardy1e904742008-01-22 22:11:17 -0800716 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 struct net_device *dev;
718 u32 clid = tcm->tcm_parent;
719 struct Qdisc *q = NULL;
720 struct Qdisc *p = NULL;
721 int err;
722
Denis V. Lunevb8542722007-12-01 00:21:31 +1100723 if (net != &init_net)
724 return -EINVAL;
725
Eric W. Biederman881d9662007-09-17 11:56:21 -0700726 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 return -ENODEV;
728
Patrick McHardy1e904742008-01-22 22:11:17 -0800729 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
730 if (err < 0)
731 return err;
732
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 if (clid) {
734 if (clid != TC_H_ROOT) {
735 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
736 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
737 return -ENOENT;
738 q = qdisc_leaf(p, clid);
739 } else { /* ingress */
David S. Miller816f3252008-07-08 22:49:00 -0700740 q = dev->rx_queue.qdisc;
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +0900741 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 } else {
David S. Millere8a04642008-07-17 00:34:19 -0700743 struct netdev_queue *dev_queue;
744 dev_queue = netdev_get_tx_queue(dev, 0);
David S. Millerb0e1e642008-07-08 17:42:10 -0700745 q = dev_queue->qdisc_sleeping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 }
747 if (!q)
748 return -ENOENT;
749
750 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
751 return -EINVAL;
752 } else {
753 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
754 return -ENOENT;
755 }
756
Patrick McHardy1e904742008-01-22 22:11:17 -0800757 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 return -EINVAL;
759
760 if (n->nlmsg_type == RTM_DELQDISC) {
761 if (!clid)
762 return -EINVAL;
763 if (q->handle == 0)
764 return -ENOENT;
765 if ((err = qdisc_graft(dev, p, clid, NULL, &q)) != 0)
766 return err;
767 if (q) {
768 qdisc_notify(skb, n, clid, q, NULL);
Patrick McHardyfd44de72007-04-16 17:07:08 -0700769 qdisc_lock_tree(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 qdisc_destroy(q);
Patrick McHardyfd44de72007-04-16 17:07:08 -0700771 qdisc_unlock_tree(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 }
773 } else {
774 qdisc_notify(skb, n, clid, NULL, q);
775 }
776 return 0;
777}
778
779/*
780 Create/change qdisc.
781 */
782
783static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
784{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +0900785 struct net *net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 struct tcmsg *tcm;
Patrick McHardy1e904742008-01-22 22:11:17 -0800787 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788 struct net_device *dev;
789 u32 clid;
790 struct Qdisc *q, *p;
791 int err;
792
Denis V. Lunevb8542722007-12-01 00:21:31 +1100793 if (net != &init_net)
794 return -EINVAL;
795
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796replay:
797 /* Reinit, just in case something touches this. */
798 tcm = NLMSG_DATA(n);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 clid = tcm->tcm_parent;
800 q = p = NULL;
801
Eric W. Biederman881d9662007-09-17 11:56:21 -0700802 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 return -ENODEV;
804
Patrick McHardy1e904742008-01-22 22:11:17 -0800805 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
806 if (err < 0)
807 return err;
808
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 if (clid) {
810 if (clid != TC_H_ROOT) {
811 if (clid != TC_H_INGRESS) {
812 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
813 return -ENOENT;
814 q = qdisc_leaf(p, clid);
815 } else { /*ingress */
David S. Miller816f3252008-07-08 22:49:00 -0700816 q = dev->rx_queue.qdisc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 }
818 } else {
David S. Millere8a04642008-07-17 00:34:19 -0700819 struct netdev_queue *dev_queue;
820 dev_queue = netdev_get_tx_queue(dev, 0);
David S. Millerb0e1e642008-07-08 17:42:10 -0700821 q = dev_queue->qdisc_sleeping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 }
823
824 /* It may be default qdisc, ignore it */
825 if (q && q->handle == 0)
826 q = NULL;
827
828 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
829 if (tcm->tcm_handle) {
830 if (q && !(n->nlmsg_flags&NLM_F_REPLACE))
831 return -EEXIST;
832 if (TC_H_MIN(tcm->tcm_handle))
833 return -EINVAL;
834 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
835 goto create_n_graft;
836 if (n->nlmsg_flags&NLM_F_EXCL)
837 return -EEXIST;
Patrick McHardy1e904742008-01-22 22:11:17 -0800838 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 return -EINVAL;
840 if (q == p ||
841 (p && check_loop(q, p, 0)))
842 return -ELOOP;
843 atomic_inc(&q->refcnt);
844 goto graft;
845 } else {
846 if (q == NULL)
847 goto create_n_graft;
848
849 /* This magic test requires explanation.
850 *
851 * We know, that some child q is already
852 * attached to this parent and have choice:
853 * either to change it or to create/graft new one.
854 *
855 * 1. We are allowed to create/graft only
856 * if CREATE and REPLACE flags are set.
857 *
858 * 2. If EXCL is set, requestor wanted to say,
859 * that qdisc tcm_handle is not expected
860 * to exist, so that we choose create/graft too.
861 *
862 * 3. The last case is when no flags are set.
863 * Alas, it is sort of hole in API, we
864 * cannot decide what to do unambiguously.
865 * For now we select create/graft, if
866 * user gave KIND, which does not match existing.
867 */
868 if ((n->nlmsg_flags&NLM_F_CREATE) &&
869 (n->nlmsg_flags&NLM_F_REPLACE) &&
870 ((n->nlmsg_flags&NLM_F_EXCL) ||
Patrick McHardy1e904742008-01-22 22:11:17 -0800871 (tca[TCA_KIND] &&
872 nla_strcmp(tca[TCA_KIND], q->ops->id))))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 goto create_n_graft;
874 }
875 }
876 } else {
877 if (!tcm->tcm_handle)
878 return -EINVAL;
879 q = qdisc_lookup(dev, tcm->tcm_handle);
880 }
881
882 /* Change qdisc parameters */
883 if (q == NULL)
884 return -ENOENT;
885 if (n->nlmsg_flags&NLM_F_EXCL)
886 return -EEXIST;
Patrick McHardy1e904742008-01-22 22:11:17 -0800887 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888 return -EINVAL;
889 err = qdisc_change(q, tca);
890 if (err == 0)
891 qdisc_notify(skb, n, clid, NULL, q);
892 return err;
893
894create_n_graft:
895 if (!(n->nlmsg_flags&NLM_F_CREATE))
896 return -ENOENT;
897 if (clid == TC_H_INGRESS)
David S. Millerbb949fb2008-07-08 16:55:56 -0700898 q = qdisc_create(dev, &dev->rx_queue,
899 tcm->tcm_parent, tcm->tcm_parent,
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700900 tca, &err);
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +0900901 else
David S. Millere8a04642008-07-17 00:34:19 -0700902 q = qdisc_create(dev, netdev_get_tx_queue(dev, 0),
David S. Millerbb949fb2008-07-08 16:55:56 -0700903 tcm->tcm_parent, tcm->tcm_handle,
Patrick McHardyffc8fef2007-07-30 17:11:50 -0700904 tca, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 if (q == NULL) {
906 if (err == -EAGAIN)
907 goto replay;
908 return err;
909 }
910
911graft:
912 if (1) {
913 struct Qdisc *old_q = NULL;
914 err = qdisc_graft(dev, p, clid, q, &old_q);
915 if (err) {
916 if (q) {
Patrick McHardyfd44de72007-04-16 17:07:08 -0700917 qdisc_lock_tree(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 qdisc_destroy(q);
Patrick McHardyfd44de72007-04-16 17:07:08 -0700919 qdisc_unlock_tree(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 }
921 return err;
922 }
923 qdisc_notify(skb, n, clid, old_q, q);
924 if (old_q) {
Patrick McHardyfd44de72007-04-16 17:07:08 -0700925 qdisc_lock_tree(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 qdisc_destroy(old_q);
Patrick McHardyfd44de72007-04-16 17:07:08 -0700927 qdisc_unlock_tree(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 }
929 }
930 return 0;
931}
932
933static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
Jamal Hadi Salime431b8c2005-06-18 22:55:31 -0700934 u32 pid, u32 seq, u16 flags, int event)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935{
936 struct tcmsg *tcm;
937 struct nlmsghdr *nlh;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700938 unsigned char *b = skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939 struct gnet_dump d;
940
Jamal Hadi Salime431b8c2005-06-18 22:55:31 -0700941 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 tcm = NLMSG_DATA(nlh);
943 tcm->tcm_family = AF_UNSPEC;
Patrick McHardy9ef1d4c2005-06-28 12:55:30 -0700944 tcm->tcm__pad1 = 0;
945 tcm->tcm__pad2 = 0;
David S. Miller5ce2d482008-07-08 17:06:30 -0700946 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 tcm->tcm_parent = clid;
948 tcm->tcm_handle = q->handle;
949 tcm->tcm_info = atomic_read(&q->refcnt);
Patrick McHardy57e1c482008-01-23 20:34:28 -0800950 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 if (q->ops->dump && q->ops->dump(q, skb) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -0800952 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953 q->qstats.qlen = q->q.qlen;
954
955 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS,
David S. Miller7698b4f2008-07-16 01:42:40 -0700956 TCA_XSTATS, qdisc_root_lock(q), &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -0800957 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958
959 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -0800960 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961
962 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 gnet_stats_copy_queue(&d, &q->qstats) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -0800965 goto nla_put_failure;
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +0900966
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967 if (gnet_stats_finish_copy(&d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -0800968 goto nla_put_failure;
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +0900969
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -0700970 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971 return skb->len;
972
973nlmsg_failure:
Patrick McHardy1e904742008-01-22 22:11:17 -0800974nla_put_failure:
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -0700975 nlmsg_trim(skb, b);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 return -1;
977}
978
979static int qdisc_notify(struct sk_buff *oskb, struct nlmsghdr *n,
980 u32 clid, struct Qdisc *old, struct Qdisc *new)
981{
982 struct sk_buff *skb;
983 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
984
985 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
986 if (!skb)
987 return -ENOBUFS;
988
989 if (old && old->handle) {
990 if (tc_fill_qdisc(skb, old, clid, pid, n->nlmsg_seq, 0, RTM_DELQDISC) < 0)
991 goto err_out;
992 }
993 if (new) {
994 if (tc_fill_qdisc(skb, new, clid, pid, n->nlmsg_seq, old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
995 goto err_out;
996 }
997
998 if (skb->len)
Denis V. Lunev97c53ca2007-11-19 22:26:51 -0800999 return rtnetlink_send(skb, &init_net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000
1001err_out:
1002 kfree_skb(skb);
1003 return -EINVAL;
1004}
1005
1006static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1007{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001008 struct net *net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 int idx, q_idx;
1010 int s_idx, s_q_idx;
1011 struct net_device *dev;
1012 struct Qdisc *q;
1013
Denis V. Lunevb8542722007-12-01 00:21:31 +11001014 if (net != &init_net)
1015 return 0;
1016
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 s_idx = cb->args[0];
1018 s_q_idx = q_idx = cb->args[1];
1019 read_lock(&dev_base_lock);
Pavel Emelianov7562f872007-05-03 15:13:45 -07001020 idx = 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001021 for_each_netdev(&init_net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022 if (idx < s_idx)
Pavel Emelianov7562f872007-05-03 15:13:45 -07001023 goto cont;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024 if (idx > s_idx)
1025 s_q_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 q_idx = 0;
David S. Milleread81cc2008-07-17 00:50:32 -07001027 list_for_each_entry(q, &dev->qdisc_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028 if (q_idx < s_q_idx) {
1029 q_idx++;
1030 continue;
1031 }
1032 if (tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
Patrick McHardy0463d4a2007-04-16 17:02:10 -07001033 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 q_idx++;
1036 }
Pavel Emelianov7562f872007-05-03 15:13:45 -07001037cont:
1038 idx++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 }
1040
1041done:
1042 read_unlock(&dev_base_lock);
1043
1044 cb->args[0] = idx;
1045 cb->args[1] = q_idx;
1046
1047 return skb->len;
1048}
1049
1050
1051
1052/************************************************
1053 * Traffic classes manipulation. *
1054 ************************************************/
1055
1056
1057
1058static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1059{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001060 struct net *net = sock_net(skb->sk);
David S. Millerb0e1e642008-07-08 17:42:10 -07001061 struct netdev_queue *dev_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 struct tcmsg *tcm = NLMSG_DATA(n);
Patrick McHardy1e904742008-01-22 22:11:17 -08001063 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064 struct net_device *dev;
1065 struct Qdisc *q = NULL;
Eric Dumazet20fea082007-11-14 01:44:41 -08001066 const struct Qdisc_class_ops *cops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 unsigned long cl = 0;
1068 unsigned long new_cl;
1069 u32 pid = tcm->tcm_parent;
1070 u32 clid = tcm->tcm_handle;
1071 u32 qid = TC_H_MAJ(clid);
1072 int err;
1073
Denis V. Lunevb8542722007-12-01 00:21:31 +11001074 if (net != &init_net)
1075 return -EINVAL;
1076
Eric W. Biederman881d9662007-09-17 11:56:21 -07001077 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078 return -ENODEV;
1079
Patrick McHardy1e904742008-01-22 22:11:17 -08001080 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1081 if (err < 0)
1082 return err;
1083
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 /*
1085 parent == TC_H_UNSPEC - unspecified parent.
1086 parent == TC_H_ROOT - class is root, which has no parent.
1087 parent == X:0 - parent is root class.
1088 parent == X:Y - parent is a node in hierarchy.
1089 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1090
1091 handle == 0:0 - generate handle from kernel pool.
1092 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1093 handle == X:Y - clear.
1094 handle == X:0 - root class.
1095 */
1096
1097 /* Step 1. Determine qdisc handle X:0 */
1098
David S. Millere8a04642008-07-17 00:34:19 -07001099 dev_queue = netdev_get_tx_queue(dev, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 if (pid != TC_H_ROOT) {
1101 u32 qid1 = TC_H_MAJ(pid);
1102
1103 if (qid && qid1) {
1104 /* If both majors are known, they must be identical. */
1105 if (qid != qid1)
1106 return -EINVAL;
1107 } else if (qid1) {
1108 qid = qid1;
1109 } else if (qid == 0)
David S. Millerb0e1e642008-07-08 17:42:10 -07001110 qid = dev_queue->qdisc_sleeping->handle;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111
1112 /* Now qid is genuine qdisc handle consistent
1113 both with parent and child.
1114
1115 TC_H_MAJ(pid) still may be unspecified, complete it now.
1116 */
1117 if (pid)
1118 pid = TC_H_MAKE(qid, pid);
1119 } else {
1120 if (qid == 0)
David S. Millerb0e1e642008-07-08 17:42:10 -07001121 qid = dev_queue->qdisc_sleeping->handle;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 }
1123
1124 /* OK. Locate qdisc */
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001125 if ((q = qdisc_lookup(dev, qid)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126 return -ENOENT;
1127
1128 /* An check that it supports classes */
1129 cops = q->ops->cl_ops;
1130 if (cops == NULL)
1131 return -EINVAL;
1132
1133 /* Now try to get class */
1134 if (clid == 0) {
1135 if (pid == TC_H_ROOT)
1136 clid = qid;
1137 } else
1138 clid = TC_H_MAKE(qid, clid);
1139
1140 if (clid)
1141 cl = cops->get(q, clid);
1142
1143 if (cl == 0) {
1144 err = -ENOENT;
1145 if (n->nlmsg_type != RTM_NEWTCLASS || !(n->nlmsg_flags&NLM_F_CREATE))
1146 goto out;
1147 } else {
1148 switch (n->nlmsg_type) {
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001149 case RTM_NEWTCLASS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 err = -EEXIST;
1151 if (n->nlmsg_flags&NLM_F_EXCL)
1152 goto out;
1153 break;
1154 case RTM_DELTCLASS:
1155 err = cops->delete(q, cl);
1156 if (err == 0)
1157 tclass_notify(skb, n, q, cl, RTM_DELTCLASS);
1158 goto out;
1159 case RTM_GETTCLASS:
1160 err = tclass_notify(skb, n, q, cl, RTM_NEWTCLASS);
1161 goto out;
1162 default:
1163 err = -EINVAL;
1164 goto out;
1165 }
1166 }
1167
1168 new_cl = cl;
1169 err = cops->change(q, clid, pid, tca, &new_cl);
1170 if (err == 0)
1171 tclass_notify(skb, n, q, new_cl, RTM_NEWTCLASS);
1172
1173out:
1174 if (cl)
1175 cops->put(q, cl);
1176
1177 return err;
1178}
1179
1180
1181static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1182 unsigned long cl,
Jamal Hadi Salime431b8c2005-06-18 22:55:31 -07001183 u32 pid, u32 seq, u16 flags, int event)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184{
1185 struct tcmsg *tcm;
1186 struct nlmsghdr *nlh;
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001187 unsigned char *b = skb_tail_pointer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188 struct gnet_dump d;
Eric Dumazet20fea082007-11-14 01:44:41 -08001189 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190
Jamal Hadi Salime431b8c2005-06-18 22:55:31 -07001191 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 tcm = NLMSG_DATA(nlh);
1193 tcm->tcm_family = AF_UNSPEC;
David S. Miller5ce2d482008-07-08 17:06:30 -07001194 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 tcm->tcm_parent = q->handle;
1196 tcm->tcm_handle = q->handle;
1197 tcm->tcm_info = 0;
Patrick McHardy57e1c482008-01-23 20:34:28 -08001198 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001200 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001201
1202 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS,
David S. Miller7698b4f2008-07-16 01:42:40 -07001203 TCA_XSTATS, qdisc_root_lock(q), &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001204 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205
1206 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001207 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208
1209 if (gnet_stats_finish_copy(&d) < 0)
Patrick McHardy1e904742008-01-22 22:11:17 -08001210 goto nla_put_failure;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211
Arnaldo Carvalho de Melo27a884d2007-04-19 20:29:13 -07001212 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 return skb->len;
1214
1215nlmsg_failure:
Patrick McHardy1e904742008-01-22 22:11:17 -08001216nla_put_failure:
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -07001217 nlmsg_trim(skb, b);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218 return -1;
1219}
1220
1221static int tclass_notify(struct sk_buff *oskb, struct nlmsghdr *n,
1222 struct Qdisc *q, unsigned long cl, int event)
1223{
1224 struct sk_buff *skb;
1225 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1226
1227 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1228 if (!skb)
1229 return -ENOBUFS;
1230
1231 if (tc_fill_tclass(skb, q, cl, pid, n->nlmsg_seq, 0, event) < 0) {
1232 kfree_skb(skb);
1233 return -EINVAL;
1234 }
1235
Denis V. Lunev97c53ca2007-11-19 22:26:51 -08001236 return rtnetlink_send(skb, &init_net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237}
1238
1239struct qdisc_dump_args
1240{
1241 struct qdisc_walker w;
1242 struct sk_buff *skb;
1243 struct netlink_callback *cb;
1244};
1245
1246static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1247{
1248 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1249
1250 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).pid,
1251 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1252}
1253
1254static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1255{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001256 struct net *net = sock_net(skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257 int t;
1258 int s_t;
1259 struct net_device *dev;
1260 struct Qdisc *q;
1261 struct tcmsg *tcm = (struct tcmsg*)NLMSG_DATA(cb->nlh);
1262 struct qdisc_dump_args arg;
1263
Denis V. Lunevb8542722007-12-01 00:21:31 +11001264 if (net != &init_net)
1265 return 0;
1266
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267 if (cb->nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*tcm)))
1268 return 0;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001269 if ((dev = dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 return 0;
1271
1272 s_t = cb->args[0];
1273 t = 0;
1274
David S. Milleread81cc2008-07-17 00:50:32 -07001275 list_for_each_entry(q, &dev->qdisc_list, list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 if (t < s_t || !q->ops->cl_ops ||
1277 (tcm->tcm_parent &&
1278 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1279 t++;
1280 continue;
1281 }
1282 if (t > s_t)
1283 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1284 arg.w.fn = qdisc_class_dump;
1285 arg.skb = skb;
1286 arg.cb = cb;
1287 arg.w.stop = 0;
1288 arg.w.skip = cb->args[1];
1289 arg.w.count = 0;
1290 q->ops->cl_ops->walk(q, &arg.w);
1291 cb->args[1] = arg.w.count;
1292 if (arg.w.stop)
1293 break;
1294 t++;
1295 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296
1297 cb->args[0] = t;
1298
1299 dev_put(dev);
1300 return skb->len;
1301}
1302
1303/* Main classifier routine: scans classifier chain attached
1304 to this qdisc, (optionally) tests for protocol and asks
1305 specific classifiers.
1306 */
Patrick McHardy73ca4912007-07-15 00:02:31 -07001307int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp,
1308 struct tcf_result *res)
1309{
1310 __be16 protocol = skb->protocol;
1311 int err = 0;
1312
1313 for (; tp; tp = tp->next) {
1314 if ((tp->protocol == protocol ||
1315 tp->protocol == htons(ETH_P_ALL)) &&
1316 (err = tp->classify(skb, tp, res)) >= 0) {
1317#ifdef CONFIG_NET_CLS_ACT
1318 if (err != TC_ACT_RECLASSIFY && skb->tc_verd)
1319 skb->tc_verd = SET_TC_VERD(skb->tc_verd, 0);
1320#endif
1321 return err;
1322 }
1323 }
1324 return -1;
1325}
1326EXPORT_SYMBOL(tc_classify_compat);
1327
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328int tc_classify(struct sk_buff *skb, struct tcf_proto *tp,
Patrick McHardy73ca4912007-07-15 00:02:31 -07001329 struct tcf_result *res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330{
1331 int err = 0;
Patrick McHardy73ca4912007-07-15 00:02:31 -07001332 __be16 protocol;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333#ifdef CONFIG_NET_CLS_ACT
1334 struct tcf_proto *otp = tp;
1335reclassify:
1336#endif
1337 protocol = skb->protocol;
1338
Patrick McHardy73ca4912007-07-15 00:02:31 -07001339 err = tc_classify_compat(skb, tp, res);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340#ifdef CONFIG_NET_CLS_ACT
Patrick McHardy73ca4912007-07-15 00:02:31 -07001341 if (err == TC_ACT_RECLASSIFY) {
1342 u32 verd = G_TC_VERD(skb->tc_verd);
1343 tp = otp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344
Patrick McHardy73ca4912007-07-15 00:02:31 -07001345 if (verd++ >= MAX_REC_LOOP) {
1346 printk("rule prio %u protocol %02x reclassify loop, "
1347 "packet dropped\n",
1348 tp->prio&0xffff, ntohs(tp->protocol));
1349 return TC_ACT_SHOT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 }
Patrick McHardy73ca4912007-07-15 00:02:31 -07001351 skb->tc_verd = SET_TC_VERD(skb->tc_verd, verd);
1352 goto reclassify;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353 }
Patrick McHardy73ca4912007-07-15 00:02:31 -07001354#endif
1355 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356}
Patrick McHardy73ca4912007-07-15 00:02:31 -07001357EXPORT_SYMBOL(tc_classify);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358
Patrick McHardya48b5a62007-03-23 11:29:43 -07001359void tcf_destroy(struct tcf_proto *tp)
1360{
1361 tp->ops->destroy(tp);
1362 module_put(tp->ops->owner);
1363 kfree(tp);
1364}
1365
Patrick McHardyff31ab52008-07-01 19:52:38 -07001366void tcf_destroy_chain(struct tcf_proto **fl)
Patrick McHardya48b5a62007-03-23 11:29:43 -07001367{
1368 struct tcf_proto *tp;
1369
Patrick McHardyff31ab52008-07-01 19:52:38 -07001370 while ((tp = *fl) != NULL) {
1371 *fl = tp->next;
Patrick McHardya48b5a62007-03-23 11:29:43 -07001372 tcf_destroy(tp);
1373 }
1374}
1375EXPORT_SYMBOL(tcf_destroy_chain);
1376
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377#ifdef CONFIG_PROC_FS
1378static int psched_show(struct seq_file *seq, void *v)
1379{
Patrick McHardy3c0cfc12007-10-10 16:32:41 -07001380 struct timespec ts;
1381
1382 hrtimer_get_res(CLOCK_MONOTONIC, &ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 seq_printf(seq, "%08x %08x %08x %08x\n",
Patrick McHardy641b9e02007-03-16 01:18:42 -07001384 (u32)NSEC_PER_USEC, (u32)PSCHED_US2NS(1),
Patrick McHardy514bca32007-03-16 12:34:52 -07001385 1000000,
Patrick McHardy3c0cfc12007-10-10 16:32:41 -07001386 (u32)NSEC_PER_SEC/(u32)ktime_to_ns(timespec_to_ktime(ts)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387
1388 return 0;
1389}
1390
1391static int psched_open(struct inode *inode, struct file *file)
1392{
1393 return single_open(file, psched_show, PDE(inode)->data);
1394}
1395
Arjan van de Venda7071d2007-02-12 00:55:36 -08001396static const struct file_operations psched_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 .owner = THIS_MODULE,
1398 .open = psched_open,
1399 .read = seq_read,
1400 .llseek = seq_lseek,
1401 .release = single_release,
YOSHIFUJI Hideaki10297b92007-02-09 23:25:16 +09001402};
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403#endif
1404
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405static int __init pktsched_init(void)
1406{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 register_qdisc(&pfifo_qdisc_ops);
1408 register_qdisc(&bfifo_qdisc_ops);
Eric W. Biederman457c4cb2007-09-12 12:01:34 +02001409 proc_net_fops_create(&init_net, "psched", 0, &psched_fops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410
Thomas Grafbe577dd2007-03-22 11:55:50 -07001411 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL);
1412 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL);
1413 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc);
1414 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL);
1415 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL);
1416 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass);
1417
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 return 0;
1419}
1420
1421subsys_initcall(pktsched_init);