net/sched/sch_api.c

   1 /*
   2  * net/sched/sch_api.c  Packet scheduler API.
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10  *
  11  * Fixes:
  12  *
  13  * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
  14  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  15  * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/types.h>
  20 #include <linux/kernel.h>
  21 #include <linux/string.h>
  22 #include <linux/errno.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/init.h>
  25 #include <linux/proc_fs.h>
  26 #include <linux/seq_file.h>
  27 #include <linux/kmod.h>
  28 #include <linux/list.h>
  29 #include <linux/hrtimer.h>
  30 #include <linux/lockdep.h>
  31 #include <linux/slab.h>
  32 #include <linux/hashtable.h>
  33
  34 #include <net/net_namespace.h>
  35 #include <net/sock.h>
  36 #include <net/netlink.h>
  37 #include <net/pkt_sched.h>
  38 #include <net/pkt_cls.h>
  39
  40 /*
  41
  42    Short review.
  43    -------------
  44
  45    This file consists of two interrelated parts:
  46
  47    1. queueing disciplines manager frontend.
  48    2. traffic classes manager frontend.
  49
  50    Generally, queueing discipline ("qdisc") is a black box,
  51    which is able to enqueue packets and to dequeue them (when
  52    device is ready to send something) in order and at times
  53    determined by algorithm hidden in it.
  54
  55    qdisc's are divided to two categories:
  56    - "queues", which have no internal structure visible from outside.
  57    - "schedulers", which split all the packets to "traffic classes",
  58      using "packet classifiers" (look at cls_api.c)
  59
  60    In turn, classes may have child qdiscs (as rule, queues)
  61    attached to them etc. etc. etc.
  62
  63    The goal of the routines in this file is to translate
  64    information supplied by user in the form of handles
  65    to more intelligible for kernel form, to make some sanity
  66    checks and part of work, which is common to all qdiscs
  67    and to provide rtnetlink notifications.
  68
  69    All real intelligent work is done inside qdisc modules.
  70
  71
  72
  73    Every discipline has two major routines: enqueue and dequeue.
  74
  75    ---dequeue
  76
  77    dequeue usually returns a skb to send. It is allowed to return NULL,
  78    but it does not mean that queue is empty, it just means that
  79    discipline does not want to send anything this time.
  80    Queue is really empty if q->q.qlen == 0.
  81    For complicated disciplines with multiple queues q->q is not
  82    real packet queue, but however q->q.qlen must be valid.
  83
  84    ---enqueue
  85
  86    enqueue returns 0, if packet was enqueued successfully.
  87    If packet (this one or another one) was dropped, it returns
  88    not zero error code.
  89    NET_XMIT_DROP        - this packet dropped
  90      Expected action: do not backoff, but wait until queue will clear.
  91    NET_XMIT_CN          - probably this packet enqueued, but another one dropped.
  92      Expected action: backoff or ignore
  93
  94    Auxiliary routines:
  95
  96    ---peek
  97
  98    like dequeue but without removing a packet from the queue
  99
 100    ---reset
 101
 102    returns qdisc to initial state: purge all buffers, clear all
 103    timers, counters (except for statistics) etc.
 104
 105    ---init
 106
 107    initializes newly created qdisc.
 108
 109    ---destroy
 110
 111    destroys resources allocated by init and during lifetime of qdisc.
 112
 113    ---change
 114
 115    changes qdisc parameters.
 116  */
 117
 118 /* Protects list of registered TC modules. It is pure SMP lock. */
 119 static DEFINE_RWLOCK(qdisc_mod_lock);
 120
 121
 122 /************************************************
 123  *      Queueing disciplines manipulation.      *
 124  ************************************************/
 125
 126
 127 /* The list of all installed queueing disciplines. */
 128
 129 static struct Qdisc_ops *qdisc_base;
 130
 131 /* Register/unregister queueing discipline */
 132
 133 int register_qdisc(struct Qdisc_ops *qops)
 134 {
 135         struct Qdisc_ops *q, **qp;
 136         int rc = -EEXIST;
 137
 138         write_lock(&qdisc_mod_lock);
 139         for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
 140                 if (!strcmp(qops->id, q->id))
 141                         goto out;
 142
 143         if (qops->enqueue == NULL)
 144                 qops->enqueue = noop_qdisc_ops.enqueue;
 145         if (qops->peek == NULL) {
 146                 if (qops->dequeue == NULL)
 147                         qops->peek = noop_qdisc_ops.peek;
 148                 else
 149                         goto out_einval;
 150         }
 151         if (qops->dequeue == NULL)
 152                 qops->dequeue = noop_qdisc_ops.dequeue;
 153
 154         if (qops->cl_ops) {
 155                 const struct Qdisc_class_ops *cops = qops->cl_ops;
 156
 157                 if (!(cops->find && cops->walk && cops->leaf))
 158                         goto out_einval;
 159
 160                 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf))
 161                         goto out_einval;
 162         }
 163
 164         qops->next = NULL;
 165         *qp = qops;
 166         rc = 0;
 167 out:
 168         write_unlock(&qdisc_mod_lock);
 169         return rc;
 170
 171 out_einval:
 172         rc = -EINVAL;
 173         goto out;
 174 }
 175 EXPORT_SYMBOL(register_qdisc);
 176
 177 int unregister_qdisc(struct Qdisc_ops *qops)
 178 {
 179         struct Qdisc_ops *q, **qp;
 180         int err = -ENOENT;
 181
 182         write_lock(&qdisc_mod_lock);
 183         for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
 184                 if (q == qops)
 185                         break;
 186         if (q) {
 187                 *qp = q->next;
 188                 q->next = NULL;
 189                 err = 0;
 190         }
 191         write_unlock(&qdisc_mod_lock);
 192         return err;
 193 }
 194 EXPORT_SYMBOL(unregister_qdisc);
 195
 196 /* Get default qdisc if not otherwise specified */
 197 void qdisc_get_default(char *name, size_t len)
 198 {
 199         read_lock(&qdisc_mod_lock);
 200         strlcpy(name, default_qdisc_ops->id, len);
 201         read_unlock(&qdisc_mod_lock);
 202 }
 203
 204 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
 205 {
 206         struct Qdisc_ops *q = NULL;
 207
 208         for (q = qdisc_base; q; q = q->next) {
 209                 if (!strcmp(name, q->id)) {
 210                         if (!try_module_get(q->owner))
 211                                 q = NULL;
 212                         break;
 213                 }
 214         }
 215
 216         return q;
 217 }
 218
 219 /* Set new default qdisc to use */
 220 int qdisc_set_default(const char *name)
 221 {
 222         const struct Qdisc_ops *ops;
 223
 224         if (!capable(CAP_NET_ADMIN))
 225                 return -EPERM;
 226
 227         write_lock(&qdisc_mod_lock);
 228         ops = qdisc_lookup_default(name);
 229         if (!ops) {
 230                 /* Not found, drop lock and try to load module */
 231                 write_unlock(&qdisc_mod_lock);
 232                 request_module("sch_%s", name);
 233                 write_lock(&qdisc_mod_lock);
 234
 235                 ops = qdisc_lookup_default(name);
 236         }
 237
 238         if (ops) {
 239                 /* Set new default */
 240                 module_put(default_qdisc_ops->owner);
 241                 default_qdisc_ops = ops;
 242         }
 243         write_unlock(&qdisc_mod_lock);
 244
 245         return ops ? 0 : -ENOENT;
 246 }
 247
 248 #ifdef CONFIG_NET_SCH_DEFAULT
 249 /* Set default value from kernel config */
 250 static int __init sch_default_qdisc(void)
 251 {
 252         return qdisc_set_default(CONFIG_DEFAULT_NET_SCH);
 253 }
 254 late_initcall(sch_default_qdisc);
 255 #endif
 256
 257 /* We know handle. Find qdisc among all qdisc's attached to device
 258  * (root qdisc, all its children, children of children etc.)
 259  * Note: caller either uses rtnl or rcu_read_lock()
 260  */
 261
 262 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
 263 {
 264         struct Qdisc *q;
 265
 266         if (!qdisc_dev(root))
 267                 return (root->handle == handle ? root : NULL);
 268
 269         if (!(root->flags & TCQ_F_BUILTIN) &&
 270             root->handle == handle)
 271                 return root;
 272
 273         hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle) {
 274                 if (q->handle == handle)
 275                         return q;
 276         }
 277         return NULL;
 278 }
 279
 280 void qdisc_hash_add(struct Qdisc *q, bool invisible)
 281 {
 282         if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
 283                 ASSERT_RTNL();
 284                 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
 285                 if (invisible)
 286                         q->flags |= TCQ_F_INVISIBLE;
 287         }
 288 }
 289 EXPORT_SYMBOL(qdisc_hash_add);
 290
 291 void qdisc_hash_del(struct Qdisc *q)
 292 {
 293         if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
 294                 ASSERT_RTNL();
 295                 hash_del_rcu(&q->hash);
 296         }
 297 }
 298 EXPORT_SYMBOL(qdisc_hash_del);
 299
 300 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
 301 {
 302         struct Qdisc *q;
 303
 304         if (!handle)
 305                 return NULL;
 306         q = qdisc_match_from_root(dev->qdisc, handle);
 307         if (q)
 308                 goto out;
 309
 310         if (dev_ingress_queue(dev))
 311                 q = qdisc_match_from_root(
 312                         dev_ingress_queue(dev)->qdisc_sleeping,
 313                         handle);
 314 out:
 315         return q;
 316 }
 317
 318 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
 319 {
 320         unsigned long cl;
 321         struct Qdisc *leaf;
 322         const struct Qdisc_class_ops *cops = p->ops->cl_ops;
 323
 324         if (cops == NULL)
 325                 return NULL;
 326         cl = cops->find(p, classid);
 327
 328         if (cl == 0)
 329                 return NULL;
 330         leaf = cops->leaf(p, cl);
 331         return leaf;
 332 }
 333
 334 /* Find queueing discipline by name */
 335
 336 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
 337 {
 338         struct Qdisc_ops *q = NULL;
 339
 340         if (kind) {
 341                 read_lock(&qdisc_mod_lock);
 342                 for (q = qdisc_base; q; q = q->next) {
 343                         if (nla_strcmp(kind, q->id) == 0) {
 344                                 if (!try_module_get(q->owner))
 345                                         q = NULL;
 346                                 break;
 347                         }
 348                 }
 349                 read_unlock(&qdisc_mod_lock);
 350         }
 351         return q;
 352 }
 353
 354 /* The linklayer setting were not transferred from iproute2, in older
 355  * versions, and the rate tables lookup systems have been dropped in
 356  * the kernel. To keep backward compatible with older iproute2 tc
 357  * utils, we detect the linklayer setting by detecting if the rate
 358  * table were modified.
 359  *
 360  * For linklayer ATM table entries, the rate table will be aligned to
 361  * 48 bytes, thus some table entries will contain the same value.  The
 362  * mpu (min packet unit) is also encoded into the old rate table, thus
 363  * starting from the mpu, we find low and high table entries for
 364  * mapping this cell.  If these entries contain the same value, when
 365  * the rate tables have been modified for linklayer ATM.
 366  *
 367  * This is done by rounding mpu to the nearest 48 bytes cell/entry,
 368  * and then roundup to the next cell, calc the table entry one below,
 369  * and compare.
 370  */
 371 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
 372 {
 373         int low       = roundup(r->mpu, 48);
 374         int high      = roundup(low+1, 48);
 375         int cell_low  = low >> r->cell_log;
 376         int cell_high = (high >> r->cell_log) - 1;
 377
 378         /* rtab is too inaccurate at rates > 100Mbit/s */
 379         if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
 380                 pr_debug("TC linklayer: Giving up ATM detection\n");
 381                 return TC_LINKLAYER_ETHERNET;
 382         }
 383
 384         if ((cell_high > cell_low) && (cell_high < 256)
 385             && (rtab[cell_low] == rtab[cell_high])) {
 386                 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
 387                          cell_low, cell_high, rtab[cell_high]);
 388                 return TC_LINKLAYER_ATM;
 389         }
 390         return TC_LINKLAYER_ETHERNET;
 391 }
 392
 393 static struct qdisc_rate_table *qdisc_rtab_list;
 394
 395 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
 396                                         struct nlattr *tab)
 397 {
 398         struct qdisc_rate_table *rtab;
 399
 400         if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
 401             nla_len(tab) != TC_RTAB_SIZE)
 402                 return NULL;
 403
 404         for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
 405                 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
 406                     !memcmp(&rtab->data, nla_data(tab), 1024)) {
 407                         rtab->refcnt++;
 408                         return rtab;
 409                 }
 410         }
 411
 412         rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
 413         if (rtab) {
 414                 rtab->rate = *r;
 415                 rtab->refcnt = 1;
 416                 memcpy(rtab->data, nla_data(tab), 1024);
 417                 if (r->linklayer == TC_LINKLAYER_UNAWARE)
 418                         r->linklayer = __detect_linklayer(r, rtab->data);
 419                 rtab->next = qdisc_rtab_list;
 420                 qdisc_rtab_list = rtab;
 421         }
 422         return rtab;
 423 }
 424 EXPORT_SYMBOL(qdisc_get_rtab);
 425
 426 void qdisc_put_rtab(struct qdisc_rate_table *tab)
 427 {
 428         struct qdisc_rate_table *rtab, **rtabp;
 429
 430         if (!tab || --tab->refcnt)
 431                 return;
 432
 433         for (rtabp = &qdisc_rtab_list;
 434              (rtab = *rtabp) != NULL;
 435              rtabp = &rtab->next) {
 436                 if (rtab == tab) {
 437                         *rtabp = rtab->next;
 438                         kfree(rtab);
 439                         return;
 440                 }
 441         }
 442 }
 443 EXPORT_SYMBOL(qdisc_put_rtab);
 444
 445 static LIST_HEAD(qdisc_stab_list);
 446
 447 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
 448         [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
 449         [TCA_STAB_DATA] = { .type = NLA_BINARY },
 450 };
 451
 452 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt)
 453 {
 454         struct nlattr *tb[TCA_STAB_MAX + 1];
 455         struct qdisc_size_table *stab;
 456         struct tc_sizespec *s;
 457         unsigned int tsize = 0;
 458         u16 *tab = NULL;
 459         int err;
 460
 461         err = nla_parse_nested(tb, TCA_STAB_MAX, opt, stab_policy, NULL);
 462         if (err < 0)
 463                 return ERR_PTR(err);
 464         if (!tb[TCA_STAB_BASE])
 465                 return ERR_PTR(-EINVAL);
 466
 467         s = nla_data(tb[TCA_STAB_BASE]);
 468
 469         if (s->tsize > 0) {
 470                 if (!tb[TCA_STAB_DATA])
 471                         return ERR_PTR(-EINVAL);
 472                 tab = nla_data(tb[TCA_STAB_DATA]);
 473                 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
 474         }
 475
 476         if (tsize != s->tsize || (!tab && tsize > 0))
 477                 return ERR_PTR(-EINVAL);
 478
 479         list_for_each_entry(stab, &qdisc_stab_list, list) {
 480                 if (memcmp(&stab->szopts, s, sizeof(*s)))
 481                         continue;
 482                 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
 483                         continue;
 484                 stab->refcnt++;
 485                 return stab;
 486         }
 487
 488         stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
 489         if (!stab)
 490                 return ERR_PTR(-ENOMEM);
 491
 492         stab->refcnt = 1;
 493         stab->szopts = *s;
 494         if (tsize > 0)
 495                 memcpy(stab->data, tab, tsize * sizeof(u16));
 496
 497         list_add_tail(&stab->list, &qdisc_stab_list);
 498
 499         return stab;
 500 }
 501
 502 static void stab_kfree_rcu(struct rcu_head *head)
 503 {
 504         kfree(container_of(head, struct qdisc_size_table, rcu));
 505 }
 506
 507 void qdisc_put_stab(struct qdisc_size_table *tab)
 508 {
 509         if (!tab)
 510                 return;
 511
 512         if (--tab->refcnt == 0) {
 513                 list_del(&tab->list);
 514                 call_rcu_bh(&tab->rcu, stab_kfree_rcu);
 515         }
 516 }
 517 EXPORT_SYMBOL(qdisc_put_stab);
 518
 519 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
 520 {
 521         struct nlattr *nest;
 522
 523         nest = nla_nest_start(skb, TCA_STAB);
 524         if (nest == NULL)
 525                 goto nla_put_failure;
 526         if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
 527                 goto nla_put_failure;
 528         nla_nest_end(skb, nest);
 529
 530         return skb->len;
 531
 532 nla_put_failure:
 533         return -1;
 534 }
 535
 536 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
 537                                const struct qdisc_size_table *stab)
 538 {
 539         int pkt_len, slot;
 540
 541         pkt_len = skb->len + stab->szopts.overhead;
 542         if (unlikely(!stab->szopts.tsize))
 543                 goto out;
 544
 545         slot = pkt_len + stab->szopts.cell_align;
 546         if (unlikely(slot < 0))
 547                 slot = 0;
 548
 549         slot >>= stab->szopts.cell_log;
 550         if (likely(slot < stab->szopts.tsize))
 551                 pkt_len = stab->data[slot];
 552         else
 553                 pkt_len = stab->data[stab->szopts.tsize - 1] *
 554                                 (slot / stab->szopts.tsize) +
 555                                 stab->data[slot % stab->szopts.tsize];
 556
 557         pkt_len <<= stab->szopts.size_log;
 558 out:
 559         if (unlikely(pkt_len < 1))
 560                 pkt_len = 1;
 561         qdisc_skb_cb(skb)->pkt_len = pkt_len;
 562 }
 563 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
 564
 565 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
 566 {
 567         if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
 568                 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
 569                         txt, qdisc->ops->id, qdisc->handle >> 16);
 570                 qdisc->flags |= TCQ_F_WARN_NONWC;
 571         }
 572 }
 573 EXPORT_SYMBOL(qdisc_warn_nonwc);
 574
 575 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
 576 {
 577         struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
 578                                                  timer);
 579
 580         rcu_read_lock();
 581         __netif_schedule(qdisc_root(wd->qdisc));
 582         rcu_read_unlock();
 583
 584         return HRTIMER_NORESTART;
 585 }
 586
 587 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
 588 {
 589         hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
 590         wd->timer.function = qdisc_watchdog;
 591         wd->qdisc = qdisc;
 592 }
 593 EXPORT_SYMBOL(qdisc_watchdog_init);
 594
 595 void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires)
 596 {
 597         if (test_bit(__QDISC_STATE_DEACTIVATED,
 598                      &qdisc_root_sleeping(wd->qdisc)->state))
 599                 return;
 600
 601         if (wd->last_expires == expires)
 602                 return;
 603
 604         wd->last_expires = expires;
 605         hrtimer_start(&wd->timer,
 606                       ns_to_ktime(expires),
 607                       HRTIMER_MODE_ABS_PINNED);
 608 }
 609 EXPORT_SYMBOL(qdisc_watchdog_schedule_ns);
 610
 611 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
 612 {
 613         hrtimer_cancel(&wd->timer);
 614 }
 615 EXPORT_SYMBOL(qdisc_watchdog_cancel);
 616
 617 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
 618 {
 619         struct hlist_head *h;
 620         unsigned int i;
 621
 622         h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL);
 623
 624         if (h != NULL) {
 625                 for (i = 0; i < n; i++)
 626                         INIT_HLIST_HEAD(&h[i]);
 627         }
 628         return h;
 629 }
 630
 631 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
 632 {
 633         struct Qdisc_class_common *cl;
 634         struct hlist_node *next;
 635         struct hlist_head *nhash, *ohash;
 636         unsigned int nsize, nmask, osize;
 637         unsigned int i, h;
 638
 639         /* Rehash when load factor exceeds 0.75 */
 640         if (clhash->hashelems * 4 <= clhash->hashsize * 3)
 641                 return;
 642         nsize = clhash->hashsize * 2;
 643         nmask = nsize - 1;
 644         nhash = qdisc_class_hash_alloc(nsize);
 645         if (nhash == NULL)
 646                 return;
 647
 648         ohash = clhash->hash;
 649         osize = clhash->hashsize;
 650
 651         sch_tree_lock(sch);
 652         for (i = 0; i < osize; i++) {
 653                 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
 654                         h = qdisc_class_hash(cl->classid, nmask);
 655                         hlist_add_head(&cl->hnode, &nhash[h]);
 656                 }
 657         }
 658         clhash->hash     = nhash;
 659         clhash->hashsize = nsize;
 660         clhash->hashmask = nmask;
 661         sch_tree_unlock(sch);
 662
 663         kvfree(ohash);
 664 }
 665 EXPORT_SYMBOL(qdisc_class_hash_grow);
 666
 667 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
 668 {
 669         unsigned int size = 4;
 670
 671         clhash->hash = qdisc_class_hash_alloc(size);
 672         if (clhash->hash == NULL)
 673                 return -ENOMEM;
 674         clhash->hashsize  = size;
 675         clhash->hashmask  = size - 1;
 676         clhash->hashelems = 0;
 677         return 0;
 678 }
 679 EXPORT_SYMBOL(qdisc_class_hash_init);
 680
 681 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
 682 {
 683         kvfree(clhash->hash);
 684 }
 685 EXPORT_SYMBOL(qdisc_class_hash_destroy);
 686
 687 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
 688                              struct Qdisc_class_common *cl)
 689 {
 690         unsigned int h;
 691
 692         INIT_HLIST_NODE(&cl->hnode);
 693         h = qdisc_class_hash(cl->classid, clhash->hashmask);
 694         hlist_add_head(&cl->hnode, &clhash->hash[h]);
 695         clhash->hashelems++;
 696 }
 697 EXPORT_SYMBOL(qdisc_class_hash_insert);
 698
 699 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
 700                              struct Qdisc_class_common *cl)
 701 {
 702         hlist_del(&cl->hnode);
 703         clhash->hashelems--;
 704 }
 705 EXPORT_SYMBOL(qdisc_class_hash_remove);
 706
 707 /* Allocate an unique handle from space managed by kernel
 708  * Possible range is [8000-FFFF]:0000 (0x8000 values)
 709  */
 710 static u32 qdisc_alloc_handle(struct net_device *dev)
 711 {
 712         int i = 0x8000;
 713         static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
 714
 715         do {
 716                 autohandle += TC_H_MAKE(0x10000U, 0);
 717                 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
 718                         autohandle = TC_H_MAKE(0x80000000U, 0);
 719                 if (!qdisc_lookup(dev, autohandle))
 720                         return autohandle;
 721                 cond_resched();
 722         } while (--i > 0);
 723
 724         return 0;
 725 }
 726
 727 void qdisc_tree_reduce_backlog(struct Qdisc *sch, unsigned int n,
 728                                unsigned int len)
 729 {
 730         const struct Qdisc_class_ops *cops;
 731         unsigned long cl;
 732         u32 parentid;
 733         bool notify;
 734         int drops;
 735
 736         if (n == 0 && len == 0)
 737                 return;
 738         drops = max_t(int, n, 0);
 739         rcu_read_lock();
 740         while ((parentid = sch->parent)) {
 741                 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
 742                         break;
 743
 744                 if (sch->flags & TCQ_F_NOPARENT)
 745                         break;
 746                 /* Notify parent qdisc only if child qdisc becomes empty.
 747                  *
 748                  * If child was empty even before update then backlog
 749                  * counter is screwed and we skip notification because
 750                  * parent class is already passive.
 751                  */
 752                 notify = !sch->q.qlen && !WARN_ON_ONCE(!n);
 753                 /* TODO: perform the search on a per txq basis */
 754                 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
 755                 if (sch == NULL) {
 756                         WARN_ON_ONCE(parentid != TC_H_ROOT);
 757                         break;
 758                 }
 759                 cops = sch->ops->cl_ops;
 760                 if (notify && cops->qlen_notify) {
 761                         cl = cops->find(sch, parentid);
 762                         cops->qlen_notify(sch, cl);
 763                 }
 764                 sch->q.qlen -= n;
 765                 sch->qstats.backlog -= len;
 766                 __qdisc_qstats_drop(sch, drops);
 767         }
 768         rcu_read_unlock();
 769 }
 770 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
 771
 772 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
 773                          u32 portid, u32 seq, u16 flags, int event)
 774 {
 775         struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
 776         struct gnet_stats_queue __percpu *cpu_qstats = NULL;
 777         struct tcmsg *tcm;
 778         struct nlmsghdr  *nlh;
 779         unsigned char *b = skb_tail_pointer(skb);
 780         struct gnet_dump d;
 781         struct qdisc_size_table *stab;
 782         __u32 qlen;
 783
 784         cond_resched();
 785         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
 786         if (!nlh)
 787                 goto out_nlmsg_trim;
 788         tcm = nlmsg_data(nlh);
 789         tcm->tcm_family = AF_UNSPEC;
 790         tcm->tcm__pad1 = 0;
 791         tcm->tcm__pad2 = 0;
 792         tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
 793         tcm->tcm_parent = clid;
 794         tcm->tcm_handle = q->handle;
 795         tcm->tcm_info = refcount_read(&q->refcnt);
 796         if (nla_put_string(skb, TCA_KIND, q->ops->id))
 797                 goto nla_put_failure;
 798         if (q->ops->dump && q->ops->dump(q, skb) < 0)
 799                 goto nla_put_failure;
 800         qlen = q->q.qlen;
 801
 802         stab = rtnl_dereference(q->stab);
 803         if (stab && qdisc_dump_stab(skb, stab) < 0)
 804                 goto nla_put_failure;
 805
 806         if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
 807                                          NULL, &d, TCA_PAD) < 0)
 808                 goto nla_put_failure;
 809
 810         if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
 811                 goto nla_put_failure;
 812
 813         if (qdisc_is_percpu_stats(q)) {
 814                 cpu_bstats = q->cpu_bstats;
 815                 cpu_qstats = q->cpu_qstats;
 816         }
 817
 818         if (gnet_stats_copy_basic(qdisc_root_sleeping_running(q),
 819                                   &d, cpu_bstats, &q->bstats) < 0 ||
 820             gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
 821             gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
 822                 goto nla_put_failure;
 823
 824         if (gnet_stats_finish_copy(&d) < 0)
 825                 goto nla_put_failure;
 826
 827         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
 828         return skb->len;
 829
 830 out_nlmsg_trim:
 831 nla_put_failure:
 832         nlmsg_trim(skb, b);
 833         return -1;
 834 }
 835
 836 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
 837 {
 838         if (q->flags & TCQ_F_BUILTIN)
 839                 return true;
 840         if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
 841                 return true;
 842
 843         return false;
 844 }
 845
 846 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
 847                         struct nlmsghdr *n, u32 clid,
 848                         struct Qdisc *old, struct Qdisc *new)
 849 {
 850         struct sk_buff *skb;
 851         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
 852
 853         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
 854         if (!skb)
 855                 return -ENOBUFS;
 856
 857         if (old && !tc_qdisc_dump_ignore(old, false)) {
 858                 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
 859                                   0, RTM_DELQDISC) < 0)
 860                         goto err_out;
 861         }
 862         if (new && !tc_qdisc_dump_ignore(new, false)) {
 863                 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
 864                                   old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
 865                         goto err_out;
 866         }
 867
 868         if (skb->len)
 869                 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
 870                                       n->nlmsg_flags & NLM_F_ECHO);
 871
 872 err_out:
 873         kfree_skb(skb);
 874         return -EINVAL;
 875 }
 876
 877 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
 878                                struct nlmsghdr *n, u32 clid,
 879                                struct Qdisc *old, struct Qdisc *new)
 880 {
 881         if (new || old)
 882                 qdisc_notify(net, skb, n, clid, old, new);
 883
 884         if (old)
 885                 qdisc_destroy(old);
 886 }
 887
 888 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
 889  * to device "dev".
 890  *
 891  * When appropriate send a netlink notification using 'skb'
 892  * and "n".
 893  *
 894  * On success, destroy old qdisc.
 895  */
 896
 897 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
 898                        struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
 899                        struct Qdisc *new, struct Qdisc *old)
 900 {
 901         struct Qdisc *q = old;
 902         struct net *net = dev_net(dev);
 903         int err = 0;
 904
 905         if (parent == NULL) {
 906                 unsigned int i, num_q, ingress;
 907
 908                 ingress = 0;
 909                 num_q = dev->num_tx_queues;
 910                 if ((q && q->flags & TCQ_F_INGRESS) ||
 911                     (new && new->flags & TCQ_F_INGRESS)) {
 912                         num_q = 1;
 913                         ingress = 1;
 914                         if (!dev_ingress_queue(dev))
 915                                 return -ENOENT;
 916                 }
 917
 918                 if (dev->flags & IFF_UP)
 919                         dev_deactivate(dev);
 920
 921                 if (new && new->ops->attach)
 922                         goto skip;
 923
 924                 for (i = 0; i < num_q; i++) {
 925                         struct netdev_queue *dev_queue = dev_ingress_queue(dev);
 926
 927                         if (!ingress)
 928                                 dev_queue = netdev_get_tx_queue(dev, i);
 929
 930                         old = dev_graft_qdisc(dev_queue, new);
 931                         if (new && i > 0)
 932                                 qdisc_refcount_inc(new);
 933
 934                         if (!ingress)
 935                                 qdisc_destroy(old);
 936                 }
 937
 938 skip:
 939                 if (!ingress) {
 940                         notify_and_destroy(net, skb, n, classid,
 941                                            dev->qdisc, new);
 942                         if (new && !new->ops->attach)
 943                                 qdisc_refcount_inc(new);
 944                         dev->qdisc = new ? : &noop_qdisc;
 945
 946                         if (new && new->ops->attach)
 947                                 new->ops->attach(new);
 948                 } else {
 949                         notify_and_destroy(net, skb, n, classid, old, new);
 950                 }
 951
 952                 if (dev->flags & IFF_UP)
 953                         dev_activate(dev);
 954         } else {
 955                 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
 956
 957                 err = -EOPNOTSUPP;
 958                 if (cops && cops->graft) {
 959                         unsigned long cl = cops->find(parent, classid);
 960
 961                         if (cl)
 962                                 err = cops->graft(parent, cl, new, &old);
 963                         else
 964                                 err = -ENOENT;
 965                 }
 966                 if (!err)
 967                         notify_and_destroy(net, skb, n, classid, old, new);
 968         }
 969         return err;
 970 }
 971
 972 /* lockdep annotation is needed for ingress; egress gets it only for name */
 973 static struct lock_class_key qdisc_tx_lock;
 974 static struct lock_class_key qdisc_rx_lock;
 975
 976 /*
 977    Allocate and initialize new qdisc.
 978
 979    Parameters are passed via opt.
 980  */
 981
 982 static struct Qdisc *qdisc_create(struct net_device *dev,
 983                                   struct netdev_queue *dev_queue,
 984                                   struct Qdisc *p, u32 parent, u32 handle,
 985                                   struct nlattr **tca, int *errp)
 986 {
 987         int err;
 988         struct nlattr *kind = tca[TCA_KIND];
 989         struct Qdisc *sch;
 990         struct Qdisc_ops *ops;
 991         struct qdisc_size_table *stab;
 992
 993         ops = qdisc_lookup_ops(kind);
 994 #ifdef CONFIG_MODULES
 995         if (ops == NULL && kind != NULL) {
 996                 char name[IFNAMSIZ];
 997                 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
 998                         /* We dropped the RTNL semaphore in order to
 999                          * perform the module load.  So, even if we
1000                          * succeeded in loading the module we have to
1001                          * tell the caller to replay the request.  We
1002                          * indicate this using -EAGAIN.
1003                          * We replay the request because the device may
1004                          * go away in the mean time.
1005                          */
1006                         rtnl_unlock();
1007                         request_module("sch_%s", name);
1008                         rtnl_lock();
1009                         ops = qdisc_lookup_ops(kind);
1010                         if (ops != NULL) {
1011                                 /* We will try again qdisc_lookup_ops,
1012                                  * so don't keep a reference.
1013                                  */
1014                                 module_put(ops->owner);
1015                                 err = -EAGAIN;
1016                                 goto err_out;
1017                         }
1018                 }
1019         }
1020 #endif
1021
1022         err = -ENOENT;
1023         if (ops == NULL)
1024                 goto err_out;
1025
1026         sch = qdisc_alloc(dev_queue, ops);
1027         if (IS_ERR(sch)) {
1028                 err = PTR_ERR(sch);
1029                 goto err_out2;
1030         }
1031
1032         sch->parent = parent;
1033
1034         if (handle == TC_H_INGRESS) {
1035                 sch->flags |= TCQ_F_INGRESS;
1036                 handle = TC_H_MAKE(TC_H_INGRESS, 0);
1037                 lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock);
1038         } else {
1039                 if (handle == 0) {
1040                         handle = qdisc_alloc_handle(dev);
1041                         err = -ENOMEM;
1042                         if (handle == 0)
1043                                 goto err_out3;
1044                 }
1045                 lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock);
1046                 if (!netif_is_multiqueue(dev))
1047                         sch->flags |= TCQ_F_ONETXQUEUE;
1048         }
1049
1050         sch->handle = handle;
1051
1052         /* This exist to keep backward compatible with a userspace
1053          * loophole, what allowed userspace to get IFF_NO_QUEUE
1054          * facility on older kernels by setting tx_queue_len=0 (prior
1055          * to qdisc init), and then forgot to reinit tx_queue_len
1056          * before again attaching a qdisc.
1057          */
1058         if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) {
1059                 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
1060                 netdev_info(dev, "Caught tx_queue_len zero misconfig\n");
1061         }
1062
1063         if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
1064                 if (qdisc_is_percpu_stats(sch)) {
1065                         sch->cpu_bstats =
1066                                 netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
1067                         if (!sch->cpu_bstats)
1068                                 goto err_out4;
1069
1070                         sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
1071                         if (!sch->cpu_qstats)
1072                                 goto err_out4;
1073                 }
1074
1075                 if (tca[TCA_STAB]) {
1076                         stab = qdisc_get_stab(tca[TCA_STAB]);
1077                         if (IS_ERR(stab)) {
1078                                 err = PTR_ERR(stab);
1079                                 goto err_out4;
1080                         }
1081                         rcu_assign_pointer(sch->stab, stab);
1082                 }
1083                 if (tca[TCA_RATE]) {
1084                         seqcount_t *running;
1085
1086                         err = -EOPNOTSUPP;
1087                         if (sch->flags & TCQ_F_MQROOT)
1088                                 goto err_out4;
1089
1090                         if ((sch->parent != TC_H_ROOT) &&
1091                             !(sch->flags & TCQ_F_INGRESS) &&
1092                             (!p || !(p->flags & TCQ_F_MQROOT)))
1093                                 running = qdisc_root_sleeping_running(sch);
1094                         else
1095                                 running = &sch->running;
1096
1097                         err = gen_new_estimator(&sch->bstats,
1098                                                 sch->cpu_bstats,
1099                                                 &sch->rate_est,
1100                                                 NULL,
1101                                                 running,
1102                                                 tca[TCA_RATE]);
1103                         if (err)
1104                                 goto err_out4;
1105                 }
1106
1107                 qdisc_hash_add(sch, false);
1108
1109                 return sch;
1110         }
1111         /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1112         if (ops->destroy)
1113                 ops->destroy(sch);
1114 err_out3:
1115         dev_put(dev);
1116         kfree((char *) sch - sch->padded);
1117 err_out2:
1118         module_put(ops->owner);
1119 err_out:
1120         *errp = err;
1121         return NULL;
1122
1123 err_out4:
1124         free_percpu(sch->cpu_bstats);
1125         free_percpu(sch->cpu_qstats);
1126         /*
1127          * Any broken qdiscs that would require a ops->reset() here?
1128          * The qdisc was never in action so it shouldn't be necessary.
1129          */
1130         qdisc_put_stab(rtnl_dereference(sch->stab));
1131         if (ops->destroy)
1132                 ops->destroy(sch);
1133         goto err_out3;
1134 }
1135
1136 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
1137 {
1138         struct qdisc_size_table *ostab, *stab = NULL;
1139         int err = 0;
1140
1141         if (tca[TCA_OPTIONS]) {
1142                 if (sch->ops->change == NULL)
1143                         return -EINVAL;
1144                 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
1145                 if (err)
1146                         return err;
1147         }
1148
1149         if (tca[TCA_STAB]) {
1150                 stab = qdisc_get_stab(tca[TCA_STAB]);
1151                 if (IS_ERR(stab))
1152                         return PTR_ERR(stab);
1153         }
1154
1155         ostab = rtnl_dereference(sch->stab);
1156         rcu_assign_pointer(sch->stab, stab);
1157         qdisc_put_stab(ostab);
1158
1159         if (tca[TCA_RATE]) {
1160                 /* NB: ignores errors from replace_estimator
1161                    because change can't be undone. */
1162                 if (sch->flags & TCQ_F_MQROOT)
1163                         goto out;
1164                 gen_replace_estimator(&sch->bstats,
1165                                       sch->cpu_bstats,
1166                                       &sch->rate_est,
1167                                       NULL,
1168                                       qdisc_root_sleeping_running(sch),
1169                                       tca[TCA_RATE]);
1170         }
1171 out:
1172         return 0;
1173 }
1174
1175 struct check_loop_arg {
1176         struct qdisc_walker     w;
1177         struct Qdisc            *p;
1178         int                     depth;
1179 };
1180
1181 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1182                          struct qdisc_walker *w);
1183
1184 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1185 {
1186         struct check_loop_arg   arg;
1187
1188         if (q->ops->cl_ops == NULL)
1189                 return 0;
1190
1191         arg.w.stop = arg.w.skip = arg.w.count = 0;
1192         arg.w.fn = check_loop_fn;
1193         arg.depth = depth;
1194         arg.p = p;
1195         q->ops->cl_ops->walk(q, &arg.w);
1196         return arg.w.stop ? -ELOOP : 0;
1197 }
1198
1199 static int
1200 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1201 {
1202         struct Qdisc *leaf;
1203         const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1204         struct check_loop_arg *arg = (struct check_loop_arg *)w;
1205
1206         leaf = cops->leaf(q, cl);
1207         if (leaf) {
1208                 if (leaf == arg->p || arg->depth > 7)
1209                         return -ELOOP;
1210                 return check_loop(leaf, arg->p, arg->depth + 1);
1211         }
1212         return 0;
1213 }
1214
1215 /*
1216  * Delete/get qdisc.
1217  */
1218
1219 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1220                         struct netlink_ext_ack *extack)
1221 {
1222         struct net *net = sock_net(skb->sk);
1223         struct tcmsg *tcm = nlmsg_data(n);
1224         struct nlattr *tca[TCA_MAX + 1];
1225         struct net_device *dev;
1226         u32 clid;
1227         struct Qdisc *q = NULL;
1228         struct Qdisc *p = NULL;
1229         int err;
1230
1231         if ((n->nlmsg_type != RTM_GETQDISC) &&
1232             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1233                 return -EPERM;
1234
1235         err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL, extack);
1236         if (err < 0)
1237                 return err;
1238
1239         dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1240         if (!dev)
1241                 return -ENODEV;
1242
1243         clid = tcm->tcm_parent;
1244         if (clid) {
1245                 if (clid != TC_H_ROOT) {
1246                         if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1247                                 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1248                                 if (!p)
1249                                         return -ENOENT;
1250                                 q = qdisc_leaf(p, clid);
1251                         } else if (dev_ingress_queue(dev)) {
1252                                 q = dev_ingress_queue(dev)->qdisc_sleeping;
1253                         }
1254                 } else {
1255                         q = dev->qdisc;
1256                 }
1257                 if (!q)
1258                         return -ENOENT;
1259
1260                 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
1261                         return -EINVAL;
1262         } else {
1263                 q = qdisc_lookup(dev, tcm->tcm_handle);
1264                 if (!q)
1265                         return -ENOENT;
1266         }
1267
1268         if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1269                 return -EINVAL;
1270
1271         if (n->nlmsg_type == RTM_DELQDISC) {
1272                 if (!clid)
1273                         return -EINVAL;
1274                 if (q->handle == 0)
1275                         return -ENOENT;
1276                 err = qdisc_graft(dev, p, skb, n, clid, NULL, q);
1277                 if (err != 0)
1278                         return err;
1279         } else {
1280                 qdisc_notify(net, skb, n, clid, NULL, q);
1281         }
1282         return 0;
1283 }
1284
1285 /*
1286  * Create/change qdisc.
1287  */
1288
1289 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1290                            struct netlink_ext_ack *extack)
1291 {
1292         struct net *net = sock_net(skb->sk);
1293         struct tcmsg *tcm;
1294         struct nlattr *tca[TCA_MAX + 1];
1295         struct net_device *dev;
1296         u32 clid;
1297         struct Qdisc *q, *p;
1298         int err;
1299
1300         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1301                 return -EPERM;
1302
1303 replay:
1304         /* Reinit, just in case something touches this. */
1305         err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL, extack);
1306         if (err < 0)
1307                 return err;
1308
1309         tcm = nlmsg_data(n);
1310         clid = tcm->tcm_parent;
1311         q = p = NULL;
1312
1313         dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1314         if (!dev)
1315                 return -ENODEV;
1316
1317
1318         if (clid) {
1319                 if (clid != TC_H_ROOT) {
1320                         if (clid != TC_H_INGRESS) {
1321                                 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1322                                 if (!p)
1323                                         return -ENOENT;
1324                                 q = qdisc_leaf(p, clid);
1325                         } else if (dev_ingress_queue_create(dev)) {
1326                                 q = dev_ingress_queue(dev)->qdisc_sleeping;
1327                         }
1328                 } else {
1329                         q = dev->qdisc;
1330                 }
1331
1332                 /* It may be default qdisc, ignore it */
1333                 if (q && q->handle == 0)
1334                         q = NULL;
1335
1336                 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1337                         if (tcm->tcm_handle) {
1338                                 if (q && !(n->nlmsg_flags & NLM_F_REPLACE))
1339                                         return -EEXIST;
1340                                 if (TC_H_MIN(tcm->tcm_handle))
1341                                         return -EINVAL;
1342                                 q = qdisc_lookup(dev, tcm->tcm_handle);
1343                                 if (!q)
1344                                         goto create_n_graft;
1345                                 if (n->nlmsg_flags & NLM_F_EXCL)
1346                                         return -EEXIST;
1347                                 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1348                                         return -EINVAL;
1349                                 if (q == p ||
1350                                     (p && check_loop(q, p, 0)))
1351                                         return -ELOOP;
1352                                 qdisc_refcount_inc(q);
1353                                 goto graft;
1354                         } else {
1355                                 if (!q)
1356                                         goto create_n_graft;
1357
1358                                 /* This magic test requires explanation.
1359                                  *
1360                                  *   We know, that some child q is already
1361                                  *   attached to this parent and have choice:
1362                                  *   either to change it or to create/graft new one.
1363                                  *
1364                                  *   1. We are allowed to create/graft only
1365                                  *   if CREATE and REPLACE flags are set.
1366                                  *
1367                                  *   2. If EXCL is set, requestor wanted to say,
1368                                  *   that qdisc tcm_handle is not expected
1369                                  *   to exist, so that we choose create/graft too.
1370                                  *
1371                                  *   3. The last case is when no flags are set.
1372                                  *   Alas, it is sort of hole in API, we
1373                                  *   cannot decide what to do unambiguously.
1374                                  *   For now we select create/graft, if
1375                                  *   user gave KIND, which does not match existing.
1376                                  */
1377                                 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1378                                     (n->nlmsg_flags & NLM_F_REPLACE) &&
1379                                     ((n->nlmsg_flags & NLM_F_EXCL) ||
1380                                      (tca[TCA_KIND] &&
1381                                       nla_strcmp(tca[TCA_KIND], q->ops->id))))
1382                                         goto create_n_graft;
1383                         }
1384                 }
1385         } else {
1386                 if (!tcm->tcm_handle)
1387                         return -EINVAL;
1388                 q = qdisc_lookup(dev, tcm->tcm_handle);
1389         }
1390
1391         /* Change qdisc parameters */
1392         if (q == NULL)
1393                 return -ENOENT;
1394         if (n->nlmsg_flags & NLM_F_EXCL)
1395                 return -EEXIST;
1396         if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1397                 return -EINVAL;
1398         err = qdisc_change(q, tca);
1399         if (err == 0)
1400                 qdisc_notify(net, skb, n, clid, NULL, q);
1401         return err;
1402
1403 create_n_graft:
1404         if (!(n->nlmsg_flags & NLM_F_CREATE))
1405                 return -ENOENT;
1406         if (clid == TC_H_INGRESS) {
1407                 if (dev_ingress_queue(dev))
1408                         q = qdisc_create(dev, dev_ingress_queue(dev), p,
1409                                          tcm->tcm_parent, tcm->tcm_parent,
1410                                          tca, &err);
1411                 else
1412                         err = -ENOENT;
1413         } else {
1414                 struct netdev_queue *dev_queue;
1415
1416                 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1417                         dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1418                 else if (p)
1419                         dev_queue = p->dev_queue;
1420                 else
1421                         dev_queue = netdev_get_tx_queue(dev, 0);
1422
1423                 q = qdisc_create(dev, dev_queue, p,
1424                                  tcm->tcm_parent, tcm->tcm_handle,
1425                                  tca, &err);
1426         }
1427         if (q == NULL) {
1428                 if (err == -EAGAIN)
1429                         goto replay;
1430                 return err;
1431         }
1432
1433 graft:
1434         err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
1435         if (err) {
1436                 if (q)
1437                         qdisc_destroy(q);
1438                 return err;
1439         }
1440
1441         return 0;
1442 }
1443
1444 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1445                               struct netlink_callback *cb,
1446                               int *q_idx_p, int s_q_idx, bool recur,
1447                               bool dump_invisible)
1448 {
1449         int ret = 0, q_idx = *q_idx_p;
1450         struct Qdisc *q;
1451         int b;
1452
1453         if (!root)
1454                 return 0;
1455
1456         q = root;
1457         if (q_idx < s_q_idx) {
1458                 q_idx++;
1459         } else {
1460                 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1461                     tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1462                                   cb->nlh->nlmsg_seq, NLM_F_MULTI,
1463                                   RTM_NEWQDISC) <= 0)
1464                         goto done;
1465                 q_idx++;
1466         }
1467
1468         /* If dumping singletons, there is no qdisc_dev(root) and the singleton
1469          * itself has already been dumped.
1470          *
1471          * If we've already dumped the top-level (ingress) qdisc above and the global
1472          * qdisc hashtable, we don't want to hit it again
1473          */
1474         if (!qdisc_dev(root) || !recur)
1475                 goto out;
1476
1477         hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1478                 if (q_idx < s_q_idx) {
1479                         q_idx++;
1480                         continue;
1481                 }
1482                 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1483                     tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1484                                   cb->nlh->nlmsg_seq, NLM_F_MULTI,
1485                                   RTM_NEWQDISC) <= 0)
1486                         goto done;
1487                 q_idx++;
1488         }
1489
1490 out:
1491         *q_idx_p = q_idx;
1492         return ret;
1493 done:
1494         ret = -1;
1495         goto out;
1496 }
1497
1498 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1499 {
1500         struct net *net = sock_net(skb->sk);
1501         int idx, q_idx;
1502         int s_idx, s_q_idx;
1503         struct net_device *dev;
1504         const struct nlmsghdr *nlh = cb->nlh;
1505         struct nlattr *tca[TCA_MAX + 1];
1506         int err;
1507
1508         s_idx = cb->args[0];
1509         s_q_idx = q_idx = cb->args[1];
1510
1511         idx = 0;
1512         ASSERT_RTNL();
1513
1514         err = nlmsg_parse(nlh, sizeof(struct tcmsg), tca, TCA_MAX, NULL, NULL);
1515         if (err < 0)
1516                 return err;
1517
1518         for_each_netdev(net, dev) {
1519                 struct netdev_queue *dev_queue;
1520
1521                 if (idx < s_idx)
1522                         goto cont;
1523                 if (idx > s_idx)
1524                         s_q_idx = 0;
1525                 q_idx = 0;
1526
1527                 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx,
1528                                        true, tca[TCA_DUMP_INVISIBLE]) < 0)
1529                         goto done;
1530
1531                 dev_queue = dev_ingress_queue(dev);
1532                 if (dev_queue &&
1533                     tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1534                                        &q_idx, s_q_idx, false,
1535                                        tca[TCA_DUMP_INVISIBLE]) < 0)
1536                         goto done;
1537
1538 cont:
1539                 idx++;
1540         }
1541
1542 done:
1543         cb->args[0] = idx;
1544         cb->args[1] = q_idx;
1545
1546         return skb->len;
1547 }
1548
1549
1550
1551 /************************************************
1552  *      Traffic classes manipulation.           *
1553  ************************************************/
1554
1555 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1556                           unsigned long cl,
1557                           u32 portid, u32 seq, u16 flags, int event)
1558 {
1559         struct tcmsg *tcm;
1560         struct nlmsghdr  *nlh;
1561         unsigned char *b = skb_tail_pointer(skb);
1562         struct gnet_dump d;
1563         const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1564
1565         cond_resched();
1566         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1567         if (!nlh)
1568                 goto out_nlmsg_trim;
1569         tcm = nlmsg_data(nlh);
1570         tcm->tcm_family = AF_UNSPEC;
1571         tcm->tcm__pad1 = 0;
1572         tcm->tcm__pad2 = 0;
1573         tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1574         tcm->tcm_parent = q->handle;
1575         tcm->tcm_handle = q->handle;
1576         tcm->tcm_info = 0;
1577         if (nla_put_string(skb, TCA_KIND, q->ops->id))
1578                 goto nla_put_failure;
1579         if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1580                 goto nla_put_failure;
1581
1582         if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1583                                          NULL, &d, TCA_PAD) < 0)
1584                 goto nla_put_failure;
1585
1586         if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1587                 goto nla_put_failure;
1588
1589         if (gnet_stats_finish_copy(&d) < 0)
1590                 goto nla_put_failure;
1591
1592         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1593         return skb->len;
1594
1595 out_nlmsg_trim:
1596 nla_put_failure:
1597         nlmsg_trim(skb, b);
1598         return -1;
1599 }
1600
1601 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1602                          struct nlmsghdr *n, struct Qdisc *q,
1603                          unsigned long cl, int event)
1604 {
1605         struct sk_buff *skb;
1606         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1607
1608         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1609         if (!skb)
1610                 return -ENOBUFS;
1611
1612         if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1613                 kfree_skb(skb);
1614                 return -EINVAL;
1615         }
1616
1617         return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1618                               n->nlmsg_flags & NLM_F_ECHO);
1619 }
1620
1621 static int tclass_del_notify(struct net *net,
1622                              const struct Qdisc_class_ops *cops,
1623                              struct sk_buff *oskb, struct nlmsghdr *n,
1624                              struct Qdisc *q, unsigned long cl)
1625 {
1626         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1627         struct sk_buff *skb;
1628         int err = 0;
1629
1630         if (!cops->delete)
1631                 return -EOPNOTSUPP;
1632
1633         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1634         if (!skb)
1635                 return -ENOBUFS;
1636
1637         if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
1638                            RTM_DELTCLASS) < 0) {
1639                 kfree_skb(skb);
1640                 return -EINVAL;
1641         }
1642
1643         err = cops->delete(q, cl);
1644         if (err) {
1645                 kfree_skb(skb);
1646                 return err;
1647         }
1648
1649         return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1650                               n->nlmsg_flags & NLM_F_ECHO);
1651 }
1652
1653 #ifdef CONFIG_NET_CLS
1654
1655 struct tcf_bind_args {
1656         struct tcf_walker w;
1657         u32 classid;
1658         unsigned long cl;
1659 };
1660
1661 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1662 {
1663         struct tcf_bind_args *a = (void *)arg;
1664
1665         if (tp->ops->bind_class) {
1666                 struct Qdisc *q = tcf_block_q(tp->chain->block);
1667
1668                 sch_tree_lock(q);
1669                 tp->ops->bind_class(n, a->classid, a->cl);
1670                 sch_tree_unlock(q);
1671         }
1672         return 0;
1673 }
1674
1675 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1676                            unsigned long new_cl)
1677 {
1678         const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1679         struct tcf_block *block;
1680         struct tcf_chain *chain;
1681         unsigned long cl;
1682
1683         cl = cops->find(q, portid);
1684         if (!cl)
1685                 return;
1686         block = cops->tcf_block(q, cl);
1687         if (!block)
1688                 return;
1689         list_for_each_entry(chain, &block->chain_list, list) {
1690                 struct tcf_proto *tp;
1691
1692                 for (tp = rtnl_dereference(chain->filter_chain);
1693                      tp; tp = rtnl_dereference(tp->next)) {
1694                         struct tcf_bind_args arg = {};
1695
1696                         arg.w.fn = tcf_node_bind;
1697                         arg.classid = clid;
1698                         arg.cl = new_cl;
1699                         tp->ops->walk(tp, &arg.w);
1700                 }
1701         }
1702 }
1703
1704 #else
1705
1706 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1707                            unsigned long new_cl)
1708 {
1709 }
1710
1711 #endif
1712
1713 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
1714                          struct netlink_ext_ack *extack)
1715 {
1716         struct net *net = sock_net(skb->sk);
1717         struct tcmsg *tcm = nlmsg_data(n);
1718         struct nlattr *tca[TCA_MAX + 1];
1719         struct net_device *dev;
1720         struct Qdisc *q = NULL;
1721         const struct Qdisc_class_ops *cops;
1722         unsigned long cl = 0;
1723         unsigned long new_cl;
1724         u32 portid;
1725         u32 clid;
1726         u32 qid;
1727         int err;
1728
1729         if ((n->nlmsg_type != RTM_GETTCLASS) &&
1730             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1731                 return -EPERM;
1732
1733         err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL, extack);
1734         if (err < 0)
1735                 return err;
1736
1737         dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1738         if (!dev)
1739                 return -ENODEV;
1740
1741         /*
1742            parent == TC_H_UNSPEC - unspecified parent.
1743            parent == TC_H_ROOT   - class is root, which has no parent.
1744            parent == X:0         - parent is root class.
1745            parent == X:Y         - parent is a node in hierarchy.
1746            parent == 0:Y         - parent is X:Y, where X:0 is qdisc.
1747
1748            handle == 0:0         - generate handle from kernel pool.
1749            handle == 0:Y         - class is X:Y, where X:0 is qdisc.
1750            handle == X:Y         - clear.
1751            handle == X:0         - root class.
1752          */
1753
1754         /* Step 1. Determine qdisc handle X:0 */
1755
1756         portid = tcm->tcm_parent;
1757         clid = tcm->tcm_handle;
1758         qid = TC_H_MAJ(clid);
1759
1760         if (portid != TC_H_ROOT) {
1761                 u32 qid1 = TC_H_MAJ(portid);
1762
1763                 if (qid && qid1) {
1764                         /* If both majors are known, they must be identical. */
1765                         if (qid != qid1)
1766                                 return -EINVAL;
1767                 } else if (qid1) {
1768                         qid = qid1;
1769                 } else if (qid == 0)
1770                         qid = dev->qdisc->handle;
1771
1772                 /* Now qid is genuine qdisc handle consistent
1773                  * both with parent and child.
1774                  *
1775                  * TC_H_MAJ(portid) still may be unspecified, complete it now.
1776                  */
1777                 if (portid)
1778                         portid = TC_H_MAKE(qid, portid);
1779         } else {
1780                 if (qid == 0)
1781                         qid = dev->qdisc->handle;
1782         }
1783
1784         /* OK. Locate qdisc */
1785         q = qdisc_lookup(dev, qid);
1786         if (!q)
1787                 return -ENOENT;
1788
1789         /* An check that it supports classes */
1790         cops = q->ops->cl_ops;
1791         if (cops == NULL)
1792                 return -EINVAL;
1793
1794         /* Now try to get class */
1795         if (clid == 0) {
1796                 if (portid == TC_H_ROOT)
1797                         clid = qid;
1798         } else
1799                 clid = TC_H_MAKE(qid, clid);
1800
1801         if (clid)
1802                 cl = cops->find(q, clid);
1803
1804         if (cl == 0) {
1805                 err = -ENOENT;
1806                 if (n->nlmsg_type != RTM_NEWTCLASS ||
1807                     !(n->nlmsg_flags & NLM_F_CREATE))
1808                         goto out;
1809         } else {
1810                 switch (n->nlmsg_type) {
1811                 case RTM_NEWTCLASS:
1812                         err = -EEXIST;
1813                         if (n->nlmsg_flags & NLM_F_EXCL)
1814                                 goto out;
1815                         break;
1816                 case RTM_DELTCLASS:
1817                         err = tclass_del_notify(net, cops, skb, n, q, cl);
1818                         /* Unbind the class with flilters with 0 */
1819                         tc_bind_tclass(q, portid, clid, 0);
1820                         goto out;
1821                 case RTM_GETTCLASS:
1822                         err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
1823                         goto out;
1824                 default:
1825                         err = -EINVAL;
1826                         goto out;
1827                 }
1828         }
1829
1830         new_cl = cl;
1831         err = -EOPNOTSUPP;
1832         if (cops->change)
1833                 err = cops->change(q, clid, portid, tca, &new_cl);
1834         if (err == 0) {
1835                 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
1836                 /* We just create a new class, need to do reverse binding. */
1837                 if (cl != new_cl)
1838                         tc_bind_tclass(q, portid, clid, new_cl);
1839         }
1840 out:
1841         return err;
1842 }
1843
1844 struct qdisc_dump_args {
1845         struct qdisc_walker     w;
1846         struct sk_buff          *skb;
1847         struct netlink_callback *cb;
1848 };
1849
1850 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,
1851                             struct qdisc_walker *arg)
1852 {
1853         struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1854
1855         return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
1856                               a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1857                               RTM_NEWTCLASS);
1858 }
1859
1860 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
1861                                 struct tcmsg *tcm, struct netlink_callback *cb,
1862                                 int *t_p, int s_t)
1863 {
1864         struct qdisc_dump_args arg;
1865
1866         if (tc_qdisc_dump_ignore(q, false) ||
1867             *t_p < s_t || !q->ops->cl_ops ||
1868             (tcm->tcm_parent &&
1869              TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1870                 (*t_p)++;
1871                 return 0;
1872         }
1873         if (*t_p > s_t)
1874                 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1875         arg.w.fn = qdisc_class_dump;
1876         arg.skb = skb;
1877         arg.cb = cb;
1878         arg.w.stop  = 0;
1879         arg.w.skip = cb->args[1];
1880         arg.w.count = 0;
1881         q->ops->cl_ops->walk(q, &arg.w);
1882         cb->args[1] = arg.w.count;
1883         if (arg.w.stop)
1884                 return -1;
1885         (*t_p)++;
1886         return 0;
1887 }
1888
1889 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
1890                                struct tcmsg *tcm, struct netlink_callback *cb,
1891                                int *t_p, int s_t)
1892 {
1893         struct Qdisc *q;
1894         int b;
1895
1896         if (!root)
1897                 return 0;
1898
1899         if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
1900                 return -1;
1901
1902         if (!qdisc_dev(root))
1903                 return 0;
1904
1905         if (tcm->tcm_parent) {
1906                 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));
1907                 if (q && tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
1908                         return -1;
1909                 return 0;
1910         }
1911         hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1912                 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
1913                         return -1;
1914         }
1915
1916         return 0;
1917 }
1918
1919 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1920 {
1921         struct tcmsg *tcm = nlmsg_data(cb->nlh);
1922         struct net *net = sock_net(skb->sk);
1923         struct netdev_queue *dev_queue;
1924         struct net_device *dev;
1925         int t, s_t;
1926
1927         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
1928                 return 0;
1929         dev = dev_get_by_index(net, tcm->tcm_ifindex);
1930         if (!dev)
1931                 return 0;
1932
1933         s_t = cb->args[0];
1934         t = 0;
1935
1936         if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0)
1937                 goto done;
1938
1939         dev_queue = dev_ingress_queue(dev);
1940         if (dev_queue &&
1941             tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
1942                                 &t, s_t) < 0)
1943                 goto done;
1944
1945 done:
1946         cb->args[0] = t;
1947
1948         dev_put(dev);
1949         return skb->len;
1950 }
1951
1952 #ifdef CONFIG_PROC_FS
1953 static int psched_show(struct seq_file *seq, void *v)
1954 {
1955         seq_printf(seq, "%08x %08x %08x %08x\n",
1956                    (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
1957                    1000000,
1958                    (u32)NSEC_PER_SEC / hrtimer_resolution);
1959
1960         return 0;
1961 }
1962
1963 static int psched_open(struct inode *inode, struct file *file)
1964 {
1965         return single_open(file, psched_show, NULL);
1966 }
1967
1968 static const struct file_operations psched_fops = {
1969         .owner = THIS_MODULE,
1970         .open = psched_open,
1971         .read  = seq_read,
1972         .llseek = seq_lseek,
1973         .release = single_release,
1974 };
1975
1976 static int __net_init psched_net_init(struct net *net)
1977 {
1978         struct proc_dir_entry *e;
1979
1980         e = proc_create("psched", 0, net->proc_net, &psched_fops);
1981         if (e == NULL)
1982                 return -ENOMEM;
1983
1984         return 0;
1985 }
1986
1987 static void __net_exit psched_net_exit(struct net *net)
1988 {
1989         remove_proc_entry("psched", net->proc_net);
1990 }
1991 #else
1992 static int __net_init psched_net_init(struct net *net)
1993 {
1994         return 0;
1995 }
1996
1997 static void __net_exit psched_net_exit(struct net *net)
1998 {
1999 }
2000 #endif
2001
2002 static struct pernet_operations psched_net_ops = {
2003         .init = psched_net_init,
2004         .exit = psched_net_exit,
2005 };
2006
2007 static int __init pktsched_init(void)
2008 {
2009         int err;
2010
2011         err = register_pernet_subsys(&psched_net_ops);
2012         if (err) {
2013                 pr_err("pktsched_init: "
2014                        "cannot initialize per netns operations\n");
2015                 return err;
2016         }
2017
2018         register_qdisc(&pfifo_fast_ops);
2019         register_qdisc(&pfifo_qdisc_ops);
2020         register_qdisc(&bfifo_qdisc_ops);
2021         register_qdisc(&pfifo_head_drop_qdisc_ops);
2022         register_qdisc(&mq_qdisc_ops);
2023         register_qdisc(&noqueue_qdisc_ops);
2024
2025         rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0);
2026         rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0);
2027         rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc,
2028                       0);
2029         rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0);
2030         rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0);
2031         rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
2032                       0);
2033
2034         return 0;
2035 }
2036
2037 subsys_initcall(pktsched_init);