1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/sch_api.c Packet scheduler API. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 * Fixes: 8 * 9 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired. 10 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 11 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support 12 */ 13 14 #include <linux/module.h> 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/string.h> 18 #include <linux/errno.h> 19 #include <linux/skbuff.h> 20 #include <linux/init.h> 21 #include <linux/proc_fs.h> 22 #include <linux/seq_file.h> 23 #include <linux/kmod.h> 24 #include <linux/list.h> 25 #include <linux/hrtimer.h> 26 #include <linux/slab.h> 27 #include <linux/hashtable.h> 28 29 #include <net/net_namespace.h> 30 #include <net/sock.h> 31 #include <net/netlink.h> 32 #include <net/pkt_sched.h> 33 #include <net/pkt_cls.h> 34 #include <net/tc_wrapper.h> 35 36 #include <trace/events/qdisc.h> 37 38 /* 39 40 Short review. 41 ------------- 42 43 This file consists of two interrelated parts: 44 45 1. queueing disciplines manager frontend. 46 2. traffic classes manager frontend. 47 48 Generally, queueing discipline ("qdisc") is a black box, 49 which is able to enqueue packets and to dequeue them (when 50 device is ready to send something) in order and at times 51 determined by algorithm hidden in it. 52 53 qdisc's are divided to two categories: 54 - "queues", which have no internal structure visible from outside. 55 - "schedulers", which split all the packets to "traffic classes", 56 using "packet classifiers" (look at cls_api.c) 57 58 In turn, classes may have child qdiscs (as rule, queues) 59 attached to them etc. etc. etc. 60 61 The goal of the routines in this file is to translate 62 information supplied by user in the form of handles 63 to more intelligible for kernel form, to make some sanity 64 checks and part of work, which is common to all qdiscs 65 and to provide rtnetlink notifications. 66 67 All real intelligent work is done inside qdisc modules. 68 69 70 71 Every discipline has two major routines: enqueue and dequeue. 72 73 ---dequeue 74 75 dequeue usually returns a skb to send. It is allowed to return NULL, 76 but it does not mean that queue is empty, it just means that 77 discipline does not want to send anything this time. 78 Queue is really empty if q->q.qlen == 0. 79 For complicated disciplines with multiple queues q->q is not 80 real packet queue, but however q->q.qlen must be valid. 81 82 ---enqueue 83 84 enqueue returns 0, if packet was enqueued successfully. 85 If packet (this one or another one) was dropped, it returns 86 not zero error code. 87 NET_XMIT_DROP - this packet dropped 88 Expected action: do not backoff, but wait until queue will clear. 89 NET_XMIT_CN - probably this packet enqueued, but another one dropped. 90 Expected action: backoff or ignore 91 92 Auxiliary routines: 93 94 ---peek 95 96 like dequeue but without removing a packet from the queue 97 98 ---reset 99 100 returns qdisc to initial state: purge all buffers, clear all 101 timers, counters (except for statistics) etc. 102 103 ---init 104 105 initializes newly created qdisc. 106 107 ---destroy 108 109 destroys resources allocated by init and during lifetime of qdisc. 110 111 ---change 112 113 changes qdisc parameters. 114 */ 115 116 /* Protects list of registered TC modules. It is pure SMP lock. */ 117 static DEFINE_RWLOCK(qdisc_mod_lock); 118 119 120 /************************************************ 121 * Queueing disciplines manipulation. * 122 ************************************************/ 123 124 125 /* The list of all installed queueing disciplines. */ 126 127 static struct Qdisc_ops *qdisc_base; 128 129 /* Register/unregister queueing discipline */ 130 131 int register_qdisc(struct Qdisc_ops *qops) 132 { 133 struct Qdisc_ops *q, **qp; 134 int rc = -EEXIST; 135 136 write_lock(&qdisc_mod_lock); 137 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 138 if (!strcmp(qops->id, q->id)) 139 goto out; 140 141 if (qops->enqueue == NULL) 142 qops->enqueue = noop_qdisc_ops.enqueue; 143 if (qops->peek == NULL) { 144 if (qops->dequeue == NULL) 145 qops->peek = noop_qdisc_ops.peek; 146 else 147 goto out_einval; 148 } 149 if (qops->dequeue == NULL) 150 qops->dequeue = noop_qdisc_ops.dequeue; 151 152 if (qops->cl_ops) { 153 const struct Qdisc_class_ops *cops = qops->cl_ops; 154 155 if (!(cops->find && cops->walk && cops->leaf)) 156 goto out_einval; 157 158 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf)) 159 goto out_einval; 160 } 161 162 qops->next = NULL; 163 *qp = qops; 164 rc = 0; 165 out: 166 write_unlock(&qdisc_mod_lock); 167 return rc; 168 169 out_einval: 170 rc = -EINVAL; 171 goto out; 172 } 173 EXPORT_SYMBOL(register_qdisc); 174 175 void unregister_qdisc(struct Qdisc_ops *qops) 176 { 177 struct Qdisc_ops *q, **qp; 178 int err = -ENOENT; 179 180 write_lock(&qdisc_mod_lock); 181 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 182 if (q == qops) 183 break; 184 if (q) { 185 *qp = q->next; 186 q->next = NULL; 187 err = 0; 188 } 189 write_unlock(&qdisc_mod_lock); 190 191 WARN(err, "unregister qdisc(%s) failed\n", qops->id); 192 } 193 EXPORT_SYMBOL(unregister_qdisc); 194 195 /* Get default qdisc if not otherwise specified */ 196 void qdisc_get_default(char *name, size_t len) 197 { 198 read_lock(&qdisc_mod_lock); 199 strscpy(name, default_qdisc_ops->id, len); 200 read_unlock(&qdisc_mod_lock); 201 } 202 203 static struct Qdisc_ops *qdisc_lookup_default(const char *name) 204 { 205 struct Qdisc_ops *q = NULL; 206 207 for (q = qdisc_base; q; q = q->next) { 208 if (!strcmp(name, q->id)) { 209 if (!try_module_get(q->owner)) 210 q = NULL; 211 break; 212 } 213 } 214 215 return q; 216 } 217 218 /* Set new default qdisc to use */ 219 int qdisc_set_default(const char *name) 220 { 221 const struct Qdisc_ops *ops; 222 223 if (!capable(CAP_NET_ADMIN)) 224 return -EPERM; 225 226 write_lock(&qdisc_mod_lock); 227 ops = qdisc_lookup_default(name); 228 if (!ops) { 229 /* Not found, drop lock and try to load module */ 230 write_unlock(&qdisc_mod_lock); 231 request_module("sch_%s", name); 232 write_lock(&qdisc_mod_lock); 233 234 ops = qdisc_lookup_default(name); 235 } 236 237 if (ops) { 238 /* Set new default */ 239 module_put(default_qdisc_ops->owner); 240 default_qdisc_ops = ops; 241 } 242 write_unlock(&qdisc_mod_lock); 243 244 return ops ? 0 : -ENOENT; 245 } 246 247 #ifdef CONFIG_NET_SCH_DEFAULT 248 /* Set default value from kernel config */ 249 static int __init sch_default_qdisc(void) 250 { 251 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH); 252 } 253 late_initcall(sch_default_qdisc); 254 #endif 255 256 /* We know handle. Find qdisc among all qdisc's attached to device 257 * (root qdisc, all its children, children of children etc.) 258 * Note: caller either uses rtnl or rcu_read_lock() 259 */ 260 261 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle) 262 { 263 struct Qdisc *q; 264 265 if (!qdisc_dev(root)) 266 return (root->handle == handle ? root : NULL); 267 268 if (!(root->flags & TCQ_F_BUILTIN) && 269 root->handle == handle) 270 return root; 271 272 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle, 273 lockdep_rtnl_is_held()) { 274 if (q->handle == handle) 275 return q; 276 } 277 return NULL; 278 } 279 280 void qdisc_hash_add(struct Qdisc *q, bool invisible) 281 { 282 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { 283 ASSERT_RTNL(); 284 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle); 285 if (invisible) 286 q->flags |= TCQ_F_INVISIBLE; 287 } 288 } 289 EXPORT_SYMBOL(qdisc_hash_add); 290 291 void qdisc_hash_del(struct Qdisc *q) 292 { 293 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) { 294 ASSERT_RTNL(); 295 hash_del_rcu(&q->hash); 296 } 297 } 298 EXPORT_SYMBOL(qdisc_hash_del); 299 300 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) 301 { 302 struct Qdisc *q; 303 304 if (!handle) 305 return NULL; 306 q = qdisc_match_from_root(rtnl_dereference(dev->qdisc), handle); 307 if (q) 308 goto out; 309 310 if (dev_ingress_queue(dev)) 311 q = qdisc_match_from_root( 312 dev_ingress_queue(dev)->qdisc_sleeping, 313 handle); 314 out: 315 return q; 316 } 317 318 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle) 319 { 320 struct netdev_queue *nq; 321 struct Qdisc *q; 322 323 if (!handle) 324 return NULL; 325 q = qdisc_match_from_root(rcu_dereference(dev->qdisc), handle); 326 if (q) 327 goto out; 328 329 nq = dev_ingress_queue_rcu(dev); 330 if (nq) 331 q = qdisc_match_from_root(nq->qdisc_sleeping, handle); 332 out: 333 return q; 334 } 335 336 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) 337 { 338 unsigned long cl; 339 const struct Qdisc_class_ops *cops = p->ops->cl_ops; 340 341 if (cops == NULL) 342 return NULL; 343 cl = cops->find(p, classid); 344 345 if (cl == 0) 346 return NULL; 347 return cops->leaf(p, cl); 348 } 349 350 /* Find queueing discipline by name */ 351 352 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind) 353 { 354 struct Qdisc_ops *q = NULL; 355 356 if (kind) { 357 read_lock(&qdisc_mod_lock); 358 for (q = qdisc_base; q; q = q->next) { 359 if (nla_strcmp(kind, q->id) == 0) { 360 if (!try_module_get(q->owner)) 361 q = NULL; 362 break; 363 } 364 } 365 read_unlock(&qdisc_mod_lock); 366 } 367 return q; 368 } 369 370 /* The linklayer setting were not transferred from iproute2, in older 371 * versions, and the rate tables lookup systems have been dropped in 372 * the kernel. To keep backward compatible with older iproute2 tc 373 * utils, we detect the linklayer setting by detecting if the rate 374 * table were modified. 375 * 376 * For linklayer ATM table entries, the rate table will be aligned to 377 * 48 bytes, thus some table entries will contain the same value. The 378 * mpu (min packet unit) is also encoded into the old rate table, thus 379 * starting from the mpu, we find low and high table entries for 380 * mapping this cell. If these entries contain the same value, when 381 * the rate tables have been modified for linklayer ATM. 382 * 383 * This is done by rounding mpu to the nearest 48 bytes cell/entry, 384 * and then roundup to the next cell, calc the table entry one below, 385 * and compare. 386 */ 387 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab) 388 { 389 int low = roundup(r->mpu, 48); 390 int high = roundup(low+1, 48); 391 int cell_low = low >> r->cell_log; 392 int cell_high = (high >> r->cell_log) - 1; 393 394 /* rtab is too inaccurate at rates > 100Mbit/s */ 395 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) { 396 pr_debug("TC linklayer: Giving up ATM detection\n"); 397 return TC_LINKLAYER_ETHERNET; 398 } 399 400 if ((cell_high > cell_low) && (cell_high < 256) 401 && (rtab[cell_low] == rtab[cell_high])) { 402 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n", 403 cell_low, cell_high, rtab[cell_high]); 404 return TC_LINKLAYER_ATM; 405 } 406 return TC_LINKLAYER_ETHERNET; 407 } 408 409 static struct qdisc_rate_table *qdisc_rtab_list; 410 411 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, 412 struct nlattr *tab, 413 struct netlink_ext_ack *extack) 414 { 415 struct qdisc_rate_table *rtab; 416 417 if (tab == NULL || r->rate == 0 || 418 r->cell_log == 0 || r->cell_log >= 32 || 419 nla_len(tab) != TC_RTAB_SIZE) { 420 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching"); 421 return NULL; 422 } 423 424 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) { 425 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) && 426 !memcmp(&rtab->data, nla_data(tab), 1024)) { 427 rtab->refcnt++; 428 return rtab; 429 } 430 } 431 432 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL); 433 if (rtab) { 434 rtab->rate = *r; 435 rtab->refcnt = 1; 436 memcpy(rtab->data, nla_data(tab), 1024); 437 if (r->linklayer == TC_LINKLAYER_UNAWARE) 438 r->linklayer = __detect_linklayer(r, rtab->data); 439 rtab->next = qdisc_rtab_list; 440 qdisc_rtab_list = rtab; 441 } else { 442 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table"); 443 } 444 return rtab; 445 } 446 EXPORT_SYMBOL(qdisc_get_rtab); 447 448 void qdisc_put_rtab(struct qdisc_rate_table *tab) 449 { 450 struct qdisc_rate_table *rtab, **rtabp; 451 452 if (!tab || --tab->refcnt) 453 return; 454 455 for (rtabp = &qdisc_rtab_list; 456 (rtab = *rtabp) != NULL; 457 rtabp = &rtab->next) { 458 if (rtab == tab) { 459 *rtabp = rtab->next; 460 kfree(rtab); 461 return; 462 } 463 } 464 } 465 EXPORT_SYMBOL(qdisc_put_rtab); 466 467 static LIST_HEAD(qdisc_stab_list); 468 469 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = { 470 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) }, 471 [TCA_STAB_DATA] = { .type = NLA_BINARY }, 472 }; 473 474 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt, 475 struct netlink_ext_ack *extack) 476 { 477 struct nlattr *tb[TCA_STAB_MAX + 1]; 478 struct qdisc_size_table *stab; 479 struct tc_sizespec *s; 480 unsigned int tsize = 0; 481 u16 *tab = NULL; 482 int err; 483 484 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy, 485 extack); 486 if (err < 0) 487 return ERR_PTR(err); 488 if (!tb[TCA_STAB_BASE]) { 489 NL_SET_ERR_MSG(extack, "Size table base attribute is missing"); 490 return ERR_PTR(-EINVAL); 491 } 492 493 s = nla_data(tb[TCA_STAB_BASE]); 494 495 if (s->tsize > 0) { 496 if (!tb[TCA_STAB_DATA]) { 497 NL_SET_ERR_MSG(extack, "Size table data attribute is missing"); 498 return ERR_PTR(-EINVAL); 499 } 500 tab = nla_data(tb[TCA_STAB_DATA]); 501 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16); 502 } 503 504 if (tsize != s->tsize || (!tab && tsize > 0)) { 505 NL_SET_ERR_MSG(extack, "Invalid size of size table"); 506 return ERR_PTR(-EINVAL); 507 } 508 509 list_for_each_entry(stab, &qdisc_stab_list, list) { 510 if (memcmp(&stab->szopts, s, sizeof(*s))) 511 continue; 512 if (tsize > 0 && 513 memcmp(stab->data, tab, flex_array_size(stab, data, tsize))) 514 continue; 515 stab->refcnt++; 516 return stab; 517 } 518 519 if (s->size_log > STAB_SIZE_LOG_MAX || 520 s->cell_log > STAB_SIZE_LOG_MAX) { 521 NL_SET_ERR_MSG(extack, "Invalid logarithmic size of size table"); 522 return ERR_PTR(-EINVAL); 523 } 524 525 stab = kmalloc(struct_size(stab, data, tsize), GFP_KERNEL); 526 if (!stab) 527 return ERR_PTR(-ENOMEM); 528 529 stab->refcnt = 1; 530 stab->szopts = *s; 531 if (tsize > 0) 532 memcpy(stab->data, tab, flex_array_size(stab, data, tsize)); 533 534 list_add_tail(&stab->list, &qdisc_stab_list); 535 536 return stab; 537 } 538 539 void qdisc_put_stab(struct qdisc_size_table *tab) 540 { 541 if (!tab) 542 return; 543 544 if (--tab->refcnt == 0) { 545 list_del(&tab->list); 546 kfree_rcu(tab, rcu); 547 } 548 } 549 EXPORT_SYMBOL(qdisc_put_stab); 550 551 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab) 552 { 553 struct nlattr *nest; 554 555 nest = nla_nest_start_noflag(skb, TCA_STAB); 556 if (nest == NULL) 557 goto nla_put_failure; 558 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts)) 559 goto nla_put_failure; 560 nla_nest_end(skb, nest); 561 562 return skb->len; 563 564 nla_put_failure: 565 return -1; 566 } 567 568 void __qdisc_calculate_pkt_len(struct sk_buff *skb, 569 const struct qdisc_size_table *stab) 570 { 571 int pkt_len, slot; 572 573 pkt_len = skb->len + stab->szopts.overhead; 574 if (unlikely(!stab->szopts.tsize)) 575 goto out; 576 577 slot = pkt_len + stab->szopts.cell_align; 578 if (unlikely(slot < 0)) 579 slot = 0; 580 581 slot >>= stab->szopts.cell_log; 582 if (likely(slot < stab->szopts.tsize)) 583 pkt_len = stab->data[slot]; 584 else 585 pkt_len = stab->data[stab->szopts.tsize - 1] * 586 (slot / stab->szopts.tsize) + 587 stab->data[slot % stab->szopts.tsize]; 588 589 pkt_len <<= stab->szopts.size_log; 590 out: 591 if (unlikely(pkt_len < 1)) 592 pkt_len = 1; 593 qdisc_skb_cb(skb)->pkt_len = pkt_len; 594 } 595 EXPORT_SYMBOL(__qdisc_calculate_pkt_len); 596 597 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc) 598 { 599 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { 600 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", 601 txt, qdisc->ops->id, qdisc->handle >> 16); 602 qdisc->flags |= TCQ_F_WARN_NONWC; 603 } 604 } 605 EXPORT_SYMBOL(qdisc_warn_nonwc); 606 607 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) 608 { 609 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, 610 timer); 611 612 rcu_read_lock(); 613 __netif_schedule(qdisc_root(wd->qdisc)); 614 rcu_read_unlock(); 615 616 return HRTIMER_NORESTART; 617 } 618 619 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc, 620 clockid_t clockid) 621 { 622 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED); 623 wd->timer.function = qdisc_watchdog; 624 wd->qdisc = qdisc; 625 } 626 EXPORT_SYMBOL(qdisc_watchdog_init_clockid); 627 628 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc) 629 { 630 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC); 631 } 632 EXPORT_SYMBOL(qdisc_watchdog_init); 633 634 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires, 635 u64 delta_ns) 636 { 637 if (test_bit(__QDISC_STATE_DEACTIVATED, 638 &qdisc_root_sleeping(wd->qdisc)->state)) 639 return; 640 641 if (hrtimer_is_queued(&wd->timer)) { 642 u64 softexpires; 643 644 softexpires = ktime_to_ns(hrtimer_get_softexpires(&wd->timer)); 645 /* If timer is already set in [expires, expires + delta_ns], 646 * do not reprogram it. 647 */ 648 if (softexpires - expires <= delta_ns) 649 return; 650 } 651 652 hrtimer_start_range_ns(&wd->timer, 653 ns_to_ktime(expires), 654 delta_ns, 655 HRTIMER_MODE_ABS_PINNED); 656 } 657 EXPORT_SYMBOL(qdisc_watchdog_schedule_range_ns); 658 659 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd) 660 { 661 hrtimer_cancel(&wd->timer); 662 } 663 EXPORT_SYMBOL(qdisc_watchdog_cancel); 664 665 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n) 666 { 667 struct hlist_head *h; 668 unsigned int i; 669 670 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL); 671 672 if (h != NULL) { 673 for (i = 0; i < n; i++) 674 INIT_HLIST_HEAD(&h[i]); 675 } 676 return h; 677 } 678 679 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash) 680 { 681 struct Qdisc_class_common *cl; 682 struct hlist_node *next; 683 struct hlist_head *nhash, *ohash; 684 unsigned int nsize, nmask, osize; 685 unsigned int i, h; 686 687 /* Rehash when load factor exceeds 0.75 */ 688 if (clhash->hashelems * 4 <= clhash->hashsize * 3) 689 return; 690 nsize = clhash->hashsize * 2; 691 nmask = nsize - 1; 692 nhash = qdisc_class_hash_alloc(nsize); 693 if (nhash == NULL) 694 return; 695 696 ohash = clhash->hash; 697 osize = clhash->hashsize; 698 699 sch_tree_lock(sch); 700 for (i = 0; i < osize; i++) { 701 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) { 702 h = qdisc_class_hash(cl->classid, nmask); 703 hlist_add_head(&cl->hnode, &nhash[h]); 704 } 705 } 706 clhash->hash = nhash; 707 clhash->hashsize = nsize; 708 clhash->hashmask = nmask; 709 sch_tree_unlock(sch); 710 711 kvfree(ohash); 712 } 713 EXPORT_SYMBOL(qdisc_class_hash_grow); 714 715 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash) 716 { 717 unsigned int size = 4; 718 719 clhash->hash = qdisc_class_hash_alloc(size); 720 if (!clhash->hash) 721 return -ENOMEM; 722 clhash->hashsize = size; 723 clhash->hashmask = size - 1; 724 clhash->hashelems = 0; 725 return 0; 726 } 727 EXPORT_SYMBOL(qdisc_class_hash_init); 728 729 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash) 730 { 731 kvfree(clhash->hash); 732 } 733 EXPORT_SYMBOL(qdisc_class_hash_destroy); 734 735 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash, 736 struct Qdisc_class_common *cl) 737 { 738 unsigned int h; 739 740 INIT_HLIST_NODE(&cl->hnode); 741 h = qdisc_class_hash(cl->classid, clhash->hashmask); 742 hlist_add_head(&cl->hnode, &clhash->hash[h]); 743 clhash->hashelems++; 744 } 745 EXPORT_SYMBOL(qdisc_class_hash_insert); 746 747 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash, 748 struct Qdisc_class_common *cl) 749 { 750 hlist_del(&cl->hnode); 751 clhash->hashelems--; 752 } 753 EXPORT_SYMBOL(qdisc_class_hash_remove); 754 755 /* Allocate an unique handle from space managed by kernel 756 * Possible range is [8000-FFFF]:0000 (0x8000 values) 757 */ 758 static u32 qdisc_alloc_handle(struct net_device *dev) 759 { 760 int i = 0x8000; 761 static u32 autohandle = TC_H_MAKE(0x80000000U, 0); 762 763 do { 764 autohandle += TC_H_MAKE(0x10000U, 0); 765 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0)) 766 autohandle = TC_H_MAKE(0x80000000U, 0); 767 if (!qdisc_lookup(dev, autohandle)) 768 return autohandle; 769 cond_resched(); 770 } while (--i > 0); 771 772 return 0; 773 } 774 775 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) 776 { 777 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED; 778 const struct Qdisc_class_ops *cops; 779 unsigned long cl; 780 u32 parentid; 781 bool notify; 782 int drops; 783 784 if (n == 0 && len == 0) 785 return; 786 drops = max_t(int, n, 0); 787 rcu_read_lock(); 788 while ((parentid = sch->parent)) { 789 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS)) 790 break; 791 792 if (sch->flags & TCQ_F_NOPARENT) 793 break; 794 /* Notify parent qdisc only if child qdisc becomes empty. 795 * 796 * If child was empty even before update then backlog 797 * counter is screwed and we skip notification because 798 * parent class is already passive. 799 * 800 * If the original child was offloaded then it is allowed 801 * to be seem as empty, so the parent is notified anyway. 802 */ 803 notify = !sch->q.qlen && !WARN_ON_ONCE(!n && 804 !qdisc_is_offloaded); 805 /* TODO: perform the search on a per txq basis */ 806 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid)); 807 if (sch == NULL) { 808 WARN_ON_ONCE(parentid != TC_H_ROOT); 809 break; 810 } 811 cops = sch->ops->cl_ops; 812 if (notify && cops->qlen_notify) { 813 cl = cops->find(sch, parentid); 814 cops->qlen_notify(sch, cl); 815 } 816 sch->q.qlen -= n; 817 sch->qstats.backlog -= len; 818 __qdisc_qstats_drop(sch, drops); 819 } 820 rcu_read_unlock(); 821 } 822 EXPORT_SYMBOL(qdisc_tree_reduce_backlog); 823 824 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type, 825 void *type_data) 826 { 827 struct net_device *dev = qdisc_dev(sch); 828 int err; 829 830 sch->flags &= ~TCQ_F_OFFLOADED; 831 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 832 return 0; 833 834 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); 835 if (err == -EOPNOTSUPP) 836 return 0; 837 838 if (!err) 839 sch->flags |= TCQ_F_OFFLOADED; 840 841 return err; 842 } 843 EXPORT_SYMBOL(qdisc_offload_dump_helper); 844 845 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, 846 struct Qdisc *new, struct Qdisc *old, 847 enum tc_setup_type type, void *type_data, 848 struct netlink_ext_ack *extack) 849 { 850 bool any_qdisc_is_offloaded; 851 int err; 852 853 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) 854 return; 855 856 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); 857 858 /* Don't report error if the graft is part of destroy operation. */ 859 if (!err || !new || new == &noop_qdisc) 860 return; 861 862 /* Don't report error if the parent, the old child and the new 863 * one are not offloaded. 864 */ 865 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED; 866 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED; 867 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED; 868 869 if (any_qdisc_is_offloaded) 870 NL_SET_ERR_MSG(extack, "Offloading graft operation failed."); 871 } 872 EXPORT_SYMBOL(qdisc_offload_graft_helper); 873 874 void qdisc_offload_query_caps(struct net_device *dev, 875 enum tc_setup_type type, 876 void *caps, size_t caps_len) 877 { 878 const struct net_device_ops *ops = dev->netdev_ops; 879 struct tc_query_caps_base base = { 880 .type = type, 881 .caps = caps, 882 }; 883 884 memset(caps, 0, caps_len); 885 886 if (ops->ndo_setup_tc) 887 ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base); 888 } 889 EXPORT_SYMBOL(qdisc_offload_query_caps); 890 891 static void qdisc_offload_graft_root(struct net_device *dev, 892 struct Qdisc *new, struct Qdisc *old, 893 struct netlink_ext_ack *extack) 894 { 895 struct tc_root_qopt_offload graft_offload = { 896 .command = TC_ROOT_GRAFT, 897 .handle = new ? new->handle : 0, 898 .ingress = (new && new->flags & TCQ_F_INGRESS) || 899 (old && old->flags & TCQ_F_INGRESS), 900 }; 901 902 qdisc_offload_graft_helper(dev, NULL, new, old, 903 TC_SETUP_ROOT_QDISC, &graft_offload, extack); 904 } 905 906 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, 907 u32 portid, u32 seq, u16 flags, int event, 908 struct netlink_ext_ack *extack) 909 { 910 struct gnet_stats_basic_sync __percpu *cpu_bstats = NULL; 911 struct gnet_stats_queue __percpu *cpu_qstats = NULL; 912 struct tcmsg *tcm; 913 struct nlmsghdr *nlh; 914 unsigned char *b = skb_tail_pointer(skb); 915 struct gnet_dump d; 916 struct qdisc_size_table *stab; 917 u32 block_index; 918 __u32 qlen; 919 920 cond_resched(); 921 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 922 if (!nlh) 923 goto out_nlmsg_trim; 924 tcm = nlmsg_data(nlh); 925 tcm->tcm_family = AF_UNSPEC; 926 tcm->tcm__pad1 = 0; 927 tcm->tcm__pad2 = 0; 928 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 929 tcm->tcm_parent = clid; 930 tcm->tcm_handle = q->handle; 931 tcm->tcm_info = refcount_read(&q->refcnt); 932 if (nla_put_string(skb, TCA_KIND, q->ops->id)) 933 goto nla_put_failure; 934 if (q->ops->ingress_block_get) { 935 block_index = q->ops->ingress_block_get(q); 936 if (block_index && 937 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index)) 938 goto nla_put_failure; 939 } 940 if (q->ops->egress_block_get) { 941 block_index = q->ops->egress_block_get(q); 942 if (block_index && 943 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index)) 944 goto nla_put_failure; 945 } 946 if (q->ops->dump && q->ops->dump(q, skb) < 0) 947 goto nla_put_failure; 948 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED))) 949 goto nla_put_failure; 950 qlen = qdisc_qlen_sum(q); 951 952 stab = rtnl_dereference(q->stab); 953 if (stab && qdisc_dump_stab(skb, stab) < 0) 954 goto nla_put_failure; 955 956 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 957 NULL, &d, TCA_PAD) < 0) 958 goto nla_put_failure; 959 960 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0) 961 goto nla_put_failure; 962 963 if (qdisc_is_percpu_stats(q)) { 964 cpu_bstats = q->cpu_bstats; 965 cpu_qstats = q->cpu_qstats; 966 } 967 968 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats, true) < 0 || 969 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 || 970 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0) 971 goto nla_put_failure; 972 973 if (gnet_stats_finish_copy(&d) < 0) 974 goto nla_put_failure; 975 976 if (extack && extack->_msg && 977 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 978 goto out_nlmsg_trim; 979 980 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 981 982 return skb->len; 983 984 out_nlmsg_trim: 985 nla_put_failure: 986 nlmsg_trim(skb, b); 987 return -1; 988 } 989 990 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible) 991 { 992 if (q->flags & TCQ_F_BUILTIN) 993 return true; 994 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible) 995 return true; 996 997 return false; 998 } 999 1000 static int qdisc_notify(struct net *net, struct sk_buff *oskb, 1001 struct nlmsghdr *n, u32 clid, 1002 struct Qdisc *old, struct Qdisc *new, 1003 struct netlink_ext_ack *extack) 1004 { 1005 struct sk_buff *skb; 1006 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1007 1008 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1009 if (!skb) 1010 return -ENOBUFS; 1011 1012 if (old && !tc_qdisc_dump_ignore(old, false)) { 1013 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq, 1014 0, RTM_DELQDISC, extack) < 0) 1015 goto err_out; 1016 } 1017 if (new && !tc_qdisc_dump_ignore(new, false)) { 1018 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq, 1019 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC, extack) < 0) 1020 goto err_out; 1021 } 1022 1023 if (skb->len) 1024 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1025 n->nlmsg_flags & NLM_F_ECHO); 1026 1027 err_out: 1028 kfree_skb(skb); 1029 return -EINVAL; 1030 } 1031 1032 static void notify_and_destroy(struct net *net, struct sk_buff *skb, 1033 struct nlmsghdr *n, u32 clid, 1034 struct Qdisc *old, struct Qdisc *new, 1035 struct netlink_ext_ack *extack) 1036 { 1037 if (new || old) 1038 qdisc_notify(net, skb, n, clid, old, new, extack); 1039 1040 if (old) 1041 qdisc_put(old); 1042 } 1043 1044 static void qdisc_clear_nolock(struct Qdisc *sch) 1045 { 1046 sch->flags &= ~TCQ_F_NOLOCK; 1047 if (!(sch->flags & TCQ_F_CPUSTATS)) 1048 return; 1049 1050 free_percpu(sch->cpu_bstats); 1051 free_percpu(sch->cpu_qstats); 1052 sch->cpu_bstats = NULL; 1053 sch->cpu_qstats = NULL; 1054 sch->flags &= ~TCQ_F_CPUSTATS; 1055 } 1056 1057 /* Graft qdisc "new" to class "classid" of qdisc "parent" or 1058 * to device "dev". 1059 * 1060 * When appropriate send a netlink notification using 'skb' 1061 * and "n". 1062 * 1063 * On success, destroy old qdisc. 1064 */ 1065 1066 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, 1067 struct sk_buff *skb, struct nlmsghdr *n, u32 classid, 1068 struct Qdisc *new, struct Qdisc *old, 1069 struct netlink_ext_ack *extack) 1070 { 1071 struct Qdisc *q = old; 1072 struct net *net = dev_net(dev); 1073 1074 if (parent == NULL) { 1075 unsigned int i, num_q, ingress; 1076 1077 ingress = 0; 1078 num_q = dev->num_tx_queues; 1079 if ((q && q->flags & TCQ_F_INGRESS) || 1080 (new && new->flags & TCQ_F_INGRESS)) { 1081 num_q = 1; 1082 ingress = 1; 1083 if (!dev_ingress_queue(dev)) { 1084 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue"); 1085 return -ENOENT; 1086 } 1087 } 1088 1089 if (dev->flags & IFF_UP) 1090 dev_deactivate(dev); 1091 1092 qdisc_offload_graft_root(dev, new, old, extack); 1093 1094 if (new && new->ops->attach && !ingress) 1095 goto skip; 1096 1097 for (i = 0; i < num_q; i++) { 1098 struct netdev_queue *dev_queue = dev_ingress_queue(dev); 1099 1100 if (!ingress) 1101 dev_queue = netdev_get_tx_queue(dev, i); 1102 1103 old = dev_graft_qdisc(dev_queue, new); 1104 if (new && i > 0) 1105 qdisc_refcount_inc(new); 1106 1107 if (!ingress) 1108 qdisc_put(old); 1109 } 1110 1111 skip: 1112 if (!ingress) { 1113 old = rtnl_dereference(dev->qdisc); 1114 if (new && !new->ops->attach) 1115 qdisc_refcount_inc(new); 1116 rcu_assign_pointer(dev->qdisc, new ? : &noop_qdisc); 1117 1118 notify_and_destroy(net, skb, n, classid, old, new, extack); 1119 1120 if (new && new->ops->attach) 1121 new->ops->attach(new); 1122 } else { 1123 notify_and_destroy(net, skb, n, classid, old, new, extack); 1124 } 1125 1126 if (dev->flags & IFF_UP) 1127 dev_activate(dev); 1128 } else { 1129 const struct Qdisc_class_ops *cops = parent->ops->cl_ops; 1130 unsigned long cl; 1131 int err; 1132 1133 /* Only support running class lockless if parent is lockless */ 1134 if (new && (new->flags & TCQ_F_NOLOCK) && !(parent->flags & TCQ_F_NOLOCK)) 1135 qdisc_clear_nolock(new); 1136 1137 if (!cops || !cops->graft) 1138 return -EOPNOTSUPP; 1139 1140 cl = cops->find(parent, classid); 1141 if (!cl) { 1142 NL_SET_ERR_MSG(extack, "Specified class not found"); 1143 return -ENOENT; 1144 } 1145 1146 if (new && new->ops == &noqueue_qdisc_ops) { 1147 NL_SET_ERR_MSG(extack, "Cannot assign noqueue to a class"); 1148 return -EINVAL; 1149 } 1150 1151 err = cops->graft(parent, cl, new, &old, extack); 1152 if (err) 1153 return err; 1154 notify_and_destroy(net, skb, n, classid, old, new, extack); 1155 } 1156 return 0; 1157 } 1158 1159 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca, 1160 struct netlink_ext_ack *extack) 1161 { 1162 u32 block_index; 1163 1164 if (tca[TCA_INGRESS_BLOCK]) { 1165 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]); 1166 1167 if (!block_index) { 1168 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0"); 1169 return -EINVAL; 1170 } 1171 if (!sch->ops->ingress_block_set) { 1172 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported"); 1173 return -EOPNOTSUPP; 1174 } 1175 sch->ops->ingress_block_set(sch, block_index); 1176 } 1177 if (tca[TCA_EGRESS_BLOCK]) { 1178 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]); 1179 1180 if (!block_index) { 1181 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0"); 1182 return -EINVAL; 1183 } 1184 if (!sch->ops->egress_block_set) { 1185 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported"); 1186 return -EOPNOTSUPP; 1187 } 1188 sch->ops->egress_block_set(sch, block_index); 1189 } 1190 return 0; 1191 } 1192 1193 /* 1194 Allocate and initialize new qdisc. 1195 1196 Parameters are passed via opt. 1197 */ 1198 1199 static struct Qdisc *qdisc_create(struct net_device *dev, 1200 struct netdev_queue *dev_queue, 1201 u32 parent, u32 handle, 1202 struct nlattr **tca, int *errp, 1203 struct netlink_ext_ack *extack) 1204 { 1205 int err; 1206 struct nlattr *kind = tca[TCA_KIND]; 1207 struct Qdisc *sch; 1208 struct Qdisc_ops *ops; 1209 struct qdisc_size_table *stab; 1210 1211 ops = qdisc_lookup_ops(kind); 1212 #ifdef CONFIG_MODULES 1213 if (ops == NULL && kind != NULL) { 1214 char name[IFNAMSIZ]; 1215 if (nla_strscpy(name, kind, IFNAMSIZ) >= 0) { 1216 /* We dropped the RTNL semaphore in order to 1217 * perform the module load. So, even if we 1218 * succeeded in loading the module we have to 1219 * tell the caller to replay the request. We 1220 * indicate this using -EAGAIN. 1221 * We replay the request because the device may 1222 * go away in the mean time. 1223 */ 1224 rtnl_unlock(); 1225 request_module("sch_%s", name); 1226 rtnl_lock(); 1227 ops = qdisc_lookup_ops(kind); 1228 if (ops != NULL) { 1229 /* We will try again qdisc_lookup_ops, 1230 * so don't keep a reference. 1231 */ 1232 module_put(ops->owner); 1233 err = -EAGAIN; 1234 goto err_out; 1235 } 1236 } 1237 } 1238 #endif 1239 1240 err = -ENOENT; 1241 if (!ops) { 1242 NL_SET_ERR_MSG(extack, "Specified qdisc kind is unknown"); 1243 goto err_out; 1244 } 1245 1246 sch = qdisc_alloc(dev_queue, ops, extack); 1247 if (IS_ERR(sch)) { 1248 err = PTR_ERR(sch); 1249 goto err_out2; 1250 } 1251 1252 sch->parent = parent; 1253 1254 if (handle == TC_H_INGRESS) { 1255 if (!(sch->flags & TCQ_F_INGRESS)) { 1256 NL_SET_ERR_MSG(extack, 1257 "Specified parent ID is reserved for ingress and clsact Qdiscs"); 1258 err = -EINVAL; 1259 goto err_out3; 1260 } 1261 handle = TC_H_MAKE(TC_H_INGRESS, 0); 1262 } else { 1263 if (handle == 0) { 1264 handle = qdisc_alloc_handle(dev); 1265 if (handle == 0) { 1266 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded"); 1267 err = -ENOSPC; 1268 goto err_out3; 1269 } 1270 } 1271 if (!netif_is_multiqueue(dev)) 1272 sch->flags |= TCQ_F_ONETXQUEUE; 1273 } 1274 1275 sch->handle = handle; 1276 1277 /* This exist to keep backward compatible with a userspace 1278 * loophole, what allowed userspace to get IFF_NO_QUEUE 1279 * facility on older kernels by setting tx_queue_len=0 (prior 1280 * to qdisc init), and then forgot to reinit tx_queue_len 1281 * before again attaching a qdisc. 1282 */ 1283 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) { 1284 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 1285 netdev_info(dev, "Caught tx_queue_len zero misconfig\n"); 1286 } 1287 1288 err = qdisc_block_indexes_set(sch, tca, extack); 1289 if (err) 1290 goto err_out3; 1291 1292 if (tca[TCA_STAB]) { 1293 stab = qdisc_get_stab(tca[TCA_STAB], extack); 1294 if (IS_ERR(stab)) { 1295 err = PTR_ERR(stab); 1296 goto err_out3; 1297 } 1298 rcu_assign_pointer(sch->stab, stab); 1299 } 1300 1301 if (ops->init) { 1302 err = ops->init(sch, tca[TCA_OPTIONS], extack); 1303 if (err != 0) 1304 goto err_out4; 1305 } 1306 1307 if (tca[TCA_RATE]) { 1308 err = -EOPNOTSUPP; 1309 if (sch->flags & TCQ_F_MQROOT) { 1310 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc"); 1311 goto err_out4; 1312 } 1313 1314 err = gen_new_estimator(&sch->bstats, 1315 sch->cpu_bstats, 1316 &sch->rate_est, 1317 NULL, 1318 true, 1319 tca[TCA_RATE]); 1320 if (err) { 1321 NL_SET_ERR_MSG(extack, "Failed to generate new estimator"); 1322 goto err_out4; 1323 } 1324 } 1325 1326 qdisc_hash_add(sch, false); 1327 trace_qdisc_create(ops, dev, parent); 1328 1329 return sch; 1330 1331 err_out4: 1332 /* Even if ops->init() failed, we call ops->destroy() 1333 * like qdisc_create_dflt(). 1334 */ 1335 if (ops->destroy) 1336 ops->destroy(sch); 1337 qdisc_put_stab(rtnl_dereference(sch->stab)); 1338 err_out3: 1339 netdev_put(dev, &sch->dev_tracker); 1340 qdisc_free(sch); 1341 err_out2: 1342 module_put(ops->owner); 1343 err_out: 1344 *errp = err; 1345 return NULL; 1346 } 1347 1348 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca, 1349 struct netlink_ext_ack *extack) 1350 { 1351 struct qdisc_size_table *ostab, *stab = NULL; 1352 int err = 0; 1353 1354 if (tca[TCA_OPTIONS]) { 1355 if (!sch->ops->change) { 1356 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc"); 1357 return -EINVAL; 1358 } 1359 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) { 1360 NL_SET_ERR_MSG(extack, "Change of blocks is not supported"); 1361 return -EOPNOTSUPP; 1362 } 1363 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack); 1364 if (err) 1365 return err; 1366 } 1367 1368 if (tca[TCA_STAB]) { 1369 stab = qdisc_get_stab(tca[TCA_STAB], extack); 1370 if (IS_ERR(stab)) 1371 return PTR_ERR(stab); 1372 } 1373 1374 ostab = rtnl_dereference(sch->stab); 1375 rcu_assign_pointer(sch->stab, stab); 1376 qdisc_put_stab(ostab); 1377 1378 if (tca[TCA_RATE]) { 1379 /* NB: ignores errors from replace_estimator 1380 because change can't be undone. */ 1381 if (sch->flags & TCQ_F_MQROOT) 1382 goto out; 1383 gen_replace_estimator(&sch->bstats, 1384 sch->cpu_bstats, 1385 &sch->rate_est, 1386 NULL, 1387 true, 1388 tca[TCA_RATE]); 1389 } 1390 out: 1391 return 0; 1392 } 1393 1394 struct check_loop_arg { 1395 struct qdisc_walker w; 1396 struct Qdisc *p; 1397 int depth; 1398 }; 1399 1400 static int check_loop_fn(struct Qdisc *q, unsigned long cl, 1401 struct qdisc_walker *w); 1402 1403 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth) 1404 { 1405 struct check_loop_arg arg; 1406 1407 if (q->ops->cl_ops == NULL) 1408 return 0; 1409 1410 arg.w.stop = arg.w.skip = arg.w.count = 0; 1411 arg.w.fn = check_loop_fn; 1412 arg.depth = depth; 1413 arg.p = p; 1414 q->ops->cl_ops->walk(q, &arg.w); 1415 return arg.w.stop ? -ELOOP : 0; 1416 } 1417 1418 static int 1419 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w) 1420 { 1421 struct Qdisc *leaf; 1422 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1423 struct check_loop_arg *arg = (struct check_loop_arg *)w; 1424 1425 leaf = cops->leaf(q, cl); 1426 if (leaf) { 1427 if (leaf == arg->p || arg->depth > 7) 1428 return -ELOOP; 1429 return check_loop(leaf, arg->p, arg->depth + 1); 1430 } 1431 return 0; 1432 } 1433 1434 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = { 1435 [TCA_KIND] = { .type = NLA_STRING }, 1436 [TCA_RATE] = { .type = NLA_BINARY, 1437 .len = sizeof(struct tc_estimator) }, 1438 [TCA_STAB] = { .type = NLA_NESTED }, 1439 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG }, 1440 [TCA_CHAIN] = { .type = NLA_U32 }, 1441 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 }, 1442 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 }, 1443 }; 1444 1445 /* 1446 * Delete/get qdisc. 1447 */ 1448 1449 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1450 struct netlink_ext_ack *extack) 1451 { 1452 struct net *net = sock_net(skb->sk); 1453 struct tcmsg *tcm = nlmsg_data(n); 1454 struct nlattr *tca[TCA_MAX + 1]; 1455 struct net_device *dev; 1456 u32 clid; 1457 struct Qdisc *q = NULL; 1458 struct Qdisc *p = NULL; 1459 int err; 1460 1461 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 1462 rtm_tca_policy, extack); 1463 if (err < 0) 1464 return err; 1465 1466 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1467 if (!dev) 1468 return -ENODEV; 1469 1470 clid = tcm->tcm_parent; 1471 if (clid) { 1472 if (clid != TC_H_ROOT) { 1473 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) { 1474 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1475 if (!p) { 1476 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid"); 1477 return -ENOENT; 1478 } 1479 q = qdisc_leaf(p, clid); 1480 } else if (dev_ingress_queue(dev)) { 1481 q = dev_ingress_queue(dev)->qdisc_sleeping; 1482 } 1483 } else { 1484 q = rtnl_dereference(dev->qdisc); 1485 } 1486 if (!q) { 1487 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device"); 1488 return -ENOENT; 1489 } 1490 1491 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) { 1492 NL_SET_ERR_MSG(extack, "Invalid handle"); 1493 return -EINVAL; 1494 } 1495 } else { 1496 q = qdisc_lookup(dev, tcm->tcm_handle); 1497 if (!q) { 1498 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle"); 1499 return -ENOENT; 1500 } 1501 } 1502 1503 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1504 NL_SET_ERR_MSG(extack, "Invalid qdisc name"); 1505 return -EINVAL; 1506 } 1507 1508 if (n->nlmsg_type == RTM_DELQDISC) { 1509 if (!clid) { 1510 NL_SET_ERR_MSG(extack, "Classid cannot be zero"); 1511 return -EINVAL; 1512 } 1513 if (q->handle == 0) { 1514 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero"); 1515 return -ENOENT; 1516 } 1517 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack); 1518 if (err != 0) 1519 return err; 1520 } else { 1521 qdisc_notify(net, skb, n, clid, NULL, q, NULL); 1522 } 1523 return 0; 1524 } 1525 1526 /* 1527 * Create/change qdisc. 1528 */ 1529 1530 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, 1531 struct netlink_ext_ack *extack) 1532 { 1533 struct net *net = sock_net(skb->sk); 1534 struct tcmsg *tcm; 1535 struct nlattr *tca[TCA_MAX + 1]; 1536 struct net_device *dev; 1537 u32 clid; 1538 struct Qdisc *q, *p; 1539 int err; 1540 1541 replay: 1542 /* Reinit, just in case something touches this. */ 1543 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 1544 rtm_tca_policy, extack); 1545 if (err < 0) 1546 return err; 1547 1548 tcm = nlmsg_data(n); 1549 clid = tcm->tcm_parent; 1550 q = p = NULL; 1551 1552 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1553 if (!dev) 1554 return -ENODEV; 1555 1556 1557 if (clid) { 1558 if (clid != TC_H_ROOT) { 1559 if (clid != TC_H_INGRESS) { 1560 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1561 if (!p) { 1562 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc"); 1563 return -ENOENT; 1564 } 1565 q = qdisc_leaf(p, clid); 1566 } else if (dev_ingress_queue_create(dev)) { 1567 q = dev_ingress_queue(dev)->qdisc_sleeping; 1568 } 1569 } else { 1570 q = rtnl_dereference(dev->qdisc); 1571 } 1572 1573 /* It may be default qdisc, ignore it */ 1574 if (q && q->handle == 0) 1575 q = NULL; 1576 1577 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) { 1578 if (tcm->tcm_handle) { 1579 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) { 1580 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override"); 1581 return -EEXIST; 1582 } 1583 if (TC_H_MIN(tcm->tcm_handle)) { 1584 NL_SET_ERR_MSG(extack, "Invalid minor handle"); 1585 return -EINVAL; 1586 } 1587 q = qdisc_lookup(dev, tcm->tcm_handle); 1588 if (!q) 1589 goto create_n_graft; 1590 if (n->nlmsg_flags & NLM_F_EXCL) { 1591 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override"); 1592 return -EEXIST; 1593 } 1594 if (tca[TCA_KIND] && 1595 nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1596 NL_SET_ERR_MSG(extack, "Invalid qdisc name"); 1597 return -EINVAL; 1598 } 1599 if (q->flags & TCQ_F_INGRESS) { 1600 NL_SET_ERR_MSG(extack, 1601 "Cannot regraft ingress or clsact Qdiscs"); 1602 return -EINVAL; 1603 } 1604 if (q == p || 1605 (p && check_loop(q, p, 0))) { 1606 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected"); 1607 return -ELOOP; 1608 } 1609 if (clid == TC_H_INGRESS) { 1610 NL_SET_ERR_MSG(extack, "Ingress cannot graft directly"); 1611 return -EINVAL; 1612 } 1613 qdisc_refcount_inc(q); 1614 goto graft; 1615 } else { 1616 if (!q) 1617 goto create_n_graft; 1618 1619 /* This magic test requires explanation. 1620 * 1621 * We know, that some child q is already 1622 * attached to this parent and have choice: 1623 * either to change it or to create/graft new one. 1624 * 1625 * 1. We are allowed to create/graft only 1626 * if CREATE and REPLACE flags are set. 1627 * 1628 * 2. If EXCL is set, requestor wanted to say, 1629 * that qdisc tcm_handle is not expected 1630 * to exist, so that we choose create/graft too. 1631 * 1632 * 3. The last case is when no flags are set. 1633 * Alas, it is sort of hole in API, we 1634 * cannot decide what to do unambiguously. 1635 * For now we select create/graft, if 1636 * user gave KIND, which does not match existing. 1637 */ 1638 if ((n->nlmsg_flags & NLM_F_CREATE) && 1639 (n->nlmsg_flags & NLM_F_REPLACE) && 1640 ((n->nlmsg_flags & NLM_F_EXCL) || 1641 (tca[TCA_KIND] && 1642 nla_strcmp(tca[TCA_KIND], q->ops->id)))) 1643 goto create_n_graft; 1644 } 1645 } 1646 } else { 1647 if (!tcm->tcm_handle) { 1648 NL_SET_ERR_MSG(extack, "Handle cannot be zero"); 1649 return -EINVAL; 1650 } 1651 q = qdisc_lookup(dev, tcm->tcm_handle); 1652 } 1653 1654 /* Change qdisc parameters */ 1655 if (!q) { 1656 NL_SET_ERR_MSG(extack, "Specified qdisc not found"); 1657 return -ENOENT; 1658 } 1659 if (n->nlmsg_flags & NLM_F_EXCL) { 1660 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify"); 1661 return -EEXIST; 1662 } 1663 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) { 1664 NL_SET_ERR_MSG(extack, "Invalid qdisc name"); 1665 return -EINVAL; 1666 } 1667 err = qdisc_change(q, tca, extack); 1668 if (err == 0) 1669 qdisc_notify(net, skb, n, clid, NULL, q, extack); 1670 return err; 1671 1672 create_n_graft: 1673 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 1674 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag"); 1675 return -ENOENT; 1676 } 1677 if (clid == TC_H_INGRESS) { 1678 if (dev_ingress_queue(dev)) { 1679 q = qdisc_create(dev, dev_ingress_queue(dev), 1680 tcm->tcm_parent, tcm->tcm_parent, 1681 tca, &err, extack); 1682 } else { 1683 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device"); 1684 err = -ENOENT; 1685 } 1686 } else { 1687 struct netdev_queue *dev_queue; 1688 1689 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue) 1690 dev_queue = p->ops->cl_ops->select_queue(p, tcm); 1691 else if (p) 1692 dev_queue = p->dev_queue; 1693 else 1694 dev_queue = netdev_get_tx_queue(dev, 0); 1695 1696 q = qdisc_create(dev, dev_queue, 1697 tcm->tcm_parent, tcm->tcm_handle, 1698 tca, &err, extack); 1699 } 1700 if (q == NULL) { 1701 if (err == -EAGAIN) 1702 goto replay; 1703 return err; 1704 } 1705 1706 graft: 1707 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack); 1708 if (err) { 1709 if (q) 1710 qdisc_put(q); 1711 return err; 1712 } 1713 1714 return 0; 1715 } 1716 1717 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, 1718 struct netlink_callback *cb, 1719 int *q_idx_p, int s_q_idx, bool recur, 1720 bool dump_invisible) 1721 { 1722 int ret = 0, q_idx = *q_idx_p; 1723 struct Qdisc *q; 1724 int b; 1725 1726 if (!root) 1727 return 0; 1728 1729 q = root; 1730 if (q_idx < s_q_idx) { 1731 q_idx++; 1732 } else { 1733 if (!tc_qdisc_dump_ignore(q, dump_invisible) && 1734 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid, 1735 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1736 RTM_NEWQDISC, NULL) <= 0) 1737 goto done; 1738 q_idx++; 1739 } 1740 1741 /* If dumping singletons, there is no qdisc_dev(root) and the singleton 1742 * itself has already been dumped. 1743 * 1744 * If we've already dumped the top-level (ingress) qdisc above and the global 1745 * qdisc hashtable, we don't want to hit it again 1746 */ 1747 if (!qdisc_dev(root) || !recur) 1748 goto out; 1749 1750 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) { 1751 if (q_idx < s_q_idx) { 1752 q_idx++; 1753 continue; 1754 } 1755 if (!tc_qdisc_dump_ignore(q, dump_invisible) && 1756 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid, 1757 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1758 RTM_NEWQDISC, NULL) <= 0) 1759 goto done; 1760 q_idx++; 1761 } 1762 1763 out: 1764 *q_idx_p = q_idx; 1765 return ret; 1766 done: 1767 ret = -1; 1768 goto out; 1769 } 1770 1771 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) 1772 { 1773 struct net *net = sock_net(skb->sk); 1774 int idx, q_idx; 1775 int s_idx, s_q_idx; 1776 struct net_device *dev; 1777 const struct nlmsghdr *nlh = cb->nlh; 1778 struct nlattr *tca[TCA_MAX + 1]; 1779 int err; 1780 1781 s_idx = cb->args[0]; 1782 s_q_idx = q_idx = cb->args[1]; 1783 1784 idx = 0; 1785 ASSERT_RTNL(); 1786 1787 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX, 1788 rtm_tca_policy, cb->extack); 1789 if (err < 0) 1790 return err; 1791 1792 for_each_netdev(net, dev) { 1793 struct netdev_queue *dev_queue; 1794 1795 if (idx < s_idx) 1796 goto cont; 1797 if (idx > s_idx) 1798 s_q_idx = 0; 1799 q_idx = 0; 1800 1801 if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc), 1802 skb, cb, &q_idx, s_q_idx, 1803 true, tca[TCA_DUMP_INVISIBLE]) < 0) 1804 goto done; 1805 1806 dev_queue = dev_ingress_queue(dev); 1807 if (dev_queue && 1808 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb, 1809 &q_idx, s_q_idx, false, 1810 tca[TCA_DUMP_INVISIBLE]) < 0) 1811 goto done; 1812 1813 cont: 1814 idx++; 1815 } 1816 1817 done: 1818 cb->args[0] = idx; 1819 cb->args[1] = q_idx; 1820 1821 return skb->len; 1822 } 1823 1824 1825 1826 /************************************************ 1827 * Traffic classes manipulation. * 1828 ************************************************/ 1829 1830 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q, 1831 unsigned long cl, u32 portid, u32 seq, u16 flags, 1832 int event, struct netlink_ext_ack *extack) 1833 { 1834 struct tcmsg *tcm; 1835 struct nlmsghdr *nlh; 1836 unsigned char *b = skb_tail_pointer(skb); 1837 struct gnet_dump d; 1838 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops; 1839 1840 cond_resched(); 1841 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 1842 if (!nlh) 1843 goto out_nlmsg_trim; 1844 tcm = nlmsg_data(nlh); 1845 tcm->tcm_family = AF_UNSPEC; 1846 tcm->tcm__pad1 = 0; 1847 tcm->tcm__pad2 = 0; 1848 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1849 tcm->tcm_parent = q->handle; 1850 tcm->tcm_handle = q->handle; 1851 tcm->tcm_info = 0; 1852 if (nla_put_string(skb, TCA_KIND, q->ops->id)) 1853 goto nla_put_failure; 1854 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0) 1855 goto nla_put_failure; 1856 1857 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 1858 NULL, &d, TCA_PAD) < 0) 1859 goto nla_put_failure; 1860 1861 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0) 1862 goto nla_put_failure; 1863 1864 if (gnet_stats_finish_copy(&d) < 0) 1865 goto nla_put_failure; 1866 1867 if (extack && extack->_msg && 1868 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 1869 goto out_nlmsg_trim; 1870 1871 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1872 1873 return skb->len; 1874 1875 out_nlmsg_trim: 1876 nla_put_failure: 1877 nlmsg_trim(skb, b); 1878 return -1; 1879 } 1880 1881 static int tclass_notify(struct net *net, struct sk_buff *oskb, 1882 struct nlmsghdr *n, struct Qdisc *q, 1883 unsigned long cl, int event, struct netlink_ext_ack *extack) 1884 { 1885 struct sk_buff *skb; 1886 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1887 1888 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1889 if (!skb) 1890 return -ENOBUFS; 1891 1892 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event, extack) < 0) { 1893 kfree_skb(skb); 1894 return -EINVAL; 1895 } 1896 1897 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1898 n->nlmsg_flags & NLM_F_ECHO); 1899 } 1900 1901 static int tclass_del_notify(struct net *net, 1902 const struct Qdisc_class_ops *cops, 1903 struct sk_buff *oskb, struct nlmsghdr *n, 1904 struct Qdisc *q, unsigned long cl, 1905 struct netlink_ext_ack *extack) 1906 { 1907 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1908 struct sk_buff *skb; 1909 int err = 0; 1910 1911 if (!cops->delete) 1912 return -EOPNOTSUPP; 1913 1914 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1915 if (!skb) 1916 return -ENOBUFS; 1917 1918 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, 1919 RTM_DELTCLASS, extack) < 0) { 1920 kfree_skb(skb); 1921 return -EINVAL; 1922 } 1923 1924 err = cops->delete(q, cl, extack); 1925 if (err) { 1926 kfree_skb(skb); 1927 return err; 1928 } 1929 1930 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1931 n->nlmsg_flags & NLM_F_ECHO); 1932 return err; 1933 } 1934 1935 #ifdef CONFIG_NET_CLS 1936 1937 struct tcf_bind_args { 1938 struct tcf_walker w; 1939 unsigned long base; 1940 unsigned long cl; 1941 u32 classid; 1942 }; 1943 1944 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 1945 { 1946 struct tcf_bind_args *a = (void *)arg; 1947 1948 if (n && tp->ops->bind_class) { 1949 struct Qdisc *q = tcf_block_q(tp->chain->block); 1950 1951 sch_tree_lock(q); 1952 tp->ops->bind_class(n, a->classid, a->cl, q, a->base); 1953 sch_tree_unlock(q); 1954 } 1955 return 0; 1956 } 1957 1958 struct tc_bind_class_args { 1959 struct qdisc_walker w; 1960 unsigned long new_cl; 1961 u32 portid; 1962 u32 clid; 1963 }; 1964 1965 static int tc_bind_class_walker(struct Qdisc *q, unsigned long cl, 1966 struct qdisc_walker *w) 1967 { 1968 struct tc_bind_class_args *a = (struct tc_bind_class_args *)w; 1969 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1970 struct tcf_block *block; 1971 struct tcf_chain *chain; 1972 1973 block = cops->tcf_block(q, cl, NULL); 1974 if (!block) 1975 return 0; 1976 for (chain = tcf_get_next_chain(block, NULL); 1977 chain; 1978 chain = tcf_get_next_chain(block, chain)) { 1979 struct tcf_proto *tp; 1980 1981 for (tp = tcf_get_next_proto(chain, NULL); 1982 tp; tp = tcf_get_next_proto(chain, tp)) { 1983 struct tcf_bind_args arg = {}; 1984 1985 arg.w.fn = tcf_node_bind; 1986 arg.classid = a->clid; 1987 arg.base = cl; 1988 arg.cl = a->new_cl; 1989 tp->ops->walk(tp, &arg.w, true); 1990 } 1991 } 1992 1993 return 0; 1994 } 1995 1996 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid, 1997 unsigned long new_cl) 1998 { 1999 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 2000 struct tc_bind_class_args args = {}; 2001 2002 if (!cops->tcf_block) 2003 return; 2004 args.portid = portid; 2005 args.clid = clid; 2006 args.new_cl = new_cl; 2007 args.w.fn = tc_bind_class_walker; 2008 q->ops->cl_ops->walk(q, &args.w); 2009 } 2010 2011 #else 2012 2013 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid, 2014 unsigned long new_cl) 2015 { 2016 } 2017 2018 #endif 2019 2020 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, 2021 struct netlink_ext_ack *extack) 2022 { 2023 struct net *net = sock_net(skb->sk); 2024 struct tcmsg *tcm = nlmsg_data(n); 2025 struct nlattr *tca[TCA_MAX + 1]; 2026 struct net_device *dev; 2027 struct Qdisc *q = NULL; 2028 const struct Qdisc_class_ops *cops; 2029 unsigned long cl = 0; 2030 unsigned long new_cl; 2031 u32 portid; 2032 u32 clid; 2033 u32 qid; 2034 int err; 2035 2036 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, 2037 rtm_tca_policy, extack); 2038 if (err < 0) 2039 return err; 2040 2041 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2042 if (!dev) 2043 return -ENODEV; 2044 2045 /* 2046 parent == TC_H_UNSPEC - unspecified parent. 2047 parent == TC_H_ROOT - class is root, which has no parent. 2048 parent == X:0 - parent is root class. 2049 parent == X:Y - parent is a node in hierarchy. 2050 parent == 0:Y - parent is X:Y, where X:0 is qdisc. 2051 2052 handle == 0:0 - generate handle from kernel pool. 2053 handle == 0:Y - class is X:Y, where X:0 is qdisc. 2054 handle == X:Y - clear. 2055 handle == X:0 - root class. 2056 */ 2057 2058 /* Step 1. Determine qdisc handle X:0 */ 2059 2060 portid = tcm->tcm_parent; 2061 clid = tcm->tcm_handle; 2062 qid = TC_H_MAJ(clid); 2063 2064 if (portid != TC_H_ROOT) { 2065 u32 qid1 = TC_H_MAJ(portid); 2066 2067 if (qid && qid1) { 2068 /* If both majors are known, they must be identical. */ 2069 if (qid != qid1) 2070 return -EINVAL; 2071 } else if (qid1) { 2072 qid = qid1; 2073 } else if (qid == 0) 2074 qid = rtnl_dereference(dev->qdisc)->handle; 2075 2076 /* Now qid is genuine qdisc handle consistent 2077 * both with parent and child. 2078 * 2079 * TC_H_MAJ(portid) still may be unspecified, complete it now. 2080 */ 2081 if (portid) 2082 portid = TC_H_MAKE(qid, portid); 2083 } else { 2084 if (qid == 0) 2085 qid = rtnl_dereference(dev->qdisc)->handle; 2086 } 2087 2088 /* OK. Locate qdisc */ 2089 q = qdisc_lookup(dev, qid); 2090 if (!q) 2091 return -ENOENT; 2092 2093 /* An check that it supports classes */ 2094 cops = q->ops->cl_ops; 2095 if (cops == NULL) 2096 return -EINVAL; 2097 2098 /* Now try to get class */ 2099 if (clid == 0) { 2100 if (portid == TC_H_ROOT) 2101 clid = qid; 2102 } else 2103 clid = TC_H_MAKE(qid, clid); 2104 2105 if (clid) 2106 cl = cops->find(q, clid); 2107 2108 if (cl == 0) { 2109 err = -ENOENT; 2110 if (n->nlmsg_type != RTM_NEWTCLASS || 2111 !(n->nlmsg_flags & NLM_F_CREATE)) 2112 goto out; 2113 } else { 2114 switch (n->nlmsg_type) { 2115 case RTM_NEWTCLASS: 2116 err = -EEXIST; 2117 if (n->nlmsg_flags & NLM_F_EXCL) 2118 goto out; 2119 break; 2120 case RTM_DELTCLASS: 2121 err = tclass_del_notify(net, cops, skb, n, q, cl, extack); 2122 /* Unbind the class with flilters with 0 */ 2123 tc_bind_tclass(q, portid, clid, 0); 2124 goto out; 2125 case RTM_GETTCLASS: 2126 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS, extack); 2127 goto out; 2128 default: 2129 err = -EINVAL; 2130 goto out; 2131 } 2132 } 2133 2134 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) { 2135 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes"); 2136 return -EOPNOTSUPP; 2137 } 2138 2139 new_cl = cl; 2140 err = -EOPNOTSUPP; 2141 if (cops->change) 2142 err = cops->change(q, clid, portid, tca, &new_cl, extack); 2143 if (err == 0) { 2144 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS, extack); 2145 /* We just create a new class, need to do reverse binding. */ 2146 if (cl != new_cl) 2147 tc_bind_tclass(q, portid, clid, new_cl); 2148 } 2149 out: 2150 return err; 2151 } 2152 2153 struct qdisc_dump_args { 2154 struct qdisc_walker w; 2155 struct sk_buff *skb; 2156 struct netlink_callback *cb; 2157 }; 2158 2159 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, 2160 struct qdisc_walker *arg) 2161 { 2162 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg; 2163 2164 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid, 2165 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2166 RTM_NEWTCLASS, NULL); 2167 } 2168 2169 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb, 2170 struct tcmsg *tcm, struct netlink_callback *cb, 2171 int *t_p, int s_t) 2172 { 2173 struct qdisc_dump_args arg; 2174 2175 if (tc_qdisc_dump_ignore(q, false) || 2176 *t_p < s_t || !q->ops->cl_ops || 2177 (tcm->tcm_parent && 2178 TC_H_MAJ(tcm->tcm_parent) != q->handle)) { 2179 (*t_p)++; 2180 return 0; 2181 } 2182 if (*t_p > s_t) 2183 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0])); 2184 arg.w.fn = qdisc_class_dump; 2185 arg.skb = skb; 2186 arg.cb = cb; 2187 arg.w.stop = 0; 2188 arg.w.skip = cb->args[1]; 2189 arg.w.count = 0; 2190 q->ops->cl_ops->walk(q, &arg.w); 2191 cb->args[1] = arg.w.count; 2192 if (arg.w.stop) 2193 return -1; 2194 (*t_p)++; 2195 return 0; 2196 } 2197 2198 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, 2199 struct tcmsg *tcm, struct netlink_callback *cb, 2200 int *t_p, int s_t, bool recur) 2201 { 2202 struct Qdisc *q; 2203 int b; 2204 2205 if (!root) 2206 return 0; 2207 2208 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0) 2209 return -1; 2210 2211 if (!qdisc_dev(root) || !recur) 2212 return 0; 2213 2214 if (tcm->tcm_parent) { 2215 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent)); 2216 if (q && q != root && 2217 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 2218 return -1; 2219 return 0; 2220 } 2221 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) { 2222 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 2223 return -1; 2224 } 2225 2226 return 0; 2227 } 2228 2229 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) 2230 { 2231 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2232 struct net *net = sock_net(skb->sk); 2233 struct netdev_queue *dev_queue; 2234 struct net_device *dev; 2235 int t, s_t; 2236 2237 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2238 return 0; 2239 dev = dev_get_by_index(net, tcm->tcm_ifindex); 2240 if (!dev) 2241 return 0; 2242 2243 s_t = cb->args[0]; 2244 t = 0; 2245 2246 if (tc_dump_tclass_root(rtnl_dereference(dev->qdisc), 2247 skb, tcm, cb, &t, s_t, true) < 0) 2248 goto done; 2249 2250 dev_queue = dev_ingress_queue(dev); 2251 if (dev_queue && 2252 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb, 2253 &t, s_t, false) < 0) 2254 goto done; 2255 2256 done: 2257 cb->args[0] = t; 2258 2259 dev_put(dev); 2260 return skb->len; 2261 } 2262 2263 #ifdef CONFIG_PROC_FS 2264 static int psched_show(struct seq_file *seq, void *v) 2265 { 2266 seq_printf(seq, "%08x %08x %08x %08x\n", 2267 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1), 2268 1000000, 2269 (u32)NSEC_PER_SEC / hrtimer_resolution); 2270 2271 return 0; 2272 } 2273 2274 static int __net_init psched_net_init(struct net *net) 2275 { 2276 struct proc_dir_entry *e; 2277 2278 e = proc_create_single("psched", 0, net->proc_net, psched_show); 2279 if (e == NULL) 2280 return -ENOMEM; 2281 2282 return 0; 2283 } 2284 2285 static void __net_exit psched_net_exit(struct net *net) 2286 { 2287 remove_proc_entry("psched", net->proc_net); 2288 } 2289 #else 2290 static int __net_init psched_net_init(struct net *net) 2291 { 2292 return 0; 2293 } 2294 2295 static void __net_exit psched_net_exit(struct net *net) 2296 { 2297 } 2298 #endif 2299 2300 static struct pernet_operations psched_net_ops = { 2301 .init = psched_net_init, 2302 .exit = psched_net_exit, 2303 }; 2304 2305 DEFINE_STATIC_KEY_FALSE(tc_skip_wrapper); 2306 2307 static int __init pktsched_init(void) 2308 { 2309 int err; 2310 2311 err = register_pernet_subsys(&psched_net_ops); 2312 if (err) { 2313 pr_err("pktsched_init: " 2314 "cannot initialize per netns operations\n"); 2315 return err; 2316 } 2317 2318 register_qdisc(&pfifo_fast_ops); 2319 register_qdisc(&pfifo_qdisc_ops); 2320 register_qdisc(&bfifo_qdisc_ops); 2321 register_qdisc(&pfifo_head_drop_qdisc_ops); 2322 register_qdisc(&mq_qdisc_ops); 2323 register_qdisc(&noqueue_qdisc_ops); 2324 2325 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0); 2326 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0); 2327 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc, 2328 0); 2329 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0); 2330 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0); 2331 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass, 2332 0); 2333 2334 tc_wrapper_init(); 2335 2336 return 0; 2337 } 2338 2339 subsys_initcall(pktsched_init); 2340