1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/rtnetlink.h> 5 #include <net/pkt_cls.h> 6 #include <net/pkt_sched.h> 7 #include <net/red.h> 8 9 #include "../nfpcore/nfp_cpp.h" 10 #include "../nfp_app.h" 11 #include "../nfp_main.h" 12 #include "../nfp_net.h" 13 #include "../nfp_port.h" 14 #include "main.h" 15 16 static bool nfp_abm_qdisc_is_red(struct nfp_qdisc *qdisc) 17 { 18 return qdisc->type == NFP_QDISC_RED || qdisc->type == NFP_QDISC_GRED; 19 } 20 21 static bool nfp_abm_qdisc_child_valid(struct nfp_qdisc *qdisc, unsigned int id) 22 { 23 return qdisc->children[id] && 24 qdisc->children[id] != NFP_QDISC_UNTRACKED; 25 } 26 27 static void *nfp_abm_qdisc_tree_deref_slot(void __rcu **slot) 28 { 29 return rtnl_dereference(*slot); 30 } 31 32 static void 33 nfp_abm_stats_propagate(struct nfp_alink_stats *parent, 34 struct nfp_alink_stats *child) 35 { 36 parent->tx_pkts += child->tx_pkts; 37 parent->tx_bytes += child->tx_bytes; 38 parent->backlog_pkts += child->backlog_pkts; 39 parent->backlog_bytes += child->backlog_bytes; 40 parent->overlimits += child->overlimits; 41 parent->drops += child->drops; 42 } 43 44 static void 45 nfp_abm_stats_update_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, 46 unsigned int queue) 47 { 48 struct nfp_cpp *cpp = alink->abm->app->cpp; 49 unsigned int i; 50 int err; 51 52 if (!qdisc->offloaded) 53 return; 54 55 for (i = 0; i < qdisc->red.num_bands; i++) { 56 err = nfp_abm_ctrl_read_q_stats(alink, i, queue, 57 &qdisc->red.band[i].stats); 58 if (err) 59 nfp_err(cpp, "RED stats (%d, %d) read failed with error %d\n", 60 i, queue, err); 61 62 err = nfp_abm_ctrl_read_q_xstats(alink, i, queue, 63 &qdisc->red.band[i].xstats); 64 if (err) 65 nfp_err(cpp, "RED xstats (%d, %d) read failed with error %d\n", 66 i, queue, err); 67 } 68 } 69 70 static void 71 nfp_abm_stats_update_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) 72 { 73 unsigned int i; 74 75 if (qdisc->type != NFP_QDISC_MQ) 76 return; 77 78 for (i = 0; i < alink->total_queues; i++) 79 if (nfp_abm_qdisc_child_valid(qdisc, i)) 80 nfp_abm_stats_update_red(alink, qdisc->children[i], i); 81 } 82 83 static void __nfp_abm_stats_update(struct nfp_abm_link *alink, u64 time_now) 84 { 85 alink->last_stats_update = time_now; 86 if (alink->root_qdisc) 87 nfp_abm_stats_update_mq(alink, alink->root_qdisc); 88 } 89 90 static void nfp_abm_stats_update(struct nfp_abm_link *alink) 91 { 92 u64 now; 93 94 /* Limit the frequency of updates - stats of non-leaf qdiscs are a sum 95 * of all their leafs, so we would read the same stat multiple times 96 * for every dump. 97 */ 98 now = ktime_get(); 99 if (now - alink->last_stats_update < NFP_ABM_STATS_REFRESH_IVAL) 100 return; 101 102 __nfp_abm_stats_update(alink, now); 103 } 104 105 static void 106 nfp_abm_qdisc_unlink_children(struct nfp_qdisc *qdisc, 107 unsigned int start, unsigned int end) 108 { 109 unsigned int i; 110 111 for (i = start; i < end; i++) 112 if (nfp_abm_qdisc_child_valid(qdisc, i)) { 113 qdisc->children[i]->use_cnt--; 114 qdisc->children[i] = NULL; 115 } 116 } 117 118 static void 119 nfp_abm_qdisc_offload_stop(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) 120 { 121 unsigned int i; 122 123 /* Don't complain when qdisc is getting unlinked */ 124 if (qdisc->use_cnt) 125 nfp_warn(alink->abm->app->cpp, "Offload of '%08x' stopped\n", 126 qdisc->handle); 127 128 if (!nfp_abm_qdisc_is_red(qdisc)) 129 return; 130 131 for (i = 0; i < qdisc->red.num_bands; i++) { 132 qdisc->red.band[i].stats.backlog_pkts = 0; 133 qdisc->red.band[i].stats.backlog_bytes = 0; 134 } 135 } 136 137 static int 138 __nfp_abm_stats_init(struct nfp_abm_link *alink, unsigned int band, 139 unsigned int queue, struct nfp_alink_stats *prev_stats, 140 struct nfp_alink_xstats *prev_xstats) 141 { 142 u64 backlog_pkts, backlog_bytes; 143 int err; 144 145 /* Don't touch the backlog, backlog can only be reset after it has 146 * been reported back to the tc qdisc stats. 147 */ 148 backlog_pkts = prev_stats->backlog_pkts; 149 backlog_bytes = prev_stats->backlog_bytes; 150 151 err = nfp_abm_ctrl_read_q_stats(alink, band, queue, prev_stats); 152 if (err) { 153 nfp_err(alink->abm->app->cpp, 154 "RED stats init (%d, %d) failed with error %d\n", 155 band, queue, err); 156 return err; 157 } 158 159 err = nfp_abm_ctrl_read_q_xstats(alink, band, queue, prev_xstats); 160 if (err) { 161 nfp_err(alink->abm->app->cpp, 162 "RED xstats init (%d, %d) failed with error %d\n", 163 band, queue, err); 164 return err; 165 } 166 167 prev_stats->backlog_pkts = backlog_pkts; 168 prev_stats->backlog_bytes = backlog_bytes; 169 return 0; 170 } 171 172 static int 173 nfp_abm_stats_init(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, 174 unsigned int queue) 175 { 176 unsigned int i; 177 int err; 178 179 for (i = 0; i < qdisc->red.num_bands; i++) { 180 err = __nfp_abm_stats_init(alink, i, queue, 181 &qdisc->red.band[i].prev_stats, 182 &qdisc->red.band[i].prev_xstats); 183 if (err) 184 return err; 185 } 186 187 return 0; 188 } 189 190 static void 191 nfp_abm_offload_compile_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, 192 unsigned int queue) 193 { 194 bool good_red, good_gred; 195 unsigned int i; 196 197 good_red = qdisc->type == NFP_QDISC_RED && 198 qdisc->params_ok && 199 qdisc->use_cnt == 1 && 200 !alink->has_prio && 201 !qdisc->children[0]; 202 good_gred = qdisc->type == NFP_QDISC_GRED && 203 qdisc->params_ok && 204 qdisc->use_cnt == 1; 205 qdisc->offload_mark = good_red || good_gred; 206 207 /* If we are starting offload init prev_stats */ 208 if (qdisc->offload_mark && !qdisc->offloaded) 209 if (nfp_abm_stats_init(alink, qdisc, queue)) 210 qdisc->offload_mark = false; 211 212 if (!qdisc->offload_mark) 213 return; 214 215 for (i = 0; i < alink->abm->num_bands; i++) { 216 enum nfp_abm_q_action act; 217 218 nfp_abm_ctrl_set_q_lvl(alink, i, queue, 219 qdisc->red.band[i].threshold); 220 act = qdisc->red.band[i].ecn ? 221 NFP_ABM_ACT_MARK_DROP : NFP_ABM_ACT_DROP; 222 nfp_abm_ctrl_set_q_act(alink, i, queue, act); 223 } 224 } 225 226 static void 227 nfp_abm_offload_compile_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) 228 { 229 unsigned int i; 230 231 qdisc->offload_mark = qdisc->type == NFP_QDISC_MQ; 232 if (!qdisc->offload_mark) 233 return; 234 235 for (i = 0; i < alink->total_queues; i++) { 236 struct nfp_qdisc *child = qdisc->children[i]; 237 238 if (!nfp_abm_qdisc_child_valid(qdisc, i)) 239 continue; 240 241 nfp_abm_offload_compile_red(alink, child, i); 242 } 243 } 244 245 void nfp_abm_qdisc_offload_update(struct nfp_abm_link *alink) 246 { 247 struct nfp_abm *abm = alink->abm; 248 struct radix_tree_iter iter; 249 struct nfp_qdisc *qdisc; 250 void __rcu **slot; 251 size_t i; 252 253 /* Mark all thresholds as unconfigured */ 254 for (i = 0; i < abm->num_bands; i++) 255 __bitmap_set(abm->threshold_undef, 256 i * NFP_NET_MAX_RX_RINGS + alink->queue_base, 257 alink->total_queues); 258 259 /* Clear offload marks */ 260 radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { 261 qdisc = nfp_abm_qdisc_tree_deref_slot(slot); 262 qdisc->offload_mark = false; 263 } 264 265 if (alink->root_qdisc) 266 nfp_abm_offload_compile_mq(alink, alink->root_qdisc); 267 268 /* Refresh offload status */ 269 radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { 270 qdisc = nfp_abm_qdisc_tree_deref_slot(slot); 271 if (!qdisc->offload_mark && qdisc->offloaded) 272 nfp_abm_qdisc_offload_stop(alink, qdisc); 273 qdisc->offloaded = qdisc->offload_mark; 274 } 275 276 /* Reset the unconfigured thresholds */ 277 for (i = 0; i < abm->num_thresholds; i++) 278 if (test_bit(i, abm->threshold_undef)) 279 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); 280 281 __nfp_abm_stats_update(alink, ktime_get()); 282 } 283 284 static void 285 nfp_abm_qdisc_clear_mq(struct net_device *netdev, struct nfp_abm_link *alink, 286 struct nfp_qdisc *qdisc) 287 { 288 struct radix_tree_iter iter; 289 unsigned int mq_refs = 0; 290 void __rcu **slot; 291 292 if (!qdisc->use_cnt) 293 return; 294 /* MQ doesn't notify well on destruction, we need special handling of 295 * MQ's children. 296 */ 297 if (qdisc->type == NFP_QDISC_MQ && 298 qdisc == alink->root_qdisc && 299 netdev->reg_state == NETREG_UNREGISTERING) 300 return; 301 302 /* Count refs held by MQ instances and clear pointers */ 303 radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { 304 struct nfp_qdisc *mq = nfp_abm_qdisc_tree_deref_slot(slot); 305 unsigned int i; 306 307 if (mq->type != NFP_QDISC_MQ || mq->netdev != netdev) 308 continue; 309 for (i = 0; i < mq->num_children; i++) 310 if (mq->children[i] == qdisc) { 311 mq->children[i] = NULL; 312 mq_refs++; 313 } 314 } 315 316 WARN(qdisc->use_cnt != mq_refs, "non-zero qdisc use count: %d (- %d)\n", 317 qdisc->use_cnt, mq_refs); 318 } 319 320 static void 321 nfp_abm_qdisc_free(struct net_device *netdev, struct nfp_abm_link *alink, 322 struct nfp_qdisc *qdisc) 323 { 324 struct nfp_port *port = nfp_port_from_netdev(netdev); 325 326 if (!qdisc) 327 return; 328 nfp_abm_qdisc_clear_mq(netdev, alink, qdisc); 329 WARN_ON(radix_tree_delete(&alink->qdiscs, 330 TC_H_MAJ(qdisc->handle)) != qdisc); 331 332 kfree(qdisc->children); 333 kfree(qdisc); 334 335 port->tc_offload_cnt--; 336 } 337 338 static struct nfp_qdisc * 339 nfp_abm_qdisc_alloc(struct net_device *netdev, struct nfp_abm_link *alink, 340 enum nfp_qdisc_type type, u32 parent_handle, u32 handle, 341 unsigned int children) 342 { 343 struct nfp_port *port = nfp_port_from_netdev(netdev); 344 struct nfp_qdisc *qdisc; 345 int err; 346 347 qdisc = kzalloc(sizeof(*qdisc), GFP_KERNEL); 348 if (!qdisc) 349 return NULL; 350 351 if (children) { 352 qdisc->children = kcalloc(children, sizeof(void *), GFP_KERNEL); 353 if (!qdisc->children) 354 goto err_free_qdisc; 355 } 356 357 qdisc->netdev = netdev; 358 qdisc->type = type; 359 qdisc->parent_handle = parent_handle; 360 qdisc->handle = handle; 361 qdisc->num_children = children; 362 363 err = radix_tree_insert(&alink->qdiscs, TC_H_MAJ(qdisc->handle), qdisc); 364 if (err) { 365 nfp_err(alink->abm->app->cpp, 366 "Qdisc insertion into radix tree failed: %d\n", err); 367 goto err_free_child_tbl; 368 } 369 370 port->tc_offload_cnt++; 371 return qdisc; 372 373 err_free_child_tbl: 374 kfree(qdisc->children); 375 err_free_qdisc: 376 kfree(qdisc); 377 return NULL; 378 } 379 380 static struct nfp_qdisc * 381 nfp_abm_qdisc_find(struct nfp_abm_link *alink, u32 handle) 382 { 383 return radix_tree_lookup(&alink->qdiscs, TC_H_MAJ(handle)); 384 } 385 386 static int 387 nfp_abm_qdisc_replace(struct net_device *netdev, struct nfp_abm_link *alink, 388 enum nfp_qdisc_type type, u32 parent_handle, u32 handle, 389 unsigned int children, struct nfp_qdisc **qdisc) 390 { 391 *qdisc = nfp_abm_qdisc_find(alink, handle); 392 if (*qdisc) { 393 if (WARN_ON((*qdisc)->type != type)) 394 return -EINVAL; 395 return 1; 396 } 397 398 *qdisc = nfp_abm_qdisc_alloc(netdev, alink, type, parent_handle, handle, 399 children); 400 return *qdisc ? 0 : -ENOMEM; 401 } 402 403 static void 404 nfp_abm_qdisc_destroy(struct net_device *netdev, struct nfp_abm_link *alink, 405 u32 handle) 406 { 407 struct nfp_qdisc *qdisc; 408 409 qdisc = nfp_abm_qdisc_find(alink, handle); 410 if (!qdisc) 411 return; 412 413 /* We don't get TC_SETUP_ROOT_QDISC w/ MQ when netdev is unregistered */ 414 if (alink->root_qdisc == qdisc) 415 qdisc->use_cnt--; 416 417 nfp_abm_qdisc_unlink_children(qdisc, 0, qdisc->num_children); 418 nfp_abm_qdisc_free(netdev, alink, qdisc); 419 420 if (alink->root_qdisc == qdisc) { 421 alink->root_qdisc = NULL; 422 /* Only root change matters, other changes are acted upon on 423 * the graft notification. 424 */ 425 nfp_abm_qdisc_offload_update(alink); 426 } 427 } 428 429 static int 430 nfp_abm_qdisc_graft(struct nfp_abm_link *alink, u32 handle, u32 child_handle, 431 unsigned int id) 432 { 433 struct nfp_qdisc *parent, *child; 434 435 parent = nfp_abm_qdisc_find(alink, handle); 436 if (!parent) 437 return 0; 438 439 if (WARN(id >= parent->num_children, 440 "graft child out of bound %d >= %d\n", 441 id, parent->num_children)) 442 return -EINVAL; 443 444 nfp_abm_qdisc_unlink_children(parent, id, id + 1); 445 446 child = nfp_abm_qdisc_find(alink, child_handle); 447 if (child) 448 child->use_cnt++; 449 else 450 child = NFP_QDISC_UNTRACKED; 451 parent->children[id] = child; 452 453 nfp_abm_qdisc_offload_update(alink); 454 455 return 0; 456 } 457 458 static void 459 nfp_abm_stats_calculate(struct nfp_alink_stats *new, 460 struct nfp_alink_stats *old, 461 struct gnet_stats_basic_packed *bstats, 462 struct gnet_stats_queue *qstats) 463 { 464 _bstats_update(bstats, new->tx_bytes - old->tx_bytes, 465 new->tx_pkts - old->tx_pkts); 466 qstats->qlen += new->backlog_pkts - old->backlog_pkts; 467 qstats->backlog += new->backlog_bytes - old->backlog_bytes; 468 qstats->overlimits += new->overlimits - old->overlimits; 469 qstats->drops += new->drops - old->drops; 470 } 471 472 static void 473 nfp_abm_stats_red_calculate(struct nfp_alink_xstats *new, 474 struct nfp_alink_xstats *old, 475 struct red_stats *stats) 476 { 477 stats->forced_mark += new->ecn_marked - old->ecn_marked; 478 stats->pdrop += new->pdrop - old->pdrop; 479 } 480 481 static int 482 nfp_abm_gred_stats(struct nfp_abm_link *alink, u32 handle, 483 struct tc_gred_qopt_offload_stats *stats) 484 { 485 struct nfp_qdisc *qdisc; 486 unsigned int i; 487 488 nfp_abm_stats_update(alink); 489 490 qdisc = nfp_abm_qdisc_find(alink, handle); 491 if (!qdisc) 492 return -EOPNOTSUPP; 493 /* If the qdisc offload has stopped we may need to adjust the backlog 494 * counters back so carry on even if qdisc is not currently offloaded. 495 */ 496 497 for (i = 0; i < qdisc->red.num_bands; i++) { 498 if (!stats->xstats[i]) 499 continue; 500 501 nfp_abm_stats_calculate(&qdisc->red.band[i].stats, 502 &qdisc->red.band[i].prev_stats, 503 &stats->bstats[i], &stats->qstats[i]); 504 qdisc->red.band[i].prev_stats = qdisc->red.band[i].stats; 505 506 nfp_abm_stats_red_calculate(&qdisc->red.band[i].xstats, 507 &qdisc->red.band[i].prev_xstats, 508 stats->xstats[i]); 509 qdisc->red.band[i].prev_xstats = qdisc->red.band[i].xstats; 510 } 511 512 return qdisc->offloaded ? 0 : -EOPNOTSUPP; 513 } 514 515 static bool 516 nfp_abm_gred_check_params(struct nfp_abm_link *alink, 517 struct tc_gred_qopt_offload *opt) 518 { 519 struct nfp_cpp *cpp = alink->abm->app->cpp; 520 struct nfp_abm *abm = alink->abm; 521 unsigned int i; 522 523 if (opt->set.grio_on || opt->set.wred_on) { 524 nfp_warn(cpp, "GRED offload failed - GRIO and WRED not supported (p:%08x h:%08x)\n", 525 opt->parent, opt->handle); 526 return false; 527 } 528 if (opt->set.dp_def != alink->def_band) { 529 nfp_warn(cpp, "GRED offload failed - default band must be %d (p:%08x h:%08x)\n", 530 alink->def_band, opt->parent, opt->handle); 531 return false; 532 } 533 if (opt->set.dp_cnt != abm->num_bands) { 534 nfp_warn(cpp, "GRED offload failed - band count must be %d (p:%08x h:%08x)\n", 535 abm->num_bands, opt->parent, opt->handle); 536 return false; 537 } 538 539 for (i = 0; i < abm->num_bands; i++) { 540 struct tc_gred_vq_qopt_offload_params *band = &opt->set.tab[i]; 541 542 if (!band->present) 543 return false; 544 if (!band->is_ecn && !nfp_abm_has_drop(abm)) { 545 nfp_warn(cpp, "GRED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x vq:%d)\n", 546 opt->parent, opt->handle, i); 547 return false; 548 } 549 if (band->is_ecn && !nfp_abm_has_mark(abm)) { 550 nfp_warn(cpp, "GRED offload failed - ECN marking not supported (p:%08x h:%08x vq:%d)\n", 551 opt->parent, opt->handle, i); 552 return false; 553 } 554 if (band->is_harddrop) { 555 nfp_warn(cpp, "GRED offload failed - harddrop is not supported (p:%08x h:%08x vq:%d)\n", 556 opt->parent, opt->handle, i); 557 return false; 558 } 559 if (band->min != band->max) { 560 nfp_warn(cpp, "GRED offload failed - threshold mismatch (p:%08x h:%08x vq:%d)\n", 561 opt->parent, opt->handle, i); 562 return false; 563 } 564 if (band->min > S32_MAX) { 565 nfp_warn(cpp, "GRED offload failed - threshold too large %d > %d (p:%08x h:%08x vq:%d)\n", 566 band->min, S32_MAX, opt->parent, opt->handle, 567 i); 568 return false; 569 } 570 } 571 572 return true; 573 } 574 575 static int 576 nfp_abm_gred_replace(struct net_device *netdev, struct nfp_abm_link *alink, 577 struct tc_gred_qopt_offload *opt) 578 { 579 struct nfp_qdisc *qdisc; 580 unsigned int i; 581 int ret; 582 583 ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_GRED, opt->parent, 584 opt->handle, 0, &qdisc); 585 if (ret < 0) 586 return ret; 587 588 qdisc->params_ok = nfp_abm_gred_check_params(alink, opt); 589 if (qdisc->params_ok) { 590 qdisc->red.num_bands = opt->set.dp_cnt; 591 for (i = 0; i < qdisc->red.num_bands; i++) { 592 qdisc->red.band[i].ecn = opt->set.tab[i].is_ecn; 593 qdisc->red.band[i].threshold = opt->set.tab[i].min; 594 } 595 } 596 597 if (qdisc->use_cnt) 598 nfp_abm_qdisc_offload_update(alink); 599 600 return 0; 601 } 602 603 int nfp_abm_setup_tc_gred(struct net_device *netdev, struct nfp_abm_link *alink, 604 struct tc_gred_qopt_offload *opt) 605 { 606 switch (opt->command) { 607 case TC_GRED_REPLACE: 608 return nfp_abm_gred_replace(netdev, alink, opt); 609 case TC_GRED_DESTROY: 610 nfp_abm_qdisc_destroy(netdev, alink, opt->handle); 611 return 0; 612 case TC_GRED_STATS: 613 return nfp_abm_gred_stats(alink, opt->handle, &opt->stats); 614 default: 615 return -EOPNOTSUPP; 616 } 617 } 618 619 static int 620 nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) 621 { 622 struct nfp_qdisc *qdisc; 623 624 nfp_abm_stats_update(alink); 625 626 qdisc = nfp_abm_qdisc_find(alink, opt->handle); 627 if (!qdisc || !qdisc->offloaded) 628 return -EOPNOTSUPP; 629 630 nfp_abm_stats_red_calculate(&qdisc->red.band[0].xstats, 631 &qdisc->red.band[0].prev_xstats, 632 opt->xstats); 633 qdisc->red.band[0].prev_xstats = qdisc->red.band[0].xstats; 634 return 0; 635 } 636 637 static int 638 nfp_abm_red_stats(struct nfp_abm_link *alink, u32 handle, 639 struct tc_qopt_offload_stats *stats) 640 { 641 struct nfp_qdisc *qdisc; 642 643 nfp_abm_stats_update(alink); 644 645 qdisc = nfp_abm_qdisc_find(alink, handle); 646 if (!qdisc) 647 return -EOPNOTSUPP; 648 /* If the qdisc offload has stopped we may need to adjust the backlog 649 * counters back so carry on even if qdisc is not currently offloaded. 650 */ 651 652 nfp_abm_stats_calculate(&qdisc->red.band[0].stats, 653 &qdisc->red.band[0].prev_stats, 654 stats->bstats, stats->qstats); 655 qdisc->red.band[0].prev_stats = qdisc->red.band[0].stats; 656 657 return qdisc->offloaded ? 0 : -EOPNOTSUPP; 658 } 659 660 static bool 661 nfp_abm_red_check_params(struct nfp_abm_link *alink, 662 struct tc_red_qopt_offload *opt) 663 { 664 struct nfp_cpp *cpp = alink->abm->app->cpp; 665 struct nfp_abm *abm = alink->abm; 666 667 if (!opt->set.is_ecn && !nfp_abm_has_drop(abm)) { 668 nfp_warn(cpp, "RED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x)\n", 669 opt->parent, opt->handle); 670 return false; 671 } 672 if (opt->set.is_ecn && !nfp_abm_has_mark(abm)) { 673 nfp_warn(cpp, "RED offload failed - ECN marking not supported (p:%08x h:%08x)\n", 674 opt->parent, opt->handle); 675 return false; 676 } 677 if (opt->set.is_harddrop) { 678 nfp_warn(cpp, "RED offload failed - harddrop is not supported (p:%08x h:%08x)\n", 679 opt->parent, opt->handle); 680 return false; 681 } 682 if (opt->set.min != opt->set.max) { 683 nfp_warn(cpp, "RED offload failed - unsupported min/max parameters (p:%08x h:%08x)\n", 684 opt->parent, opt->handle); 685 return false; 686 } 687 if (opt->set.min > NFP_ABM_LVL_INFINITY) { 688 nfp_warn(cpp, "RED offload failed - threshold too large %d > %d (p:%08x h:%08x)\n", 689 opt->set.min, NFP_ABM_LVL_INFINITY, opt->parent, 690 opt->handle); 691 return false; 692 } 693 694 return true; 695 } 696 697 static int 698 nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink, 699 struct tc_red_qopt_offload *opt) 700 { 701 struct nfp_qdisc *qdisc; 702 int ret; 703 704 ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_RED, opt->parent, 705 opt->handle, 1, &qdisc); 706 if (ret < 0) 707 return ret; 708 709 /* If limit != 0 child gets reset */ 710 if (opt->set.limit) { 711 if (nfp_abm_qdisc_child_valid(qdisc, 0)) 712 qdisc->children[0]->use_cnt--; 713 qdisc->children[0] = NULL; 714 } else { 715 /* Qdisc was just allocated without a limit will use noop_qdisc, 716 * i.e. a block hole. 717 */ 718 if (!ret) 719 qdisc->children[0] = NFP_QDISC_UNTRACKED; 720 } 721 722 qdisc->params_ok = nfp_abm_red_check_params(alink, opt); 723 if (qdisc->params_ok) { 724 qdisc->red.num_bands = 1; 725 qdisc->red.band[0].ecn = opt->set.is_ecn; 726 qdisc->red.band[0].threshold = opt->set.min; 727 } 728 729 if (qdisc->use_cnt == 1) 730 nfp_abm_qdisc_offload_update(alink); 731 732 return 0; 733 } 734 735 int nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, 736 struct tc_red_qopt_offload *opt) 737 { 738 switch (opt->command) { 739 case TC_RED_REPLACE: 740 return nfp_abm_red_replace(netdev, alink, opt); 741 case TC_RED_DESTROY: 742 nfp_abm_qdisc_destroy(netdev, alink, opt->handle); 743 return 0; 744 case TC_RED_STATS: 745 return nfp_abm_red_stats(alink, opt->handle, &opt->stats); 746 case TC_RED_XSTATS: 747 return nfp_abm_red_xstats(alink, opt); 748 case TC_RED_GRAFT: 749 return nfp_abm_qdisc_graft(alink, opt->handle, 750 opt->child_handle, 0); 751 default: 752 return -EOPNOTSUPP; 753 } 754 } 755 756 static int 757 nfp_abm_mq_create(struct net_device *netdev, struct nfp_abm_link *alink, 758 struct tc_mq_qopt_offload *opt) 759 { 760 struct nfp_qdisc *qdisc; 761 int ret; 762 763 ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_MQ, 764 TC_H_ROOT, opt->handle, alink->total_queues, 765 &qdisc); 766 if (ret < 0) 767 return ret; 768 769 qdisc->params_ok = true; 770 qdisc->offloaded = true; 771 nfp_abm_qdisc_offload_update(alink); 772 return 0; 773 } 774 775 static int 776 nfp_abm_mq_stats(struct nfp_abm_link *alink, u32 handle, 777 struct tc_qopt_offload_stats *stats) 778 { 779 struct nfp_qdisc *qdisc, *red; 780 unsigned int i, j; 781 782 qdisc = nfp_abm_qdisc_find(alink, handle); 783 if (!qdisc) 784 return -EOPNOTSUPP; 785 786 nfp_abm_stats_update(alink); 787 788 /* MQ stats are summed over the children in the core, so we need 789 * to add up the unreported child values. 790 */ 791 memset(&qdisc->mq.stats, 0, sizeof(qdisc->mq.stats)); 792 memset(&qdisc->mq.prev_stats, 0, sizeof(qdisc->mq.prev_stats)); 793 794 for (i = 0; i < qdisc->num_children; i++) { 795 if (!nfp_abm_qdisc_child_valid(qdisc, i)) 796 continue; 797 798 if (!nfp_abm_qdisc_is_red(qdisc->children[i])) 799 continue; 800 red = qdisc->children[i]; 801 802 for (j = 0; j < red->red.num_bands; j++) { 803 nfp_abm_stats_propagate(&qdisc->mq.stats, 804 &red->red.band[j].stats); 805 nfp_abm_stats_propagate(&qdisc->mq.prev_stats, 806 &red->red.band[j].prev_stats); 807 } 808 } 809 810 nfp_abm_stats_calculate(&qdisc->mq.stats, &qdisc->mq.prev_stats, 811 stats->bstats, stats->qstats); 812 813 return qdisc->offloaded ? 0 : -EOPNOTSUPP; 814 } 815 816 int nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink, 817 struct tc_mq_qopt_offload *opt) 818 { 819 switch (opt->command) { 820 case TC_MQ_CREATE: 821 return nfp_abm_mq_create(netdev, alink, opt); 822 case TC_MQ_DESTROY: 823 nfp_abm_qdisc_destroy(netdev, alink, opt->handle); 824 return 0; 825 case TC_MQ_STATS: 826 return nfp_abm_mq_stats(alink, opt->handle, &opt->stats); 827 case TC_MQ_GRAFT: 828 return nfp_abm_qdisc_graft(alink, opt->handle, 829 opt->graft_params.child_handle, 830 opt->graft_params.queue); 831 default: 832 return -EOPNOTSUPP; 833 } 834 } 835 836 int nfp_abm_setup_root(struct net_device *netdev, struct nfp_abm_link *alink, 837 struct tc_root_qopt_offload *opt) 838 { 839 if (opt->ingress) 840 return -EOPNOTSUPP; 841 if (alink->root_qdisc) 842 alink->root_qdisc->use_cnt--; 843 alink->root_qdisc = nfp_abm_qdisc_find(alink, opt->handle); 844 if (alink->root_qdisc) 845 alink->root_qdisc->use_cnt++; 846 847 nfp_abm_qdisc_offload_update(alink); 848 849 return 0; 850 } 851