1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 #include <linux/math64.h> 5 #include "lib/aso.h" 6 #include "en/tc/post_act.h" 7 #include "meter.h" 8 #include "en/tc_priv.h" 9 10 #define MLX5_START_COLOR_SHIFT 28 11 #define MLX5_METER_MODE_SHIFT 24 12 #define MLX5_CBS_EXP_SHIFT 24 13 #define MLX5_CBS_MAN_SHIFT 16 14 #define MLX5_CIR_EXP_SHIFT 8 15 16 /* cir = 8*(10^9)*cir_mantissa/(2^cir_exponent)) bits/s */ 17 #define MLX5_CONST_CIR 8000000000ULL 18 #define MLX5_CALC_CIR(m, e) ((MLX5_CONST_CIR * (m)) >> (e)) 19 #define MLX5_MAX_CIR ((MLX5_CONST_CIR * 0x100) - 1) 20 21 /* cbs = cbs_mantissa*2^cbs_exponent */ 22 #define MLX5_CALC_CBS(m, e) ((m) << (e)) 23 #define MLX5_MAX_CBS ((0x100ULL << 0x1F) - 1) 24 #define MLX5_MAX_HW_CBS 0x7FFFFFFF 25 26 struct mlx5e_flow_meter_aso_obj { 27 struct list_head entry; 28 int base_id; 29 int total_meters; 30 31 unsigned long meters_map[0]; /* must be at the end of this struct */ 32 }; 33 34 struct mlx5e_flow_meters { 35 enum mlx5_flow_namespace_type ns_type; 36 struct mlx5_aso *aso; 37 struct mutex aso_lock; /* Protects aso operations */ 38 int log_granularity; 39 u32 pdn; 40 41 DECLARE_HASHTABLE(hashtbl, 8); 42 43 struct mutex sync_lock; /* protect flow meter operations */ 44 struct list_head partial_list; 45 struct list_head full_list; 46 47 struct mlx5_core_dev *mdev; 48 struct mlx5e_post_act *post_act; 49 }; 50 51 static void 52 mlx5e_flow_meter_cir_calc(u64 cir, u8 *man, u8 *exp) 53 { 54 s64 _cir, _delta, delta = S64_MAX; 55 u8 e, _man = 0, _exp = 0; 56 u64 m; 57 58 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 59 m = cir << e; 60 if ((s64)m < 0) /* overflow */ 61 break; 62 m = div64_u64(m, MLX5_CONST_CIR); 63 if (m > 0xFF) /* man width 8 bit */ 64 continue; 65 _cir = MLX5_CALC_CIR(m, e); 66 _delta = cir - _cir; 67 if (_delta < delta) { 68 _man = m; 69 _exp = e; 70 if (!_delta) 71 goto found; 72 delta = _delta; 73 } 74 } 75 76 found: 77 *man = _man; 78 *exp = _exp; 79 } 80 81 static void 82 mlx5e_flow_meter_cbs_calc(u64 cbs, u8 *man, u8 *exp) 83 { 84 s64 _cbs, _delta, delta = S64_MAX; 85 u8 e, _man = 0, _exp = 0; 86 u64 m; 87 88 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 89 m = cbs >> e; 90 if (m > 0xFF) /* man width 8 bit */ 91 continue; 92 _cbs = MLX5_CALC_CBS(m, e); 93 _delta = cbs - _cbs; 94 if (_delta < delta) { 95 _man = m; 96 _exp = e; 97 if (!_delta) 98 goto found; 99 delta = _delta; 100 } 101 } 102 103 found: 104 *man = _man; 105 *exp = _exp; 106 } 107 108 int 109 mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, 110 struct mlx5e_flow_meter_handle *meter, 111 struct mlx5e_flow_meter_params *meter_params) 112 { 113 struct mlx5_wqe_aso_ctrl_seg *aso_ctrl; 114 struct mlx5_wqe_aso_data_seg *aso_data; 115 struct mlx5e_flow_meters *flow_meters; 116 u8 cir_man, cir_exp, cbs_man, cbs_exp; 117 struct mlx5_aso_wqe *aso_wqe; 118 unsigned long expires; 119 struct mlx5_aso *aso; 120 u64 rate, burst; 121 u8 ds_cnt; 122 int err; 123 124 rate = meter_params->rate; 125 burst = meter_params->burst; 126 127 /* HW treats each packet as 128 bytes in PPS mode */ 128 if (meter_params->mode == MLX5_RATE_LIMIT_PPS) { 129 rate <<= 10; 130 burst <<= 7; 131 } 132 133 if (!rate || rate > MLX5_MAX_CIR || !burst || burst > MLX5_MAX_CBS) 134 return -EINVAL; 135 136 /* HW has limitation of total 31 bits for cbs */ 137 if (burst > MLX5_MAX_HW_CBS) { 138 mlx5_core_warn(mdev, 139 "burst(%lld) is too large, use HW allowed value(%d)\n", 140 burst, MLX5_MAX_HW_CBS); 141 burst = MLX5_MAX_HW_CBS; 142 } 143 144 mlx5_core_dbg(mdev, "meter mode=%d\n", meter_params->mode); 145 mlx5e_flow_meter_cir_calc(rate, &cir_man, &cir_exp); 146 mlx5_core_dbg(mdev, "rate=%lld, cir=%lld, exp=%d, man=%d\n", 147 rate, MLX5_CALC_CIR(cir_man, cir_exp), cir_exp, cir_man); 148 mlx5e_flow_meter_cbs_calc(burst, &cbs_man, &cbs_exp); 149 mlx5_core_dbg(mdev, "burst=%lld, cbs=%lld, exp=%d, man=%d\n", 150 burst, MLX5_CALC_CBS((u64)cbs_man, cbs_exp), cbs_exp, cbs_man); 151 152 if (!cir_man || !cbs_man) 153 return -EINVAL; 154 155 flow_meters = meter->flow_meters; 156 aso = flow_meters->aso; 157 158 mutex_lock(&flow_meters->aso_lock); 159 aso_wqe = mlx5_aso_get_wqe(aso); 160 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_DS); 161 mlx5_aso_build_wqe(aso, ds_cnt, aso_wqe, meter->obj_id, 162 MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); 163 164 aso_ctrl = &aso_wqe->aso_ctrl; 165 aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; 166 aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | 167 MLX5_ASO_ALWAYS_TRUE << 4; 168 aso_ctrl->data_offset_condition_operand = MLX5_ASO_LOGICAL_OR << 6; 169 aso_ctrl->data_mask = cpu_to_be64(0x80FFFFFFULL << (meter->idx ? 0 : 32)); 170 171 aso_data = (struct mlx5_wqe_aso_data_seg *)(aso_wqe + 1); 172 memset(aso_data, 0, sizeof(*aso_data)); 173 aso_data->bytewise_data[meter->idx * 8] = cpu_to_be32((0x1 << 31) | /* valid */ 174 (MLX5_FLOW_METER_COLOR_GREEN << MLX5_START_COLOR_SHIFT)); 175 if (meter_params->mode == MLX5_RATE_LIMIT_PPS) 176 aso_data->bytewise_data[meter->idx * 8] |= 177 cpu_to_be32(MLX5_FLOW_METER_MODE_NUM_PACKETS << MLX5_METER_MODE_SHIFT); 178 else 179 aso_data->bytewise_data[meter->idx * 8] |= 180 cpu_to_be32(MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH << MLX5_METER_MODE_SHIFT); 181 182 aso_data->bytewise_data[meter->idx * 8 + 2] = cpu_to_be32((cbs_exp << MLX5_CBS_EXP_SHIFT) | 183 (cbs_man << MLX5_CBS_MAN_SHIFT) | 184 (cir_exp << MLX5_CIR_EXP_SHIFT) | 185 cir_man); 186 187 mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); 188 189 /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ 190 expires = jiffies + msecs_to_jiffies(10); 191 do { 192 err = mlx5_aso_poll_cq(aso, true); 193 if (err) 194 usleep_range(2, 10); 195 } while (err && time_is_after_jiffies(expires)); 196 mutex_unlock(&flow_meters->aso_lock); 197 198 return err; 199 } 200 201 static int 202 mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_id) 203 { 204 u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {}; 205 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 206 struct mlx5_core_dev *mdev = flow_meters->mdev; 207 void *obj; 208 int err; 209 210 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 211 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 212 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); 213 MLX5_SET(general_obj_in_cmd_hdr, in, log_obj_range, flow_meters->log_granularity); 214 215 obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj); 216 MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn); 217 218 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 219 if (!err) { 220 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 221 mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) created\n", *obj_id); 222 } 223 224 return err; 225 } 226 227 static void 228 mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id) 229 { 230 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 231 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 232 233 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 234 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 235 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); 236 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); 237 238 mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 239 mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) destroyed\n", obj_id); 240 } 241 242 static struct mlx5e_flow_meter_handle * 243 __mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters, bool alloc_aso) 244 { 245 struct mlx5_core_dev *mdev = flow_meters->mdev; 246 struct mlx5e_flow_meter_aso_obj *meters_obj; 247 struct mlx5e_flow_meter_handle *meter; 248 struct mlx5_fc *counter; 249 int err, pos, total; 250 u32 id; 251 252 meter = kzalloc(sizeof(*meter), GFP_KERNEL); 253 if (!meter) 254 return ERR_PTR(-ENOMEM); 255 256 counter = mlx5_fc_create(mdev, true); 257 if (IS_ERR(counter)) { 258 err = PTR_ERR(counter); 259 goto err_drop_counter; 260 } 261 meter->drop_counter = counter; 262 263 counter = mlx5_fc_create(mdev, true); 264 if (IS_ERR(counter)) { 265 err = PTR_ERR(counter); 266 goto err_act_counter; 267 } 268 meter->act_counter = counter; 269 270 if (!alloc_aso) 271 goto no_aso; 272 273 meters_obj = list_first_entry_or_null(&flow_meters->partial_list, 274 struct mlx5e_flow_meter_aso_obj, 275 entry); 276 /* 2 meters in one object */ 277 total = 1 << (flow_meters->log_granularity + 1); 278 if (!meters_obj) { 279 err = mlx5e_flow_meter_create_aso_obj(flow_meters, &id); 280 if (err) { 281 mlx5_core_err(mdev, "Failed to create flow meter ASO object\n"); 282 goto err_create; 283 } 284 285 meters_obj = kzalloc(sizeof(*meters_obj) + BITS_TO_BYTES(total), 286 GFP_KERNEL); 287 if (!meters_obj) { 288 err = -ENOMEM; 289 goto err_mem; 290 } 291 292 meters_obj->base_id = id; 293 meters_obj->total_meters = total; 294 list_add(&meters_obj->entry, &flow_meters->partial_list); 295 pos = 0; 296 } else { 297 pos = find_first_zero_bit(meters_obj->meters_map, total); 298 if (bitmap_weight(meters_obj->meters_map, total) == total - 1) { 299 list_del(&meters_obj->entry); 300 list_add(&meters_obj->entry, &flow_meters->full_list); 301 } 302 } 303 304 bitmap_set(meters_obj->meters_map, pos, 1); 305 meter->meters_obj = meters_obj; 306 meter->obj_id = meters_obj->base_id + pos / 2; 307 meter->idx = pos % 2; 308 309 no_aso: 310 meter->flow_meters = flow_meters; 311 mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n", 312 meter->obj_id, meter->idx); 313 314 return meter; 315 316 err_mem: 317 mlx5e_flow_meter_destroy_aso_obj(mdev, id); 318 err_create: 319 mlx5_fc_destroy(mdev, meter->act_counter); 320 err_act_counter: 321 mlx5_fc_destroy(mdev, meter->drop_counter); 322 err_drop_counter: 323 kfree(meter); 324 return ERR_PTR(err); 325 } 326 327 static void 328 __mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter) 329 { 330 struct mlx5e_flow_meters *flow_meters = meter->flow_meters; 331 struct mlx5_core_dev *mdev = flow_meters->mdev; 332 struct mlx5e_flow_meter_aso_obj *meters_obj; 333 int n, pos; 334 335 mlx5_fc_destroy(mdev, meter->act_counter); 336 mlx5_fc_destroy(mdev, meter->drop_counter); 337 338 if (meter->params.mtu) 339 goto out_no_aso; 340 341 meters_obj = meter->meters_obj; 342 pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx; 343 bitmap_clear(meters_obj->meters_map, pos, 1); 344 n = bitmap_weight(meters_obj->meters_map, meters_obj->total_meters); 345 if (n == 0) { 346 list_del(&meters_obj->entry); 347 mlx5e_flow_meter_destroy_aso_obj(mdev, meters_obj->base_id); 348 kfree(meters_obj); 349 } else if (n == meters_obj->total_meters - 1) { 350 list_del(&meters_obj->entry); 351 list_add(&meters_obj->entry, &flow_meters->partial_list); 352 } 353 354 out_no_aso: 355 mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n", 356 meter->obj_id, meter->idx); 357 kfree(meter); 358 } 359 360 static struct mlx5e_flow_meter_handle * 361 __mlx5e_tc_meter_get(struct mlx5e_flow_meters *flow_meters, u32 index) 362 { 363 struct mlx5e_flow_meter_handle *meter; 364 365 hash_for_each_possible(flow_meters->hashtbl, meter, hlist, index) 366 if (meter->params.index == index) 367 goto add_ref; 368 369 return ERR_PTR(-ENOENT); 370 371 add_ref: 372 meter->refcnt++; 373 374 return meter; 375 } 376 377 struct mlx5e_flow_meter_handle * 378 mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) 379 { 380 struct mlx5e_flow_meters *flow_meters; 381 struct mlx5e_flow_meter_handle *meter; 382 383 flow_meters = mlx5e_get_flow_meters(mdev); 384 if (!flow_meters) 385 return ERR_PTR(-EOPNOTSUPP); 386 387 mutex_lock(&flow_meters->sync_lock); 388 meter = __mlx5e_tc_meter_get(flow_meters, params->index); 389 mutex_unlock(&flow_meters->sync_lock); 390 391 return meter; 392 } 393 394 static void 395 __mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) 396 { 397 if (--meter->refcnt == 0) { 398 hash_del(&meter->hlist); 399 __mlx5e_flow_meter_free(meter); 400 } 401 } 402 403 void 404 mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) 405 { 406 struct mlx5e_flow_meters *flow_meters = meter->flow_meters; 407 408 mutex_lock(&flow_meters->sync_lock); 409 __mlx5e_tc_meter_put(meter); 410 mutex_unlock(&flow_meters->sync_lock); 411 } 412 413 static struct mlx5e_flow_meter_handle * 414 mlx5e_tc_meter_alloc(struct mlx5e_flow_meters *flow_meters, 415 struct mlx5e_flow_meter_params *params) 416 { 417 struct mlx5e_flow_meter_handle *meter; 418 419 meter = __mlx5e_flow_meter_alloc(flow_meters, !params->mtu); 420 if (IS_ERR(meter)) 421 return meter; 422 423 hash_add(flow_meters->hashtbl, &meter->hlist, params->index); 424 meter->params.index = params->index; 425 meter->params.mtu = params->mtu; 426 meter->refcnt++; 427 428 return meter; 429 } 430 431 static int 432 __mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, 433 struct mlx5e_flow_meter_params *params) 434 { 435 struct mlx5_core_dev *mdev = meter->flow_meters->mdev; 436 int err = 0; 437 438 if (meter->params.mode != params->mode || meter->params.rate != params->rate || 439 meter->params.burst != params->burst) { 440 err = mlx5e_tc_meter_modify(mdev, meter, params); 441 if (err) 442 goto out; 443 444 meter->params.mode = params->mode; 445 meter->params.rate = params->rate; 446 meter->params.burst = params->burst; 447 } 448 449 out: 450 return err; 451 } 452 453 int 454 mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, 455 struct mlx5e_flow_meter_params *params) 456 { 457 struct mlx5_core_dev *mdev = meter->flow_meters->mdev; 458 struct mlx5e_flow_meters *flow_meters; 459 int err; 460 461 flow_meters = mlx5e_get_flow_meters(mdev); 462 if (!flow_meters) 463 return -EOPNOTSUPP; 464 465 mutex_lock(&flow_meters->sync_lock); 466 err = __mlx5e_tc_meter_update(meter, params); 467 mutex_unlock(&flow_meters->sync_lock); 468 return err; 469 } 470 471 struct mlx5e_flow_meter_handle * 472 mlx5e_tc_meter_replace(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) 473 { 474 struct mlx5e_flow_meters *flow_meters; 475 struct mlx5e_flow_meter_handle *meter; 476 int err; 477 478 flow_meters = mlx5e_get_flow_meters(mdev); 479 if (!flow_meters) 480 return ERR_PTR(-EOPNOTSUPP); 481 482 mutex_lock(&flow_meters->sync_lock); 483 meter = __mlx5e_tc_meter_get(flow_meters, params->index); 484 if (IS_ERR(meter)) { 485 meter = mlx5e_tc_meter_alloc(flow_meters, params); 486 if (IS_ERR(meter)) { 487 err = PTR_ERR(meter); 488 goto err_get; 489 } 490 } 491 492 err = __mlx5e_tc_meter_update(meter, params); 493 if (err) 494 goto err_update; 495 496 mutex_unlock(&flow_meters->sync_lock); 497 return meter; 498 499 err_update: 500 __mlx5e_tc_meter_put(meter); 501 err_get: 502 mutex_unlock(&flow_meters->sync_lock); 503 return ERR_PTR(err); 504 } 505 506 enum mlx5_flow_namespace_type 507 mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters *flow_meters) 508 { 509 return flow_meters->ns_type; 510 } 511 512 struct mlx5e_flow_meters * 513 mlx5e_flow_meters_init(struct mlx5e_priv *priv, 514 enum mlx5_flow_namespace_type ns_type, 515 struct mlx5e_post_act *post_act) 516 { 517 struct mlx5_core_dev *mdev = priv->mdev; 518 struct mlx5e_flow_meters *flow_meters; 519 int err; 520 521 if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & 522 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO)) 523 return ERR_PTR(-EOPNOTSUPP); 524 525 if (IS_ERR_OR_NULL(post_act)) { 526 netdev_dbg(priv->netdev, 527 "flow meter offload is not supported, post action is missing\n"); 528 return ERR_PTR(-EOPNOTSUPP); 529 } 530 531 flow_meters = kzalloc(sizeof(*flow_meters), GFP_KERNEL); 532 if (!flow_meters) 533 return ERR_PTR(-ENOMEM); 534 535 err = mlx5_core_alloc_pd(mdev, &flow_meters->pdn); 536 if (err) { 537 mlx5_core_err(mdev, "Failed to alloc pd for flow meter aso, err=%d\n", err); 538 goto err_out; 539 } 540 541 flow_meters->aso = mlx5_aso_create(mdev, flow_meters->pdn); 542 if (IS_ERR(flow_meters->aso)) { 543 mlx5_core_warn(mdev, "Failed to create aso wqe for flow meter\n"); 544 err = PTR_ERR(flow_meters->aso); 545 goto err_sq; 546 } 547 548 mutex_init(&flow_meters->sync_lock); 549 INIT_LIST_HEAD(&flow_meters->partial_list); 550 INIT_LIST_HEAD(&flow_meters->full_list); 551 552 flow_meters->ns_type = ns_type; 553 flow_meters->mdev = mdev; 554 flow_meters->post_act = post_act; 555 mutex_init(&flow_meters->aso_lock); 556 flow_meters->log_granularity = min_t(int, 6, 557 MLX5_CAP_QOS(mdev, log_meter_aso_max_alloc)); 558 559 return flow_meters; 560 561 err_sq: 562 mlx5_core_dealloc_pd(mdev, flow_meters->pdn); 563 err_out: 564 kfree(flow_meters); 565 return ERR_PTR(err); 566 } 567 568 void 569 mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters) 570 { 571 if (IS_ERR_OR_NULL(flow_meters)) 572 return; 573 574 mlx5_aso_destroy(flow_meters->aso); 575 mlx5_core_dealloc_pd(flow_meters->mdev, flow_meters->pdn); 576 kfree(flow_meters); 577 } 578 579 void 580 mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter, 581 u64 *bytes, u64 *packets, u64 *drops, u64 *lastuse) 582 { 583 u64 bytes1, packets1, lastuse1; 584 u64 bytes2, packets2, lastuse2; 585 586 mlx5_fc_query_cached(meter->act_counter, &bytes1, &packets1, &lastuse1); 587 mlx5_fc_query_cached(meter->drop_counter, &bytes2, &packets2, &lastuse2); 588 589 *bytes = bytes1 + bytes2; 590 *packets = packets1 + packets2; 591 *drops = packets2; 592 *lastuse = max_t(u64, lastuse1, lastuse2); 593 } 594