1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include "cn10k.h" 9 #include "otx2_reg.h" 10 #include "otx2_struct.h" 11 12 static struct dev_hw_ops otx2_hw_ops = { 13 .sq_aq_init = otx2_sq_aq_init, 14 .sqe_flush = otx2_sqe_flush, 15 .aura_freeptr = otx2_aura_freeptr, 16 .refill_pool_ptrs = otx2_refill_pool_ptrs, 17 }; 18 19 static struct dev_hw_ops cn10k_hw_ops = { 20 .sq_aq_init = cn10k_sq_aq_init, 21 .sqe_flush = cn10k_sqe_flush, 22 .aura_freeptr = cn10k_aura_freeptr, 23 .refill_pool_ptrs = cn10k_refill_pool_ptrs, 24 }; 25 26 int cn10k_lmtst_init(struct otx2_nic *pfvf) 27 { 28 29 struct lmtst_tbl_setup_req *req; 30 struct otx2_lmt_info *lmt_info; 31 int err, cpu; 32 33 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 34 pfvf->hw_ops = &otx2_hw_ops; 35 return 0; 36 } 37 38 pfvf->hw_ops = &cn10k_hw_ops; 39 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/ 40 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE); 41 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info); 42 43 mutex_lock(&pfvf->mbox.lock); 44 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox); 45 if (!req) { 46 mutex_unlock(&pfvf->mbox.lock); 47 return -ENOMEM; 48 } 49 50 req->use_local_lmt_region = true; 51 52 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines, 53 LMT_LINE_SIZE); 54 if (err) { 55 mutex_unlock(&pfvf->mbox.lock); 56 return err; 57 } 58 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base; 59 req->lmt_iova = (u64)pfvf->dync_lmt->iova; 60 61 err = otx2_sync_mbox_msg(&pfvf->mbox); 62 mutex_unlock(&pfvf->mbox.lock); 63 64 for_each_possible_cpu(cpu) { 65 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu); 66 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base + 67 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE)); 68 lmt_info->lmt_id = cpu * LMT_BURST_SIZE; 69 } 70 71 return 0; 72 } 73 EXPORT_SYMBOL(cn10k_lmtst_init); 74 75 int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) 76 { 77 struct nix_cn10k_aq_enq_req *aq; 78 struct otx2_nic *pfvf = dev; 79 80 /* Get memory to put this msg */ 81 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 82 if (!aq) 83 return -ENOMEM; 84 85 aq->sq.cq = pfvf->hw.rx_queues + qidx; 86 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 87 aq->sq.cq_ena = 1; 88 aq->sq.ena = 1; 89 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); 90 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); 91 aq->sq.default_chan = pfvf->hw.tx_chan_base; 92 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 93 aq->sq.sqb_aura = sqb_aura; 94 aq->sq.sq_int_ena = NIX_SQINT_BITS; 95 aq->sq.qint_idx = 0; 96 /* Due pipelining impact minimum 2000 unused SQ CQE's 97 * need to maintain to avoid CQ overflow. 98 */ 99 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 100 101 /* Fill AQ info */ 102 aq->qidx = qidx; 103 aq->ctype = NIX_AQ_CTYPE_SQ; 104 aq->op = NIX_AQ_INSTOP_INIT; 105 106 return otx2_sync_mbox_msg(&pfvf->mbox); 107 } 108 109 #define NPA_MAX_BURST 16 110 void cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 111 { 112 struct otx2_nic *pfvf = dev; 113 u64 ptrs[NPA_MAX_BURST]; 114 int num_ptrs = 1; 115 dma_addr_t bufptr; 116 117 /* Refill pool with new buffers */ 118 while (cq->pool_ptrs) { 119 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { 120 if (num_ptrs--) 121 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 122 num_ptrs); 123 break; 124 } 125 cq->pool_ptrs--; 126 ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM; 127 num_ptrs++; 128 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { 129 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 130 num_ptrs); 131 num_ptrs = 1; 132 } 133 } 134 } 135 136 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) 137 { 138 struct otx2_lmt_info *lmt_info; 139 struct otx2_nic *pfvf = dev; 140 u64 val = 0, tar_addr = 0; 141 142 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id()); 143 /* FIXME: val[0:10] LMT_ID. 144 * [12:15] no of LMTST - 1 in the burst. 145 * [19:63] data size of each LMTST in the burst except first. 146 */ 147 val = (lmt_info->lmt_id & 0x7FF); 148 /* Target address for LMTST flush tells HW how many 128bit 149 * words are present. 150 * tar_addr[6:4] size of first LMTST - 1 in units of 128b. 151 */ 152 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; 153 dma_wmb(); 154 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size); 155 cn10k_lmt_flush(val, tar_addr); 156 157 sq->head++; 158 sq->head &= (sq->sqe_cnt - 1); 159 } 160 161 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) 162 { 163 struct nix_bandprof_free_req *req; 164 int rc; 165 166 if (is_dev_otx2(pfvf->pdev)) 167 return 0; 168 169 mutex_lock(&pfvf->mbox.lock); 170 171 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 172 if (!req) { 173 rc = -ENOMEM; 174 goto out; 175 } 176 177 /* Free all bandwidth profiles allocated */ 178 req->free_all = true; 179 180 rc = otx2_sync_mbox_msg(&pfvf->mbox); 181 out: 182 mutex_unlock(&pfvf->mbox.lock); 183 return rc; 184 } 185 186 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) 187 { 188 struct nix_bandprof_alloc_req *req; 189 struct nix_bandprof_alloc_rsp *rsp; 190 int rc; 191 192 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox); 193 if (!req) 194 return -ENOMEM; 195 196 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 197 198 rc = otx2_sync_mbox_msg(&pfvf->mbox); 199 if (rc) 200 goto out; 201 202 rsp = (struct nix_bandprof_alloc_rsp *) 203 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 204 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { 205 rc = -EIO; 206 goto out; 207 } 208 209 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; 210 out: 211 if (rc) { 212 dev_warn(pfvf->dev, 213 "Failed to allocate ingress bandwidth policer\n"); 214 } 215 216 return rc; 217 } 218 219 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) 220 { 221 struct otx2_hw *hw = &pfvf->hw; 222 int ret; 223 224 mutex_lock(&pfvf->mbox.lock); 225 226 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer); 227 228 mutex_unlock(&pfvf->mbox.lock); 229 230 return ret; 231 } 232 233 #define POLICER_TIMESTAMP 1 /* 1 second */ 234 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ 235 236 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, 237 u32 *burst_mantissa) 238 { 239 int tmp; 240 241 /* Burst is calculated as 242 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] 243 * This is the upper limit on number tokens (bytes) that 244 * can be accumulated in the bucket. 245 */ 246 *burst_exp = ilog2(burst); 247 if (burst < 256) { 248 /* No float: can't express mantissa in this case */ 249 *burst_mantissa = 0; 250 return; 251 } 252 253 if (*burst_exp > MAX_RATE_EXP) 254 *burst_exp = MAX_RATE_EXP; 255 256 /* Calculate mantissa 257 * Find remaining bytes 'burst - 2^burst_exp' 258 * mantissa = (remaining bytes) / 2^ (burst_exp - 8) 259 */ 260 tmp = burst - rounddown_pow_of_two(burst); 261 *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); 262 } 263 264 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, 265 u32 *rate_mantissa, u32 *rdiv) 266 { 267 u32 div = 0; 268 u32 exp = 0; 269 u64 tmp; 270 271 /* Figure out mantissa, exponent and divider from given max pkt rate 272 * 273 * To achieve desired rate HW adds 274 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every 275 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. 276 * Here policer timeunit is 2 usecs and rate is in bits per sec. 277 * Since floating point cannot be used below algorithm uses 1000000 278 * scale factor to support rates upto 100Gbps. 279 */ 280 tmp = rate * 32 * 2; 281 if (tmp < 256000000) { 282 while (tmp < 256000000) { 283 tmp = tmp * 2; 284 div++; 285 } 286 } else { 287 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) 288 tmp = tmp / 2; 289 290 if (exp > MAX_RATE_EXP) 291 exp = MAX_RATE_EXP; 292 } 293 294 *rate_mantissa = (tmp - 256000000) / 1000000; 295 *rate_exp = exp; 296 *rdiv = div; 297 } 298 299 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, 300 u16 policer, bool map) 301 { 302 struct nix_cn10k_aq_enq_req *aq; 303 304 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 305 if (!aq) 306 return -ENOMEM; 307 308 /* Enable policing and set the bandwidth profile (policer) index */ 309 if (map) 310 aq->rq.policer_ena = 1; 311 else 312 aq->rq.policer_ena = 0; 313 aq->rq_mask.policer_ena = 1; 314 315 aq->rq.band_prof_id = policer; 316 aq->rq_mask.band_prof_id = GENMASK(9, 0); 317 318 /* Fill AQ info */ 319 aq->qidx = rq_idx; 320 aq->ctype = NIX_AQ_CTYPE_RQ; 321 aq->op = NIX_AQ_INSTOP_WRITE; 322 323 return otx2_sync_mbox_msg(&pfvf->mbox); 324 } 325 326 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) 327 { 328 struct nix_bandprof_free_req *req; 329 330 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 331 if (!req) 332 return -ENOMEM; 333 334 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 335 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; 336 337 return otx2_sync_mbox_msg(&pfvf->mbox); 338 } 339 340 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) 341 { 342 struct otx2_hw *hw = &pfvf->hw; 343 int qidx, rc; 344 345 mutex_lock(&pfvf->mbox.lock); 346 347 /* Remove RQ's policer mapping */ 348 for (qidx = 0; qidx < hw->rx_queues; qidx++) 349 cn10k_map_unmap_rq_policer(pfvf, qidx, 350 hw->matchall_ipolicer, false); 351 352 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); 353 354 mutex_unlock(&pfvf->mbox.lock); 355 return rc; 356 } 357 358 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, 359 u32 burst, u64 rate, bool pps) 360 { 361 struct nix_cn10k_aq_enq_req *aq; 362 u32 burst_exp, burst_mantissa; 363 u32 rate_exp, rate_mantissa; 364 u32 rdiv; 365 366 /* Get exponent and mantissa values for the desired rate */ 367 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa); 368 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv); 369 370 /* Init bandwidth profile */ 371 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 372 if (!aq) 373 return -ENOMEM; 374 375 /* Set initial color mode to blind */ 376 aq->prof.icolor = 0x03; 377 aq->prof_mask.icolor = 0x03; 378 379 /* Set rate and burst values */ 380 aq->prof.cir_exponent = rate_exp; 381 aq->prof_mask.cir_exponent = 0x1F; 382 383 aq->prof.cir_mantissa = rate_mantissa; 384 aq->prof_mask.cir_mantissa = 0xFF; 385 386 aq->prof.cbs_exponent = burst_exp; 387 aq->prof_mask.cbs_exponent = 0x1F; 388 389 aq->prof.cbs_mantissa = burst_mantissa; 390 aq->prof_mask.cbs_mantissa = 0xFF; 391 392 aq->prof.rdiv = rdiv; 393 aq->prof_mask.rdiv = 0xF; 394 395 if (pps) { 396 /* The amount of decremented tokens is calculated according to 397 * the following equation: 398 * max([ LMODE ? 0 : (packet_length - LXPTR)] + 399 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], 400 * 1/256) 401 * if LMODE is 1 then rate limiting will be based on 402 * PPS otherwise bps. 403 * The aim of the ADJUST value is to specify a token cost per 404 * packet in contrary to the packet length that specifies a 405 * cost per byte. To rate limit based on PPS adjust mantissa 406 * is set as 384 and exponent as 1 so that number of tokens 407 * decremented becomes 1 i.e, 1 token per packeet. 408 */ 409 aq->prof.adjust_exponent = 1; 410 aq->prof_mask.adjust_exponent = 0x1F; 411 412 aq->prof.adjust_mantissa = 384; 413 aq->prof_mask.adjust_mantissa = 0x1FF; 414 415 aq->prof.lmode = 0x1; 416 aq->prof_mask.lmode = 0x1; 417 } 418 419 /* Two rate three color marker 420 * With PEIR/EIR set to zero, color will be either green or red 421 */ 422 aq->prof.meter_algo = 2; 423 aq->prof_mask.meter_algo = 0x3; 424 425 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; 426 aq->prof_mask.rc_action = 0x3; 427 428 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 429 aq->prof_mask.yc_action = 0x3; 430 431 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 432 aq->prof_mask.gc_action = 0x3; 433 434 /* Setting exponent value as 24 and mantissa as 0 configures 435 * the bucket with zero values making bucket unused. Peak 436 * information rate and Excess information rate buckets are 437 * unused here. 438 */ 439 aq->prof.peir_exponent = 24; 440 aq->prof_mask.peir_exponent = 0x1F; 441 442 aq->prof.peir_mantissa = 0; 443 aq->prof_mask.peir_mantissa = 0xFF; 444 445 aq->prof.pebs_exponent = 24; 446 aq->prof_mask.pebs_exponent = 0x1F; 447 448 aq->prof.pebs_mantissa = 0; 449 aq->prof_mask.pebs_mantissa = 0xFF; 450 451 /* Fill AQ info */ 452 aq->qidx = profile; 453 aq->ctype = NIX_AQ_CTYPE_BANDPROF; 454 aq->op = NIX_AQ_INSTOP_WRITE; 455 456 return otx2_sync_mbox_msg(&pfvf->mbox); 457 } 458 459 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, 460 u32 burst, u64 rate) 461 { 462 struct otx2_hw *hw = &pfvf->hw; 463 int qidx, rc; 464 465 mutex_lock(&pfvf->mbox.lock); 466 467 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst, 468 rate, false); 469 if (rc) 470 goto out; 471 472 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 473 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, 474 hw->matchall_ipolicer, true); 475 if (rc) 476 break; 477 } 478 479 out: 480 mutex_unlock(&pfvf->mbox.lock); 481 return rc; 482 } 483