1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 #include <linux/mlx5/device.h> 5 #include <linux/mlx5/transobj.h> 6 #include "clock.h" 7 #include "aso.h" 8 #include "wq.h" 9 10 struct mlx5_aso_cq { 11 /* data path - accessed per cqe */ 12 struct mlx5_cqwq wq; 13 14 /* data path - accessed per napi poll */ 15 struct mlx5_core_cq mcq; 16 17 /* control */ 18 struct mlx5_core_dev *mdev; 19 struct mlx5_wq_ctrl wq_ctrl; 20 } ____cacheline_aligned_in_smp; 21 22 struct mlx5_aso { 23 /* data path */ 24 u16 cc; 25 u16 pc; 26 27 struct mlx5_wqe_ctrl_seg *doorbell_cseg; 28 struct mlx5_aso_cq cq; 29 30 /* read only */ 31 struct mlx5_wq_cyc wq; 32 void __iomem *uar_map; 33 u32 sqn; 34 35 /* control path */ 36 struct mlx5_wq_ctrl wq_ctrl; 37 38 } ____cacheline_aligned_in_smp; 39 40 static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq) 41 { 42 mlx5_wq_destroy(&cq->wq_ctrl); 43 } 44 45 static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node, 46 void *cqc_data, struct mlx5_aso_cq *cq) 47 { 48 struct mlx5_core_cq *mcq = &cq->mcq; 49 struct mlx5_wq_param param; 50 int err; 51 u32 i; 52 53 param.buf_numa_node = numa_node; 54 param.db_numa_node = numa_node; 55 56 err = mlx5_cqwq_create(mdev, ¶m, cqc_data, &cq->wq, &cq->wq_ctrl); 57 if (err) 58 return err; 59 60 mcq->cqe_sz = 64; 61 mcq->set_ci_db = cq->wq_ctrl.db.db; 62 mcq->arm_db = cq->wq_ctrl.db.db + 1; 63 64 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 65 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 66 67 cqe->op_own = 0xf1; 68 } 69 70 cq->mdev = mdev; 71 72 return 0; 73 } 74 75 static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data) 76 { 77 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 78 struct mlx5_core_dev *mdev = cq->mdev; 79 struct mlx5_core_cq *mcq = &cq->mcq; 80 void *in, *cqc; 81 int inlen, eqn; 82 int err; 83 84 err = mlx5_vector2eqn(mdev, 0, &eqn); 85 if (err) 86 return err; 87 88 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 89 sizeof(u64) * cq->wq_ctrl.buf.npages; 90 in = kvzalloc(inlen, GFP_KERNEL); 91 if (!in) 92 return -ENOMEM; 93 94 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 95 96 memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc)); 97 98 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 99 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 100 101 MLX5_SET(cqc, cqc, cq_period_mode, DIM_CQ_PERIOD_MODE_START_FROM_EQE); 102 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 103 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 104 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 105 MLX5_ADAPTER_PAGE_SHIFT); 106 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 107 108 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 109 110 kvfree(in); 111 112 return err; 113 } 114 115 static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq) 116 { 117 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 118 mlx5_wq_destroy(&cq->wq_ctrl); 119 } 120 121 static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node, 122 struct mlx5_aso_cq *cq) 123 { 124 void *cqc_data; 125 int err; 126 127 cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL); 128 if (!cqc_data) 129 return -ENOMEM; 130 131 MLX5_SET(cqc, cqc_data, log_cq_size, 1); 132 MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index); 133 if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) 134 MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD); 135 136 err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq); 137 if (err) { 138 mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err); 139 goto err_out; 140 } 141 142 err = create_aso_cq(cq, cqc_data); 143 if (err) { 144 mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err); 145 goto err_free_cq; 146 } 147 148 kvfree(cqc_data); 149 return 0; 150 151 err_free_cq: 152 mlx5_aso_free_cq(cq); 153 err_out: 154 kvfree(cqc_data); 155 return err; 156 } 157 158 static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node, 159 void *sqc_data, struct mlx5_aso *sq) 160 { 161 void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq); 162 struct mlx5_wq_cyc *wq = &sq->wq; 163 struct mlx5_wq_param param; 164 int err; 165 166 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; 167 168 param.db_numa_node = numa_node; 169 param.buf_numa_node = numa_node; 170 err = mlx5_wq_cyc_create(mdev, ¶m, sqc_wq, wq, &sq->wq_ctrl); 171 if (err) 172 return err; 173 wq->db = &wq->db[MLX5_SND_DBR]; 174 175 return 0; 176 } 177 178 static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn, 179 void *sqc_data, struct mlx5_aso *sq) 180 { 181 void *in, *sqc, *wq; 182 int inlen, err; 183 u8 ts_format; 184 185 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 186 sizeof(u64) * sq->wq_ctrl.buf.npages; 187 in = kvzalloc(inlen, GFP_KERNEL); 188 if (!in) 189 return -ENOMEM; 190 191 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 192 wq = MLX5_ADDR_OF(sqc, sqc, wq); 193 194 memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc)); 195 MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn); 196 197 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 198 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 199 200 ts_format = mlx5_is_real_time_sq(mdev) ? 201 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 202 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 203 MLX5_SET(sqc, sqc, ts_format, ts_format); 204 205 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 206 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); 207 MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - 208 MLX5_ADAPTER_PAGE_SHIFT); 209 MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); 210 211 mlx5_fill_page_frag_array(&sq->wq_ctrl.buf, 212 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 213 214 err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn); 215 216 kvfree(in); 217 218 return err; 219 } 220 221 static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn) 222 { 223 void *in, *sqc; 224 int inlen, err; 225 226 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 227 in = kvzalloc(inlen, GFP_KERNEL); 228 if (!in) 229 return -ENOMEM; 230 231 MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST); 232 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 233 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY); 234 235 err = mlx5_core_modify_sq(mdev, sqn, in); 236 237 kvfree(in); 238 239 return err; 240 } 241 242 static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn, 243 void *sqc_data, struct mlx5_aso *sq) 244 { 245 int err; 246 247 err = create_aso_sq(mdev, pdn, sqc_data, sq); 248 if (err) 249 return err; 250 251 err = mlx5_aso_set_sq_rdy(mdev, sq->sqn); 252 if (err) 253 mlx5_core_destroy_sq(mdev, sq->sqn); 254 255 return err; 256 } 257 258 static void mlx5_aso_free_sq(struct mlx5_aso *sq) 259 { 260 mlx5_wq_destroy(&sq->wq_ctrl); 261 } 262 263 static void mlx5_aso_destroy_sq(struct mlx5_aso *sq) 264 { 265 mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn); 266 mlx5_aso_free_sq(sq); 267 } 268 269 static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node, 270 u32 pdn, struct mlx5_aso *sq) 271 { 272 void *sqc_data, *wq; 273 int err; 274 275 sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL); 276 if (!sqc_data) 277 return -ENOMEM; 278 279 wq = MLX5_ADDR_OF(sqc, sqc_data, wq); 280 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); 281 MLX5_SET(wq, wq, pd, pdn); 282 MLX5_SET(wq, wq, log_wq_sz, 1); 283 284 err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq); 285 if (err) { 286 mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err); 287 goto err_out; 288 } 289 290 err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq); 291 if (err) { 292 mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err); 293 goto err_free_asosq; 294 } 295 296 mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn); 297 298 kvfree(sqc_data); 299 return 0; 300 301 err_free_asosq: 302 mlx5_aso_free_sq(sq); 303 err_out: 304 kvfree(sqc_data); 305 return err; 306 } 307 308 struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn) 309 { 310 int numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 311 struct mlx5_aso *aso; 312 int err; 313 314 aso = kzalloc(sizeof(*aso), GFP_KERNEL); 315 if (!aso) 316 return ERR_PTR(-ENOMEM); 317 318 err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq); 319 if (err) 320 goto err_cq; 321 322 err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso); 323 if (err) 324 goto err_sq; 325 326 return aso; 327 328 err_sq: 329 mlx5_aso_destroy_cq(&aso->cq); 330 err_cq: 331 kfree(aso); 332 return ERR_PTR(err); 333 } 334 335 void mlx5_aso_destroy(struct mlx5_aso *aso) 336 { 337 mlx5_aso_destroy_sq(aso); 338 mlx5_aso_destroy_cq(&aso->cq); 339 kfree(aso); 340 } 341 342 void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, 343 struct mlx5_aso_wqe *aso_wqe, 344 u32 obj_id, u32 opc_mode) 345 { 346 struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl; 347 348 cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) | 349 (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | 350 MLX5_OPCODE_ACCESS_ASO); 351 cseg->qpn_ds = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt); 352 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 353 cseg->general_id = cpu_to_be32(obj_id); 354 } 355 356 struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso) 357 { 358 struct mlx5_aso_wqe *wqe; 359 u16 pi; 360 361 pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc); 362 wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi); 363 memset(wqe, 0, sizeof(*wqe)); 364 return wqe; 365 } 366 367 void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, 368 struct mlx5_wqe_ctrl_seg *doorbell_cseg) 369 { 370 doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; 371 /* ensure wqe is visible to device before updating doorbell record */ 372 dma_wmb(); 373 374 if (with_data) 375 aso->pc += MLX5_ASO_WQEBBS_DATA; 376 else 377 aso->pc += MLX5_ASO_WQEBBS; 378 *aso->wq.db = cpu_to_be32(aso->pc); 379 380 /* ensure doorbell record is visible to device before ringing the 381 * doorbell 382 */ 383 wmb(); 384 385 mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map); 386 387 /* Ensure doorbell is written on uar_page before poll_cq */ 388 WRITE_ONCE(doorbell_cseg, NULL); 389 } 390 391 int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data) 392 { 393 struct mlx5_aso_cq *cq = &aso->cq; 394 struct mlx5_cqe64 *cqe; 395 396 cqe = mlx5_cqwq_get_cqe(&cq->wq); 397 if (!cqe) 398 return -ETIMEDOUT; 399 400 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), 401 * otherwise a cq overrun may occur 402 */ 403 mlx5_cqwq_pop(&cq->wq); 404 405 if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) { 406 struct mlx5_err_cqe *err_cqe; 407 408 mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n", 409 get_cqe_opcode(cqe)); 410 411 err_cqe = (struct mlx5_err_cqe *)cqe; 412 mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n", 413 err_cqe->vendor_err_synd); 414 mlx5_core_err(cq->mdev, "syndrome=%x\n", 415 err_cqe->syndrome); 416 print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 417 16, 1, err_cqe, 418 sizeof(*err_cqe), false); 419 } 420 421 mlx5_cqwq_update_db_record(&cq->wq); 422 423 /* ensure cq space is freed before enabling more cqes */ 424 wmb(); 425 426 if (with_data) 427 aso->cc += MLX5_ASO_WQEBBS_DATA; 428 else 429 aso->cc += MLX5_ASO_WQEBBS; 430 431 return 0; 432 } 433