1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/mlx5/driver.h> 34 #include "mlx5_core.h" 35 #include <linux/mlx5/transobj.h> 36 37 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn) 38 { 39 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {0}; 40 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0}; 41 int err; 42 43 MLX5_SET(alloc_transport_domain_in, in, opcode, 44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN); 45 46 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 47 if (!err) 48 *tdn = MLX5_GET(alloc_transport_domain_out, out, 49 transport_domain); 50 51 return err; 52 } 53 EXPORT_SYMBOL(mlx5_core_alloc_transport_domain); 54 55 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn) 56 { 57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {0}; 58 u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0}; 59 60 MLX5_SET(dealloc_transport_domain_in, in, opcode, 61 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 62 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn); 63 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 64 } 65 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain); 66 67 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn) 68 { 69 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {0}; 70 int err; 71 72 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ); 73 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 74 if (!err) 75 *rqn = MLX5_GET(create_rq_out, out, rqn); 76 77 return err; 78 } 79 EXPORT_SYMBOL(mlx5_core_create_rq); 80 81 int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen) 82 { 83 u32 out[MLX5_ST_SZ_DW(modify_rq_out)]; 84 85 MLX5_SET(modify_rq_in, in, rqn, rqn); 86 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ); 87 88 memset(out, 0, sizeof(out)); 89 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 90 } 91 EXPORT_SYMBOL(mlx5_core_modify_rq); 92 93 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn) 94 { 95 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {0}; 96 u32 out[MLX5_ST_SZ_DW(destroy_rq_out)] = {0}; 97 98 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ); 99 MLX5_SET(destroy_rq_in, in, rqn, rqn); 100 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 101 } 102 EXPORT_SYMBOL(mlx5_core_destroy_rq); 103 104 int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out) 105 { 106 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {0}; 107 int outlen = MLX5_ST_SZ_BYTES(query_rq_out); 108 109 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ); 110 MLX5_SET(query_rq_in, in, rqn, rqn); 111 112 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); 113 } 114 EXPORT_SYMBOL(mlx5_core_query_rq); 115 116 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn) 117 { 118 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {0}; 119 int err; 120 121 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ); 122 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 123 if (!err) 124 *sqn = MLX5_GET(create_sq_out, out, sqn); 125 126 return err; 127 } 128 129 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen) 130 { 131 u32 out[MLX5_ST_SZ_DW(modify_sq_out)] = {0}; 132 133 MLX5_SET(modify_sq_in, in, sqn, sqn); 134 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ); 135 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 136 } 137 EXPORT_SYMBOL(mlx5_core_modify_sq); 138 139 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn) 140 { 141 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {0}; 142 u32 out[MLX5_ST_SZ_DW(destroy_sq_out)] = {0}; 143 144 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ); 145 MLX5_SET(destroy_sq_in, in, sqn, sqn); 146 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 147 } 148 149 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out) 150 { 151 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {0}; 152 int outlen = MLX5_ST_SZ_BYTES(query_sq_out); 153 154 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ); 155 MLX5_SET(query_sq_in, in, sqn, sqn); 156 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); 157 } 158 EXPORT_SYMBOL(mlx5_core_query_sq); 159 160 int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state) 161 { 162 void *out; 163 void *sqc; 164 int inlen; 165 int err; 166 167 inlen = MLX5_ST_SZ_BYTES(query_sq_out); 168 out = kvzalloc(inlen, GFP_KERNEL); 169 if (!out) 170 return -ENOMEM; 171 172 err = mlx5_core_query_sq(dev, sqn, out); 173 if (err) 174 goto out; 175 176 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context); 177 *state = MLX5_GET(sqc, sqc, state); 178 179 out: 180 kvfree(out); 181 return err; 182 } 183 EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state); 184 185 int mlx5_core_create_tir_out(struct mlx5_core_dev *dev, 186 u32 *in, int inlen, 187 u32 *out, int outlen) 188 { 189 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); 190 191 return mlx5_cmd_exec(dev, in, inlen, out, outlen); 192 } 193 EXPORT_SYMBOL(mlx5_core_create_tir_out); 194 195 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen, 196 u32 *tirn) 197 { 198 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {}; 199 int err; 200 201 err = mlx5_core_create_tir_out(dev, in, inlen, 202 out, sizeof(out)); 203 if (!err) 204 *tirn = MLX5_GET(create_tir_out, out, tirn); 205 206 return err; 207 } 208 EXPORT_SYMBOL(mlx5_core_create_tir); 209 210 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in, 211 int inlen) 212 { 213 u32 out[MLX5_ST_SZ_DW(modify_tir_out)] = {0}; 214 215 MLX5_SET(modify_tir_in, in, tirn, tirn); 216 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR); 217 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 218 } 219 220 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn) 221 { 222 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {0}; 223 u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {0}; 224 225 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR); 226 MLX5_SET(destroy_tir_in, in, tirn, tirn); 227 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 228 } 229 EXPORT_SYMBOL(mlx5_core_destroy_tir); 230 231 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen, 232 u32 *tisn) 233 { 234 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {0}; 235 int err; 236 237 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); 238 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 239 if (!err) 240 *tisn = MLX5_GET(create_tis_out, out, tisn); 241 242 return err; 243 } 244 EXPORT_SYMBOL(mlx5_core_create_tis); 245 246 int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in, 247 int inlen) 248 { 249 u32 out[MLX5_ST_SZ_DW(modify_tis_out)] = {0}; 250 251 MLX5_SET(modify_tis_in, in, tisn, tisn); 252 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS); 253 254 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 255 } 256 EXPORT_SYMBOL(mlx5_core_modify_tis); 257 258 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn) 259 { 260 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {0}; 261 u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0}; 262 263 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); 264 MLX5_SET(destroy_tis_in, in, tisn, tisn); 265 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 266 } 267 EXPORT_SYMBOL(mlx5_core_destroy_tis); 268 269 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen, 270 u32 *rqtn) 271 { 272 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0}; 273 int err; 274 275 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); 276 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 277 if (!err) 278 *rqtn = MLX5_GET(create_rqt_out, out, rqtn); 279 280 return err; 281 } 282 EXPORT_SYMBOL(mlx5_core_create_rqt); 283 284 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in, 285 int inlen) 286 { 287 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0}; 288 289 MLX5_SET(modify_rqt_in, in, rqtn, rqtn); 290 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT); 291 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 292 } 293 294 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn) 295 { 296 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0}; 297 u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0}; 298 299 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); 300 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn); 301 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 302 } 303 EXPORT_SYMBOL(mlx5_core_destroy_rqt); 304 305 static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev, 306 struct mlx5_hairpin_params *params, u32 *rqn) 307 { 308 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0}; 309 void *rqc, *wq; 310 311 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 312 wq = MLX5_ADDR_OF(rqc, rqc, wq); 313 314 MLX5_SET(rqc, rqc, hairpin, 1); 315 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 316 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter); 317 318 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 319 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets); 320 321 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn); 322 } 323 324 static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev, 325 struct mlx5_hairpin_params *params, u32 *sqn) 326 { 327 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0}; 328 void *sqc, *wq; 329 330 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 331 wq = MLX5_ADDR_OF(sqc, sqc, wq); 332 333 MLX5_SET(sqc, sqc, hairpin, 1); 334 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 335 336 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 337 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets); 338 339 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn); 340 } 341 342 static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp, 343 struct mlx5_hairpin_params *params) 344 { 345 int i, j, err; 346 347 for (i = 0; i < hp->num_channels; i++) { 348 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]); 349 if (err) 350 goto out_err_rq; 351 } 352 353 for (i = 0; i < hp->num_channels; i++) { 354 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]); 355 if (err) 356 goto out_err_sq; 357 } 358 359 return 0; 360 361 out_err_sq: 362 for (j = 0; j < i; j++) 363 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]); 364 i = hp->num_channels; 365 out_err_rq: 366 for (j = 0; j < i; j++) 367 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]); 368 return err; 369 } 370 371 static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp) 372 { 373 int i; 374 375 for (i = 0; i < hp->num_channels; i++) { 376 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]); 377 if (!hp->peer_gone) 378 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]); 379 } 380 } 381 382 static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn, 383 int curr_state, int next_state, 384 u16 peer_vhca, u32 peer_sq) 385 { 386 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {0}; 387 void *rqc; 388 389 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 390 391 if (next_state == MLX5_RQC_STATE_RDY) { 392 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq); 393 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca); 394 } 395 396 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 397 MLX5_SET(rqc, rqc, state, next_state); 398 399 return mlx5_core_modify_rq(func_mdev, rqn, 400 in, MLX5_ST_SZ_BYTES(modify_rq_in)); 401 } 402 403 static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn, 404 int curr_state, int next_state, 405 u16 peer_vhca, u32 peer_rq) 406 { 407 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0}; 408 void *sqc; 409 410 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 411 412 if (next_state == MLX5_SQC_STATE_RDY) { 413 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq); 414 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca); 415 } 416 417 MLX5_SET(modify_sq_in, in, sq_state, curr_state); 418 MLX5_SET(sqc, sqc, state, next_state); 419 420 return mlx5_core_modify_sq(peer_mdev, sqn, 421 in, MLX5_ST_SZ_BYTES(modify_sq_in)); 422 } 423 424 static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp) 425 { 426 int i, j, err; 427 428 /* set peer SQs */ 429 for (i = 0; i < hp->num_channels; i++) { 430 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], 431 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY, 432 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]); 433 if (err) 434 goto err_modify_sq; 435 } 436 437 /* set func RQs */ 438 for (i = 0; i < hp->num_channels; i++) { 439 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], 440 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY, 441 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]); 442 if (err) 443 goto err_modify_rq; 444 } 445 446 return 0; 447 448 err_modify_rq: 449 for (j = 0; j < i; j++) 450 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY, 451 MLX5_RQC_STATE_RST, 0, 0); 452 i = hp->num_channels; 453 err_modify_sq: 454 for (j = 0; j < i; j++) 455 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY, 456 MLX5_SQC_STATE_RST, 0, 0); 457 return err; 458 } 459 460 static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp) 461 { 462 int i; 463 464 /* unset func RQs */ 465 for (i = 0; i < hp->num_channels; i++) 466 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY, 467 MLX5_RQC_STATE_RST, 0, 0); 468 469 /* unset peer SQs */ 470 if (hp->peer_gone) 471 return; 472 for (i = 0; i < hp->num_channels; i++) 473 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY, 474 MLX5_SQC_STATE_RST, 0, 0); 475 } 476 477 struct mlx5_hairpin * 478 mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev, 479 struct mlx5_core_dev *peer_mdev, 480 struct mlx5_hairpin_params *params) 481 { 482 struct mlx5_hairpin *hp; 483 int size, err; 484 485 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32); 486 hp = kzalloc(size, GFP_KERNEL); 487 if (!hp) 488 return ERR_PTR(-ENOMEM); 489 490 hp->func_mdev = func_mdev; 491 hp->peer_mdev = peer_mdev; 492 hp->num_channels = params->num_channels; 493 494 hp->rqn = (void *)hp + sizeof(*hp); 495 hp->sqn = hp->rqn + params->num_channels; 496 497 /* alloc and pair func --> peer hairpin */ 498 err = mlx5_hairpin_create_queues(hp, params); 499 if (err) 500 goto err_create_queues; 501 502 err = mlx5_hairpin_pair_queues(hp); 503 if (err) 504 goto err_pair_queues; 505 506 return hp; 507 508 err_pair_queues: 509 mlx5_hairpin_destroy_queues(hp); 510 err_create_queues: 511 kfree(hp); 512 return ERR_PTR(err); 513 } 514 515 void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp) 516 { 517 mlx5_hairpin_unpair_queues(hp); 518 mlx5_hairpin_destroy_queues(hp); 519 kfree(hp); 520 } 521