1 /*- 2 * Copyright (c) 2017 Broadcom. All rights reserved. 3 * The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright notice, 12 * this list of conditions and the following disclaimer in the documentation 13 * and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the copyright holder nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 /** 35 * @file 36 * 37 */ 38 39 #include "ocs_os.h" 40 #include "ocs_hw.h" 41 #include "ocs_hw_queues.h" 42 43 #define HW_QTOP_DEBUG 0 44 45 /** 46 * @brief Initialize queues 47 * 48 * Given the parsed queue topology spec, the SLI queues are created and 49 * initialized 50 * 51 * @param hw pointer to HW object 52 * @param qtop pointer to queue topology 53 * 54 * @return returns 0 for success, an error code value for failure. 55 */ 56 ocs_hw_rtn_e 57 ocs_hw_init_queues(ocs_hw_t *hw, ocs_hw_qtop_t *qtop) 58 { 59 uint32_t i, j; 60 uint32_t default_lengths[QTOP_LAST], len; 61 uint32_t rqset_len = 0, rqset_ulp = 0, rqset_count = 0; 62 uint8_t rqset_filter_mask = 0; 63 hw_eq_t *eqs[hw->config.n_rq]; 64 hw_cq_t *cqs[hw->config.n_rq]; 65 hw_rq_t *rqs[hw->config.n_rq]; 66 ocs_hw_qtop_entry_t *qt, *next_qt; 67 ocs_hw_mrq_t mrq; 68 bool use_mrq = FALSE; 69 70 hw_eq_t *eq = NULL; 71 hw_cq_t *cq = NULL; 72 hw_wq_t *wq = NULL; 73 hw_rq_t *rq = NULL; 74 hw_mq_t *mq = NULL; 75 76 mrq.num_pairs = 0; 77 default_lengths[QTOP_EQ] = 1024; 78 default_lengths[QTOP_CQ] = hw->num_qentries[SLI_QTYPE_CQ]; 79 default_lengths[QTOP_WQ] = hw->num_qentries[SLI_QTYPE_WQ]; 80 default_lengths[QTOP_RQ] = hw->num_qentries[SLI_QTYPE_RQ]; 81 default_lengths[QTOP_MQ] = OCS_HW_MQ_DEPTH; 82 83 ocs_hw_verify(hw != NULL, OCS_HW_RTN_INVALID_ARG); 84 85 hw->eq_count = 0; 86 hw->cq_count = 0; 87 hw->mq_count = 0; 88 hw->wq_count = 0; 89 hw->rq_count = 0; 90 hw->hw_rq_count = 0; 91 ocs_list_init(&hw->eq_list, hw_eq_t, link); 92 93 /* If MRQ is requested, Check if it is supported by SLI. */ 94 if ((hw->config.n_rq > 1 ) && !hw->sli.config.features.flag.mrqp) { 95 ocs_log_err(hw->os, "MRQ topology not supported by SLI4.\n"); 96 return OCS_HW_RTN_ERROR; 97 } 98 99 if (hw->config.n_rq > 1) 100 use_mrq = TRUE; 101 102 /* Allocate class WQ pools */ 103 for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) { 104 hw->wq_class_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 105 if (hw->wq_class_array[i] == NULL) { 106 ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n"); 107 return OCS_HW_RTN_NO_MEMORY; 108 } 109 } 110 111 /* Allocate per CPU WQ pools */ 112 for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) { 113 hw->wq_cpu_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 114 if (hw->wq_cpu_array[i] == NULL) { 115 ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n"); 116 return OCS_HW_RTN_NO_MEMORY; 117 } 118 } 119 120 121 ocs_hw_assert(qtop != NULL); 122 123 for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) { 124 if (i == qtop->inuse_count - 1) 125 next_qt = NULL; 126 else 127 next_qt = qt + 1; 128 129 switch(qt->entry) { 130 case QTOP_EQ: 131 len = (qt->len) ? qt->len : default_lengths[QTOP_EQ]; 132 133 if (qt->set_default) { 134 default_lengths[QTOP_EQ] = len; 135 break; 136 } 137 138 eq = hw_new_eq(hw, len); 139 if (eq == NULL) { 140 hw_queue_teardown(hw); 141 return OCS_HW_RTN_NO_MEMORY; 142 } 143 break; 144 145 case QTOP_CQ: 146 len = (qt->len) ? qt->len : default_lengths[QTOP_CQ]; 147 148 if (qt->set_default) { 149 default_lengths[QTOP_CQ] = len; 150 break; 151 } 152 153 if (!eq || !next_qt) { 154 goto fail; 155 } 156 157 /* If this CQ is for MRQ, then delay the creation */ 158 if (!use_mrq || next_qt->entry != QTOP_RQ) { 159 cq = hw_new_cq(eq, len); 160 if (cq == NULL) { 161 goto fail; 162 } 163 } 164 break; 165 166 case QTOP_WQ: { 167 168 len = (qt->len) ? qt->len : default_lengths[QTOP_WQ]; 169 if (qt->set_default) { 170 default_lengths[QTOP_WQ] = len; 171 break; 172 } 173 174 if ((hw->ulp_start + qt->ulp) > hw->ulp_max) { 175 ocs_log_err(hw->os, "invalid ULP %d for WQ\n", qt->ulp); 176 hw_queue_teardown(hw); 177 return OCS_HW_RTN_NO_MEMORY; 178 } 179 180 if (cq == NULL) 181 goto fail; 182 183 wq = hw_new_wq(cq, len, qt->class, hw->ulp_start + qt->ulp); 184 if (wq == NULL) { 185 goto fail; 186 } 187 188 /* Place this WQ on the EQ WQ array */ 189 if (ocs_varray_add(eq->wq_array, wq)) { 190 ocs_log_err(hw->os, "QTOP_WQ: EQ ocs_varray_add failed\n"); 191 hw_queue_teardown(hw); 192 return OCS_HW_RTN_ERROR; 193 } 194 195 /* Place this WQ on the HW class array */ 196 if (qt->class < ARRAY_SIZE(hw->wq_class_array)) { 197 if (ocs_varray_add(hw->wq_class_array[qt->class], wq)) { 198 ocs_log_err(hw->os, "HW wq_class_array ocs_varray_add failed\n"); 199 hw_queue_teardown(hw); 200 return OCS_HW_RTN_ERROR; 201 } 202 } else { 203 ocs_log_err(hw->os, "Invalid class value: %d\n", qt->class); 204 hw_queue_teardown(hw); 205 return OCS_HW_RTN_ERROR; 206 } 207 208 /* 209 * Place this WQ on the per CPU list, asumming that EQs are mapped to cpu given 210 * by the EQ instance modulo number of CPUs 211 */ 212 if (ocs_varray_add(hw->wq_cpu_array[eq->instance % ocs_get_num_cpus()], wq)) { 213 ocs_log_err(hw->os, "HW wq_cpu_array ocs_varray_add failed\n"); 214 hw_queue_teardown(hw); 215 return OCS_HW_RTN_ERROR; 216 } 217 218 break; 219 } 220 case QTOP_RQ: { 221 len = (qt->len) ? qt->len : default_lengths[QTOP_RQ]; 222 if (qt->set_default) { 223 default_lengths[QTOP_RQ] = len; 224 break; 225 } 226 227 if ((hw->ulp_start + qt->ulp) > hw->ulp_max) { 228 ocs_log_err(hw->os, "invalid ULP %d for RQ\n", qt->ulp); 229 hw_queue_teardown(hw); 230 return OCS_HW_RTN_NO_MEMORY; 231 } 232 233 if (use_mrq) { 234 mrq.rq_cfg[mrq.num_pairs].len = len; 235 mrq.rq_cfg[mrq.num_pairs].ulp = hw->ulp_start + qt->ulp; 236 mrq.rq_cfg[mrq.num_pairs].filter_mask = qt->filter_mask; 237 mrq.rq_cfg[mrq.num_pairs].eq = eq; 238 mrq.num_pairs ++; 239 } else { 240 rq = hw_new_rq(cq, len, hw->ulp_start + qt->ulp); 241 if (rq == NULL) { 242 hw_queue_teardown(hw); 243 return OCS_HW_RTN_NO_MEMORY; 244 } 245 rq->filter_mask = qt->filter_mask; 246 } 247 break; 248 } 249 250 case QTOP_MQ: 251 len = (qt->len) ? qt->len : default_lengths[QTOP_MQ]; 252 if (qt->set_default) { 253 default_lengths[QTOP_MQ] = len; 254 break; 255 } 256 257 if (cq == NULL) 258 goto fail; 259 260 mq = hw_new_mq(cq, len); 261 if (mq == NULL) { 262 goto fail; 263 } 264 break; 265 266 default: 267 ocs_hw_assert(0); 268 break; 269 } 270 } 271 272 if (mrq.num_pairs) { 273 /* First create normal RQs. */ 274 for (i = 0; i < mrq.num_pairs; i++) { 275 for (j = 0; j < mrq.num_pairs; j++) { 276 if ((i != j) && (mrq.rq_cfg[i].filter_mask == mrq.rq_cfg[j].filter_mask)) { 277 /* This should be created using set */ 278 if (rqset_filter_mask && (rqset_filter_mask != mrq.rq_cfg[i].filter_mask)) { 279 ocs_log_crit(hw->os, "Cant create morethan one RQ Set\n"); 280 hw_queue_teardown(hw); 281 return OCS_HW_RTN_ERROR; 282 } else if (!rqset_filter_mask){ 283 rqset_filter_mask = mrq.rq_cfg[i].filter_mask; 284 rqset_len = mrq.rq_cfg[i].len; 285 rqset_ulp = mrq.rq_cfg[i].ulp; 286 } 287 eqs[rqset_count] = mrq.rq_cfg[i].eq; 288 rqset_count++; 289 break; 290 } 291 } 292 if (j == mrq.num_pairs) { 293 /* Normal RQ */ 294 cq = hw_new_cq(mrq.rq_cfg[i].eq, default_lengths[QTOP_CQ]); 295 if (cq == NULL) { 296 hw_queue_teardown(hw); 297 return OCS_HW_RTN_NO_MEMORY; 298 } 299 300 rq = hw_new_rq(cq, mrq.rq_cfg[i].len, mrq.rq_cfg[i].ulp); 301 if (rq == NULL) { 302 hw_queue_teardown(hw); 303 return OCS_HW_RTN_NO_MEMORY; 304 } 305 rq->filter_mask = mrq.rq_cfg[i].filter_mask; 306 } 307 } 308 309 /* Now create RQ Set */ 310 if (rqset_count) { 311 if (rqset_count > OCE_HW_MAX_NUM_MRQ_PAIRS) { 312 ocs_log_crit(hw->os, 313 "Max Supported MRQ pairs = %d\n", 314 OCE_HW_MAX_NUM_MRQ_PAIRS); 315 hw_queue_teardown(hw); 316 return OCS_HW_RTN_ERROR; 317 } 318 319 /* Create CQ set */ 320 if (hw_new_cq_set(eqs, cqs, rqset_count, default_lengths[QTOP_CQ])) { 321 hw_queue_teardown(hw); 322 return OCS_HW_RTN_ERROR; 323 } 324 325 /* Create RQ set */ 326 if (hw_new_rq_set(cqs, rqs, rqset_count, rqset_len, rqset_ulp)) { 327 hw_queue_teardown(hw); 328 return OCS_HW_RTN_ERROR; 329 } 330 331 for (i = 0; i < rqset_count ; i++) { 332 rqs[i]->filter_mask = rqset_filter_mask; 333 rqs[i]->is_mrq = TRUE; 334 rqs[i]->base_mrq_id = rqs[0]->hdr->id; 335 } 336 337 hw->hw_mrq_count = rqset_count; 338 } 339 } 340 341 return OCS_HW_RTN_SUCCESS; 342 fail: 343 hw_queue_teardown(hw); 344 return OCS_HW_RTN_NO_MEMORY; 345 346 } 347 348 /** 349 * @brief Allocate a new EQ object 350 * 351 * A new EQ object is instantiated 352 * 353 * @param hw pointer to HW object 354 * @param entry_count number of entries in the EQ 355 * 356 * @return pointer to allocated EQ object 357 */ 358 hw_eq_t* 359 hw_new_eq(ocs_hw_t *hw, uint32_t entry_count) 360 { 361 hw_eq_t *eq = ocs_malloc(hw->os, sizeof(*eq), OCS_M_ZERO | OCS_M_NOWAIT); 362 363 if (eq != NULL) { 364 eq->type = SLI_QTYPE_EQ; 365 eq->hw = hw; 366 eq->entry_count = entry_count; 367 eq->instance = hw->eq_count++; 368 eq->queue = &hw->eq[eq->instance]; 369 ocs_list_init(&eq->cq_list, hw_cq_t, link); 370 371 eq->wq_array = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 372 if (eq->wq_array == NULL) { 373 ocs_free(hw->os, eq, sizeof(*eq)); 374 eq = NULL; 375 } else { 376 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_EQ, eq->queue, entry_count, NULL, 0)) { 377 ocs_log_err(hw->os, "EQ[%d] allocation failure\n", eq->instance); 378 ocs_free(hw->os, eq, sizeof(*eq)); 379 eq = NULL; 380 } else { 381 sli_eq_modify_delay(&hw->sli, eq->queue, 1, 0, 8); 382 hw->hw_eq[eq->instance] = eq; 383 ocs_list_add_tail(&hw->eq_list, eq); 384 ocs_log_debug(hw->os, "create eq[%2d] id %3d len %4d\n", eq->instance, eq->queue->id, 385 eq->entry_count); 386 } 387 } 388 } 389 return eq; 390 } 391 392 /** 393 * @brief Allocate a new CQ object 394 * 395 * A new CQ object is instantiated 396 * 397 * @param eq pointer to parent EQ object 398 * @param entry_count number of entries in the CQ 399 * 400 * @return pointer to allocated CQ object 401 */ 402 hw_cq_t* 403 hw_new_cq(hw_eq_t *eq, uint32_t entry_count) 404 { 405 ocs_hw_t *hw = eq->hw; 406 hw_cq_t *cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT); 407 408 if (cq != NULL) { 409 cq->eq = eq; 410 cq->type = SLI_QTYPE_CQ; 411 cq->instance = eq->hw->cq_count++; 412 cq->entry_count = entry_count; 413 cq->queue = &hw->cq[cq->instance]; 414 415 ocs_list_init(&cq->q_list, hw_q_t, link); 416 417 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_CQ, cq->queue, cq->entry_count, eq->queue, 0)) { 418 ocs_log_err(hw->os, "CQ[%d] allocation failure len=%d\n", 419 eq->instance, 420 eq->entry_count); 421 ocs_free(hw->os, cq, sizeof(*cq)); 422 cq = NULL; 423 } else { 424 hw->hw_cq[cq->instance] = cq; 425 ocs_list_add_tail(&eq->cq_list, cq); 426 ocs_log_debug(hw->os, "create cq[%2d] id %3d len %4d\n", cq->instance, cq->queue->id, 427 cq->entry_count); 428 } 429 } 430 return cq; 431 } 432 433 /** 434 * @brief Allocate a new CQ Set of objects. 435 * 436 * @param eqs pointer to a set of EQ objects. 437 * @param cqs pointer to a set of CQ objects to be returned. 438 * @param num_cqs number of CQ queues in the set. 439 * @param entry_count number of entries in the CQ. 440 * 441 * @return 0 on success and -1 on failure. 442 */ 443 uint32_t 444 hw_new_cq_set(hw_eq_t *eqs[], hw_cq_t *cqs[], uint32_t num_cqs, uint32_t entry_count) 445 { 446 uint32_t i; 447 ocs_hw_t *hw = eqs[0]->hw; 448 sli4_t *sli4 = &hw->sli; 449 hw_cq_t *cq = NULL; 450 sli4_queue_t *qs[SLI_MAX_CQ_SET_COUNT], *assocs[SLI_MAX_CQ_SET_COUNT]; 451 452 /* Initialise CQS pointers to NULL */ 453 for (i = 0; i < num_cqs; i++) { 454 cqs[i] = NULL; 455 } 456 457 for (i = 0; i < num_cqs; i++) { 458 cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT); 459 if (cq == NULL) 460 goto error; 461 462 cqs[i] = cq; 463 cq->eq = eqs[i]; 464 cq->type = SLI_QTYPE_CQ; 465 cq->instance = hw->cq_count++; 466 cq->entry_count = entry_count; 467 cq->queue = &hw->cq[cq->instance]; 468 qs[i] = cq->queue; 469 assocs[i] = eqs[i]->queue; 470 ocs_list_init(&cq->q_list, hw_q_t, link); 471 } 472 473 if (sli_cq_alloc_set(sli4, qs, num_cqs, entry_count, assocs)) { 474 ocs_log_err(NULL, "Failed to create CQ Set. \n"); 475 goto error; 476 } 477 478 for (i = 0; i < num_cqs; i++) { 479 hw->hw_cq[cqs[i]->instance] = cqs[i]; 480 ocs_list_add_tail(&cqs[i]->eq->cq_list, cqs[i]); 481 } 482 483 return 0; 484 485 error: 486 for (i = 0; i < num_cqs; i++) { 487 if (cqs[i]) { 488 ocs_free(hw->os, cqs[i], sizeof(*cqs[i])); 489 cqs[i] = NULL; 490 } 491 } 492 return -1; 493 } 494 495 496 /** 497 * @brief Allocate a new MQ object 498 * 499 * A new MQ object is instantiated 500 * 501 * @param cq pointer to parent CQ object 502 * @param entry_count number of entries in the MQ 503 * 504 * @return pointer to allocated MQ object 505 */ 506 hw_mq_t* 507 hw_new_mq(hw_cq_t *cq, uint32_t entry_count) 508 { 509 ocs_hw_t *hw = cq->eq->hw; 510 hw_mq_t *mq = ocs_malloc(hw->os, sizeof(*mq), OCS_M_ZERO | OCS_M_NOWAIT); 511 512 if (mq != NULL) { 513 mq->cq = cq; 514 mq->type = SLI_QTYPE_MQ; 515 mq->instance = cq->eq->hw->mq_count++; 516 mq->entry_count = entry_count; 517 mq->entry_size = OCS_HW_MQ_DEPTH; 518 mq->queue = &hw->mq[mq->instance]; 519 520 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_MQ, 521 mq->queue, 522 mq->entry_size, 523 cq->queue, 0)) { 524 ocs_log_err(hw->os, "MQ allocation failure\n"); 525 ocs_free(hw->os, mq, sizeof(*mq)); 526 mq = NULL; 527 } else { 528 hw->hw_mq[mq->instance] = mq; 529 ocs_list_add_tail(&cq->q_list, mq); 530 ocs_log_debug(hw->os, "create mq[%2d] id %3d len %4d\n", mq->instance, mq->queue->id, 531 mq->entry_count); 532 } 533 } 534 return mq; 535 } 536 537 /** 538 * @brief Allocate a new WQ object 539 * 540 * A new WQ object is instantiated 541 * 542 * @param cq pointer to parent CQ object 543 * @param entry_count number of entries in the WQ 544 * @param class WQ class 545 * @param ulp index of chute 546 * 547 * @return pointer to allocated WQ object 548 */ 549 hw_wq_t* 550 hw_new_wq(hw_cq_t *cq, uint32_t entry_count, uint32_t class, uint32_t ulp) 551 { 552 ocs_hw_t *hw = cq->eq->hw; 553 hw_wq_t *wq = ocs_malloc(hw->os, sizeof(*wq), OCS_M_ZERO | OCS_M_NOWAIT); 554 555 if (wq != NULL) { 556 wq->hw = cq->eq->hw; 557 wq->cq = cq; 558 wq->type = SLI_QTYPE_WQ; 559 wq->instance = cq->eq->hw->wq_count++; 560 wq->entry_count = entry_count; 561 wq->queue = &hw->wq[wq->instance]; 562 wq->ulp = ulp; 563 wq->wqec_set_count = OCS_HW_WQEC_SET_COUNT; 564 wq->wqec_count = wq->wqec_set_count; 565 wq->free_count = wq->entry_count - 1; 566 wq->class = class; 567 ocs_list_init(&wq->pending_list, ocs_hw_wqe_t, link); 568 569 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_WQ, wq->queue, wq->entry_count, cq->queue, ulp)) { 570 ocs_log_err(hw->os, "WQ allocation failure\n"); 571 ocs_free(hw->os, wq, sizeof(*wq)); 572 wq = NULL; 573 } else { 574 hw->hw_wq[wq->instance] = wq; 575 ocs_list_add_tail(&cq->q_list, wq); 576 ocs_log_debug(hw->os, "create wq[%2d] id %3d len %4d cls %d ulp %d\n", wq->instance, wq->queue->id, 577 wq->entry_count, wq->class, wq->ulp); 578 } 579 } 580 return wq; 581 } 582 583 /** 584 * @brief Allocate a hw_rq_t object 585 * 586 * Allocate an RQ object, which encapsulates 2 SLI queues (for rq pair) 587 * 588 * @param cq pointer to parent CQ object 589 * @param entry_count number of entries in the RQs 590 * @param ulp ULP index for this RQ 591 * 592 * @return pointer to newly allocated hw_rq_t 593 */ 594 hw_rq_t* 595 hw_new_rq(hw_cq_t *cq, uint32_t entry_count, uint32_t ulp) 596 { 597 ocs_hw_t *hw = cq->eq->hw; 598 hw_rq_t *rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT); 599 uint32_t max_hw_rq; 600 601 ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq); 602 603 604 if (rq != NULL) { 605 rq->instance = hw->hw_rq_count++; 606 rq->cq = cq; 607 rq->type = SLI_QTYPE_RQ; 608 rq->ulp = ulp; 609 610 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR)); 611 612 /* Create the header RQ */ 613 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq)); 614 rq->hdr = &hw->rq[hw->rq_count]; 615 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE; 616 617 if (sli_fc_rq_alloc(&hw->sli, rq->hdr, 618 rq->entry_count, 619 rq->hdr_entry_size, 620 cq->queue, 621 ulp, TRUE)) { 622 ocs_log_err(hw->os, "RQ allocation failure - header\n"); 623 ocs_free(hw->os, rq, sizeof(*rq)); 624 return NULL; 625 } 626 hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */ 627 hw->rq_count++; 628 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d hdr size %4d ulp %d\n", 629 rq->instance, rq->hdr->id, rq->entry_count, rq->hdr_entry_size, rq->ulp); 630 631 /* Create the default data RQ */ 632 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq)); 633 rq->data = &hw->rq[hw->rq_count]; 634 rq->data_entry_size = hw->config.rq_default_buffer_size; 635 636 if (sli_fc_rq_alloc(&hw->sli, rq->data, 637 rq->entry_count, 638 rq->data_entry_size, 639 cq->queue, 640 ulp, FALSE)) { 641 ocs_log_err(hw->os, "RQ allocation failure - first burst\n"); 642 ocs_free(hw->os, rq, sizeof(*rq)); 643 return NULL; 644 } 645 hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */ 646 hw->rq_count++; 647 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d data size %4d ulp %d\n", rq->instance, 648 rq->data->id, rq->entry_count, rq->data_entry_size, rq->ulp); 649 650 hw->hw_rq[rq->instance] = rq; 651 ocs_list_add_tail(&cq->q_list, rq); 652 653 rq->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) * 654 rq->entry_count, OCS_M_ZERO | OCS_M_NOWAIT); 655 if (rq->rq_tracker == NULL) { 656 ocs_log_err(hw->os, "RQ tracker buf allocation failure\n"); 657 return NULL; 658 } 659 } 660 return rq; 661 } 662 663 664 /** 665 * @brief Allocate a hw_rq_t object SET 666 * 667 * Allocate an RQ object SET, where each element in set 668 * encapsulates 2 SLI queues (for rq pair) 669 * 670 * @param cqs pointers to be associated with RQs. 671 * @param rqs RQ pointers to be returned on success. 672 * @param num_rq_pairs number of rq pairs in the Set. 673 * @param entry_count number of entries in the RQs 674 * @param ulp ULP index for this RQ 675 * 676 * @return 0 in success and -1 on failure. 677 */ 678 uint32_t 679 hw_new_rq_set(hw_cq_t *cqs[], hw_rq_t *rqs[], uint32_t num_rq_pairs, uint32_t entry_count, uint32_t ulp) 680 { 681 ocs_hw_t *hw = cqs[0]->eq->hw; 682 hw_rq_t *rq = NULL; 683 sli4_queue_t *qs[SLI_MAX_RQ_SET_COUNT * 2] = { NULL }; 684 uint32_t max_hw_rq, i, q_count; 685 686 ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq); 687 688 /* Initialise RQS pointers */ 689 for (i = 0; i < num_rq_pairs; i++) { 690 rqs[i] = NULL; 691 } 692 693 for (i = 0, q_count = 0; i < num_rq_pairs; i++, q_count += 2) { 694 rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT); 695 if (rq == NULL) 696 goto error; 697 698 rqs[i] = rq; 699 rq->instance = hw->hw_rq_count++; 700 rq->cq = cqs[i]; 701 rq->type = SLI_QTYPE_RQ; 702 rq->ulp = ulp; 703 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR)); 704 705 /* Header RQ */ 706 rq->hdr = &hw->rq[hw->rq_count]; 707 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE; 708 hw->hw_rq_lookup[hw->rq_count] = rq->instance; 709 hw->rq_count++; 710 qs[q_count] = rq->hdr; 711 712 /* Data RQ */ 713 rq->data = &hw->rq[hw->rq_count]; 714 rq->data_entry_size = hw->config.rq_default_buffer_size; 715 hw->hw_rq_lookup[hw->rq_count] = rq->instance; 716 hw->rq_count++; 717 qs[q_count + 1] = rq->data; 718 719 rq->rq_tracker = NULL; 720 } 721 722 if (sli_fc_rq_set_alloc(&hw->sli, num_rq_pairs, qs, 723 cqs[0]->queue->id, 724 rqs[0]->entry_count, 725 rqs[0]->hdr_entry_size, 726 rqs[0]->data_entry_size, 727 ulp)) { 728 ocs_log_err(hw->os, "RQ Set allocation failure for base CQ=%d\n", cqs[0]->queue->id); 729 goto error; 730 } 731 732 733 for (i = 0; i < num_rq_pairs; i++) { 734 hw->hw_rq[rqs[i]->instance] = rqs[i]; 735 ocs_list_add_tail(&cqs[i]->q_list, rqs[i]); 736 rqs[i]->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) * 737 rqs[i]->entry_count, OCS_M_ZERO | OCS_M_NOWAIT); 738 if (rqs[i]->rq_tracker == NULL) { 739 ocs_log_err(hw->os, "RQ tracker buf allocation failure\n"); 740 goto error; 741 } 742 } 743 744 return 0; 745 746 error: 747 for (i = 0; i < num_rq_pairs; i++) { 748 if (rqs[i] != NULL) { 749 if (rqs[i]->rq_tracker != NULL) { 750 ocs_free(hw->os, rqs[i]->rq_tracker, 751 sizeof(ocs_hw_sequence_t*) * 752 rqs[i]->entry_count); 753 } 754 ocs_free(hw->os, rqs[i], sizeof(*rqs[i])); 755 } 756 } 757 758 return -1; 759 } 760 761 762 /** 763 * @brief Free an EQ object 764 * 765 * The EQ object and any child queue objects are freed 766 * 767 * @param eq pointer to EQ object 768 * 769 * @return none 770 */ 771 void 772 hw_del_eq(hw_eq_t *eq) 773 { 774 if (eq != NULL) { 775 hw_cq_t *cq; 776 hw_cq_t *cq_next; 777 778 ocs_list_foreach_safe(&eq->cq_list, cq, cq_next) { 779 hw_del_cq(cq); 780 } 781 ocs_varray_free(eq->wq_array); 782 ocs_list_remove(&eq->hw->eq_list, eq); 783 eq->hw->hw_eq[eq->instance] = NULL; 784 ocs_free(eq->hw->os, eq, sizeof(*eq)); 785 } 786 } 787 788 /** 789 * @brief Free a CQ object 790 * 791 * The CQ object and any child queue objects are freed 792 * 793 * @param cq pointer to CQ object 794 * 795 * @return none 796 */ 797 void 798 hw_del_cq(hw_cq_t *cq) 799 { 800 if (cq != NULL) { 801 hw_q_t *q; 802 hw_q_t *q_next; 803 804 ocs_list_foreach_safe(&cq->q_list, q, q_next) { 805 switch(q->type) { 806 case SLI_QTYPE_MQ: 807 hw_del_mq((hw_mq_t*) q); 808 break; 809 case SLI_QTYPE_WQ: 810 hw_del_wq((hw_wq_t*) q); 811 break; 812 case SLI_QTYPE_RQ: 813 hw_del_rq((hw_rq_t*) q); 814 break; 815 default: 816 break; 817 } 818 } 819 ocs_list_remove(&cq->eq->cq_list, cq); 820 cq->eq->hw->hw_cq[cq->instance] = NULL; 821 ocs_free(cq->eq->hw->os, cq, sizeof(*cq)); 822 } 823 } 824 825 /** 826 * @brief Free a MQ object 827 * 828 * The MQ object is freed 829 * 830 * @param mq pointer to MQ object 831 * 832 * @return none 833 */ 834 void 835 hw_del_mq(hw_mq_t *mq) 836 { 837 if (mq != NULL) { 838 ocs_list_remove(&mq->cq->q_list, mq); 839 mq->cq->eq->hw->hw_mq[mq->instance] = NULL; 840 ocs_free(mq->cq->eq->hw->os, mq, sizeof(*mq)); 841 } 842 } 843 844 /** 845 * @brief Free a WQ object 846 * 847 * The WQ object is freed 848 * 849 * @param wq pointer to WQ object 850 * 851 * @return none 852 */ 853 void 854 hw_del_wq(hw_wq_t *wq) 855 { 856 if (wq != NULL) { 857 ocs_list_remove(&wq->cq->q_list, wq); 858 wq->cq->eq->hw->hw_wq[wq->instance] = NULL; 859 ocs_free(wq->cq->eq->hw->os, wq, sizeof(*wq)); 860 } 861 } 862 863 /** 864 * @brief Free an RQ object 865 * 866 * The RQ object is freed 867 * 868 * @param rq pointer to RQ object 869 * 870 * @return none 871 */ 872 void 873 hw_del_rq(hw_rq_t *rq) 874 { 875 876 if (rq != NULL) { 877 ocs_hw_t *hw = rq->cq->eq->hw; 878 /* Free RQ tracker */ 879 if (rq->rq_tracker != NULL) { 880 ocs_free(hw->os, rq->rq_tracker, sizeof(ocs_hw_sequence_t*) * rq->entry_count); 881 rq->rq_tracker = NULL; 882 } 883 ocs_list_remove(&rq->cq->q_list, rq); 884 hw->hw_rq[rq->instance] = NULL; 885 ocs_free(hw->os, rq, sizeof(*rq)); 886 } 887 } 888 889 /** 890 * @brief Display HW queue objects 891 * 892 * The HW queue objects are displayed using ocs_log 893 * 894 * @param hw pointer to HW object 895 * 896 * @return none 897 */ 898 void 899 hw_queue_dump(ocs_hw_t *hw) 900 { 901 hw_eq_t *eq; 902 hw_cq_t *cq; 903 hw_q_t *q; 904 hw_mq_t *mq; 905 hw_wq_t *wq; 906 hw_rq_t *rq; 907 908 ocs_list_foreach(&hw->eq_list, eq) { 909 ocs_printf("eq[%d] id %2d\n", eq->instance, eq->queue->id); 910 ocs_list_foreach(&eq->cq_list, cq) { 911 ocs_printf(" cq[%d] id %2d current\n", cq->instance, cq->queue->id); 912 ocs_list_foreach(&cq->q_list, q) { 913 switch(q->type) { 914 case SLI_QTYPE_MQ: 915 mq = (hw_mq_t *) q; 916 ocs_printf(" mq[%d] id %2d\n", mq->instance, mq->queue->id); 917 break; 918 case SLI_QTYPE_WQ: 919 wq = (hw_wq_t *) q; 920 ocs_printf(" wq[%d] id %2d\n", wq->instance, wq->queue->id); 921 break; 922 case SLI_QTYPE_RQ: 923 rq = (hw_rq_t *) q; 924 ocs_printf(" rq[%d] hdr id %2d\n", rq->instance, rq->hdr->id); 925 break; 926 default: 927 break; 928 } 929 } 930 } 931 } 932 } 933 934 /** 935 * @brief Teardown HW queue objects 936 * 937 * The HW queue objects are freed 938 * 939 * @param hw pointer to HW object 940 * 941 * @return none 942 */ 943 void 944 hw_queue_teardown(ocs_hw_t *hw) 945 { 946 uint32_t i; 947 hw_eq_t *eq; 948 hw_eq_t *eq_next; 949 950 if (ocs_list_valid(&hw->eq_list)) { 951 ocs_list_foreach_safe(&hw->eq_list, eq, eq_next) { 952 hw_del_eq(eq); 953 } 954 } 955 for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) { 956 ocs_varray_free(hw->wq_cpu_array[i]); 957 hw->wq_cpu_array[i] = NULL; 958 } 959 for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) { 960 ocs_varray_free(hw->wq_class_array[i]); 961 hw->wq_class_array[i] = NULL; 962 } 963 } 964 965 /** 966 * @brief Allocate a WQ to an IO object 967 * 968 * The next work queue index is used to assign a WQ to an IO. 969 * 970 * If wq_steering is OCS_HW_WQ_STEERING_CLASS, a WQ from io->wq_class is 971 * selected. 972 * 973 * If wq_steering is OCS_HW_WQ_STEERING_REQUEST, then a WQ from the EQ that 974 * the IO request came in on is selected. 975 * 976 * If wq_steering is OCS_HW_WQ_STEERING_CPU, then a WQ associted with the 977 * CPU the request is made on is selected. 978 * 979 * @param hw pointer to HW object 980 * @param io pointer to IO object 981 * 982 * @return Return pointer to next WQ 983 */ 984 hw_wq_t * 985 ocs_hw_queue_next_wq(ocs_hw_t *hw, ocs_hw_io_t *io) 986 { 987 hw_eq_t *eq; 988 hw_wq_t *wq = NULL; 989 990 switch(io->wq_steering) { 991 case OCS_HW_WQ_STEERING_CLASS: 992 if (likely(io->wq_class < ARRAY_SIZE(hw->wq_class_array))) { 993 wq = ocs_varray_iter_next(hw->wq_class_array[io->wq_class]); 994 } 995 break; 996 case OCS_HW_WQ_STEERING_REQUEST: 997 eq = io->eq; 998 if (likely(eq != NULL)) { 999 wq = ocs_varray_iter_next(eq->wq_array); 1000 } 1001 break; 1002 case OCS_HW_WQ_STEERING_CPU: { 1003 uint32_t cpuidx = ocs_thread_getcpu(); 1004 1005 if (likely(cpuidx < ARRAY_SIZE(hw->wq_cpu_array))) { 1006 wq = ocs_varray_iter_next(hw->wq_cpu_array[cpuidx]); 1007 } 1008 break; 1009 } 1010 } 1011 1012 if (unlikely(wq == NULL)) { 1013 wq = hw->hw_wq[0]; 1014 } 1015 1016 return wq; 1017 } 1018 1019 /** 1020 * @brief Return count of EQs for a queue topology object 1021 * 1022 * The EQ count for in the HWs queue topology (hw->qtop) object is returned 1023 * 1024 * @param hw pointer to HW object 1025 * 1026 * @return count of EQs 1027 */ 1028 uint32_t 1029 ocs_hw_qtop_eq_count(ocs_hw_t *hw) 1030 { 1031 return hw->qtop->entry_counts[QTOP_EQ]; 1032 } 1033 1034 #define TOKEN_LEN 32 1035 1036 /** 1037 * @brief return string given a QTOP entry 1038 * 1039 * @param entry QTOP entry 1040 * 1041 * @return returns string or "unknown" 1042 */ 1043 #if HW_QTOP_DEBUG 1044 static char * 1045 qtopentry2s(ocs_hw_qtop_entry_e entry) { 1046 switch(entry) { 1047 #define P(x) case x: return #x; 1048 P(QTOP_EQ) 1049 P(QTOP_CQ) 1050 P(QTOP_WQ) 1051 P(QTOP_RQ) 1052 P(QTOP_MQ) 1053 P(QTOP_THREAD_START) 1054 P(QTOP_THREAD_END) 1055 P(QTOP_LAST) 1056 #undef P 1057 } 1058 return "unknown"; 1059 } 1060 #endif 1061 1062 /** 1063 * @brief Declare token types 1064 */ 1065 typedef enum { 1066 TOK_LPAREN = 1, 1067 TOK_RPAREN, 1068 TOK_COLON, 1069 TOK_EQUALS, 1070 TOK_QUEUE, 1071 TOK_ATTR_NAME, 1072 TOK_NUMBER, 1073 TOK_NUMBER_VALUE, 1074 TOK_NUMBER_LIST, 1075 } tok_type_e; 1076 1077 /** 1078 * @brief Declare token sub-types 1079 */ 1080 typedef enum { 1081 TOK_SUB_EQ = 100, 1082 TOK_SUB_CQ, 1083 TOK_SUB_RQ, 1084 TOK_SUB_MQ, 1085 TOK_SUB_WQ, 1086 TOK_SUB_LEN, 1087 TOK_SUB_CLASS, 1088 TOK_SUB_ULP, 1089 TOK_SUB_FILTER, 1090 } tok_subtype_e; 1091 1092 /** 1093 * @brief convert queue subtype to QTOP entry 1094 * 1095 * @param q queue subtype 1096 * 1097 * @return QTOP entry or 0 1098 */ 1099 static ocs_hw_qtop_entry_e 1100 subtype2qtop(tok_subtype_e q) 1101 { 1102 switch(q) { 1103 case TOK_SUB_EQ: return QTOP_EQ; 1104 case TOK_SUB_CQ: return QTOP_CQ; 1105 case TOK_SUB_RQ: return QTOP_RQ; 1106 case TOK_SUB_MQ: return QTOP_MQ; 1107 case TOK_SUB_WQ: return QTOP_WQ; 1108 default: 1109 break; 1110 } 1111 return 0; 1112 } 1113 1114 /** 1115 * @brief Declare token object 1116 */ 1117 typedef struct { 1118 tok_type_e type; 1119 tok_subtype_e subtype; 1120 char string[TOKEN_LEN]; 1121 } tok_t; 1122 1123 /** 1124 * @brief Declare token array object 1125 */ 1126 typedef struct { 1127 tok_t *tokens; /* Pointer to array of tokens */ 1128 uint32_t alloc_count; /* Number of tokens in the array */ 1129 uint32_t inuse_count; /* Number of tokens posted to array */ 1130 uint32_t iter_idx; /* Iterator index */ 1131 } tokarray_t; 1132 1133 /** 1134 * @brief Declare token match structure 1135 */ 1136 typedef struct { 1137 char *s; 1138 tok_type_e type; 1139 tok_subtype_e subtype; 1140 } tokmatch_t; 1141 1142 /** 1143 * @brief test if character is ID start character 1144 * 1145 * @param c character to test 1146 * 1147 * @return TRUE if character is an ID start character 1148 */ 1149 static int32_t 1150 idstart(int c) 1151 { 1152 return isalpha(c) || (c == '_') || (c == '$'); 1153 } 1154 1155 /** 1156 * @brief test if character is an ID character 1157 * 1158 * @param c character to test 1159 * 1160 * @return TRUE if character is an ID character 1161 */ 1162 static int32_t 1163 idchar(int c) 1164 { 1165 return idstart(c) || ocs_isdigit(c); 1166 } 1167 1168 /** 1169 * @brief Declare single character matches 1170 */ 1171 static tokmatch_t cmatches[] = { 1172 {"(", TOK_LPAREN}, 1173 {")", TOK_RPAREN}, 1174 {":", TOK_COLON}, 1175 {"=", TOK_EQUALS}, 1176 }; 1177 1178 /** 1179 * @brief Declare identifier match strings 1180 */ 1181 static tokmatch_t smatches[] = { 1182 {"eq", TOK_QUEUE, TOK_SUB_EQ}, 1183 {"cq", TOK_QUEUE, TOK_SUB_CQ}, 1184 {"rq", TOK_QUEUE, TOK_SUB_RQ}, 1185 {"mq", TOK_QUEUE, TOK_SUB_MQ}, 1186 {"wq", TOK_QUEUE, TOK_SUB_WQ}, 1187 {"len", TOK_ATTR_NAME, TOK_SUB_LEN}, 1188 {"class", TOK_ATTR_NAME, TOK_SUB_CLASS}, 1189 {"ulp", TOK_ATTR_NAME, TOK_SUB_ULP}, 1190 {"filter", TOK_ATTR_NAME, TOK_SUB_FILTER}, 1191 }; 1192 1193 /** 1194 * @brief Scan string and return next token 1195 * 1196 * The string is scanned and the next token is returned 1197 * 1198 * @param s input string to scan 1199 * @param tok pointer to place scanned token 1200 * 1201 * @return pointer to input string following scanned token, or NULL 1202 */ 1203 static const char * 1204 tokenize(const char *s, tok_t *tok) 1205 { 1206 uint32_t i; 1207 1208 memset(tok, 0, sizeof(*tok)); 1209 1210 /* Skip over whitespace */ 1211 while (*s && ocs_isspace(*s)) { 1212 s++; 1213 } 1214 1215 /* Return if nothing left in this string */ 1216 if (*s == 0) { 1217 return NULL; 1218 } 1219 1220 /* Look for single character matches */ 1221 for (i = 0; i < ARRAY_SIZE(cmatches); i++) { 1222 if (cmatches[i].s[0] == *s) { 1223 tok->type = cmatches[i].type; 1224 tok->subtype = cmatches[i].subtype; 1225 tok->string[0] = *s++; 1226 return s; 1227 } 1228 } 1229 1230 /* Scan for a hex number or decimal */ 1231 if ((s[0] == '0') && ((s[1] == 'x') || (s[1] == 'X'))) { 1232 char *p = tok->string; 1233 1234 tok->type = TOK_NUMBER; 1235 1236 *p++ = *s++; 1237 *p++ = *s++; 1238 while ((*s == '.') || ocs_isxdigit(*s)) { 1239 if ((p - tok->string) < (int32_t)sizeof(tok->string)) { 1240 *p++ = *s; 1241 } 1242 if (*s == ',') { 1243 tok->type = TOK_NUMBER_LIST; 1244 } 1245 s++; 1246 } 1247 *p = 0; 1248 return s; 1249 } else if (ocs_isdigit(*s)) { 1250 char *p = tok->string; 1251 1252 tok->type = TOK_NUMBER; 1253 while ((*s == ',') || ocs_isdigit(*s)) { 1254 if ((p - tok->string) < (int32_t)sizeof(tok->string)) { 1255 *p++ = *s; 1256 } 1257 if (*s == ',') { 1258 tok->type = TOK_NUMBER_LIST; 1259 } 1260 s++; 1261 } 1262 *p = 0; 1263 return s; 1264 } 1265 1266 /* Scan for an ID */ 1267 if (idstart(*s)) { 1268 char *p = tok->string; 1269 1270 for (*p++ = *s++; idchar(*s); s++) { 1271 if ((p - tok->string) < TOKEN_LEN) { 1272 *p++ = *s; 1273 } 1274 } 1275 1276 /* See if this is a $ number value */ 1277 if (tok->string[0] == '$') { 1278 tok->type = TOK_NUMBER_VALUE; 1279 } else { 1280 /* Look for a string match */ 1281 for (i = 0; i < ARRAY_SIZE(smatches); i++) { 1282 if (strcmp(smatches[i].s, tok->string) == 0) { 1283 tok->type = smatches[i].type; 1284 tok->subtype = smatches[i].subtype; 1285 return s; 1286 } 1287 } 1288 } 1289 } 1290 return s; 1291 } 1292 1293 /** 1294 * @brief convert token type to string 1295 * 1296 * @param type token type 1297 * 1298 * @return string, or "unknown" 1299 */ 1300 static const char * 1301 token_type2s(tok_type_e type) 1302 { 1303 switch(type) { 1304 #define P(x) case x: return #x; 1305 P(TOK_LPAREN) 1306 P(TOK_RPAREN) 1307 P(TOK_COLON) 1308 P(TOK_EQUALS) 1309 P(TOK_QUEUE) 1310 P(TOK_ATTR_NAME) 1311 P(TOK_NUMBER) 1312 P(TOK_NUMBER_VALUE) 1313 P(TOK_NUMBER_LIST) 1314 #undef P 1315 } 1316 return "unknown"; 1317 } 1318 1319 /** 1320 * @brief convert token sub-type to string 1321 * 1322 * @param subtype token sub-type 1323 * 1324 * @return string, or "unknown" 1325 */ 1326 static const char * 1327 token_subtype2s(tok_subtype_e subtype) 1328 { 1329 switch(subtype) { 1330 #define P(x) case x: return #x; 1331 P(TOK_SUB_EQ) 1332 P(TOK_SUB_CQ) 1333 P(TOK_SUB_RQ) 1334 P(TOK_SUB_MQ) 1335 P(TOK_SUB_WQ) 1336 P(TOK_SUB_LEN) 1337 P(TOK_SUB_CLASS) 1338 P(TOK_SUB_ULP) 1339 P(TOK_SUB_FILTER) 1340 #undef P 1341 } 1342 return ""; 1343 } 1344 1345 /** 1346 * @brief Generate syntax error message 1347 * 1348 * A syntax error message is found, the input tokens are dumped up to and including 1349 * the token that failed as indicated by the current iterator index. 1350 * 1351 * @param hw pointer to HW object 1352 * @param tokarray pointer to token array object 1353 * 1354 * @return none 1355 */ 1356 static void 1357 tok_syntax(ocs_hw_t *hw, tokarray_t *tokarray) 1358 { 1359 uint32_t i; 1360 tok_t *tok; 1361 1362 ocs_log_test(hw->os, "Syntax error:\n"); 1363 1364 for (i = 0, tok = tokarray->tokens; (i <= tokarray->inuse_count); i++, tok++) { 1365 ocs_log_test(hw->os, "%s [%2d] %-16s %-16s %s\n", (i == tokarray->iter_idx) ? ">>>" : " ", i, 1366 token_type2s(tok->type), token_subtype2s(tok->subtype), tok->string); 1367 } 1368 } 1369 1370 /** 1371 * @brief parse a number 1372 * 1373 * Parses tokens of type TOK_NUMBER and TOK_NUMBER_VALUE, returning a numeric value 1374 * 1375 * @param hw pointer to HW object 1376 * @param qtop pointer to QTOP object 1377 * @param tok pointer to token to parse 1378 * 1379 * @return numeric value 1380 */ 1381 static uint32_t 1382 tok_getnumber(ocs_hw_t *hw, ocs_hw_qtop_t *qtop, tok_t *tok) 1383 { 1384 uint32_t rval = 0; 1385 uint32_t num_cpus = ocs_get_num_cpus(); 1386 1387 switch(tok->type) { 1388 case TOK_NUMBER_VALUE: 1389 if (ocs_strcmp(tok->string, "$ncpu") == 0) { 1390 rval = num_cpus; 1391 } else if (ocs_strcmp(tok->string, "$ncpu1") == 0) { 1392 rval = num_cpus - 1; 1393 } else if (ocs_strcmp(tok->string, "$nwq") == 0) { 1394 if (hw != NULL) { 1395 rval = hw->config.n_wq; 1396 } 1397 } else if (ocs_strcmp(tok->string, "$maxmrq") == 0) { 1398 rval = MIN(num_cpus, OCS_HW_MAX_MRQS); 1399 } else if (ocs_strcmp(tok->string, "$nulp") == 0) { 1400 rval = hw->ulp_max - hw->ulp_start + 1; 1401 } else if ((qtop->rptcount_idx > 0) && ocs_strcmp(tok->string, "$rpt0") == 0) { 1402 rval = qtop->rptcount[qtop->rptcount_idx-1]; 1403 } else if ((qtop->rptcount_idx > 1) && ocs_strcmp(tok->string, "$rpt1") == 0) { 1404 rval = qtop->rptcount[qtop->rptcount_idx-2]; 1405 } else if ((qtop->rptcount_idx > 2) && ocs_strcmp(tok->string, "$rpt2") == 0) { 1406 rval = qtop->rptcount[qtop->rptcount_idx-3]; 1407 } else if ((qtop->rptcount_idx > 3) && ocs_strcmp(tok->string, "$rpt3") == 0) { 1408 rval = qtop->rptcount[qtop->rptcount_idx-4]; 1409 } else { 1410 rval = ocs_strtoul(tok->string, 0, 0); 1411 } 1412 break; 1413 case TOK_NUMBER: 1414 rval = ocs_strtoul(tok->string, 0, 0); 1415 break; 1416 default: 1417 break; 1418 } 1419 return rval; 1420 } 1421 1422 1423 /** 1424 * @brief parse an array of tokens 1425 * 1426 * The tokens are semantically parsed, to generate QTOP entries. 1427 * 1428 * @param hw pointer to HW object 1429 * @param tokarray array array of tokens 1430 * @param qtop ouptut QTOP object 1431 * 1432 * @return returns 0 for success, a negative error code value for failure. 1433 */ 1434 static int32_t 1435 parse_topology(ocs_hw_t *hw, tokarray_t *tokarray, ocs_hw_qtop_t *qtop) 1436 { 1437 ocs_hw_qtop_entry_t *qt = qtop->entries + qtop->inuse_count; 1438 tok_t *tok; 1439 1440 for (; (tokarray->iter_idx < tokarray->inuse_count) && 1441 ((tok = &tokarray->tokens[tokarray->iter_idx]) != NULL); ) { 1442 if (qtop->inuse_count >= qtop->alloc_count) { 1443 return -1; 1444 } 1445 1446 qt = qtop->entries + qtop->inuse_count; 1447 1448 switch (tok[0].type) 1449 { 1450 case TOK_QUEUE: 1451 qt->entry = subtype2qtop(tok[0].subtype); 1452 qt->set_default = FALSE; 1453 qt->len = 0; 1454 qt->class = 0; 1455 qtop->inuse_count++; 1456 1457 tokarray->iter_idx++; /* Advance current token index */ 1458 1459 /* Parse for queue attributes, possibly multiple instances */ 1460 while ((tokarray->iter_idx + 4) <= tokarray->inuse_count) { 1461 tok = &tokarray->tokens[tokarray->iter_idx]; 1462 if( (tok[0].type == TOK_COLON) && 1463 (tok[1].type == TOK_ATTR_NAME) && 1464 (tok[2].type == TOK_EQUALS) && 1465 ((tok[3].type == TOK_NUMBER) || 1466 (tok[3].type == TOK_NUMBER_VALUE) || 1467 (tok[3].type == TOK_NUMBER_LIST))) { 1468 1469 switch (tok[1].subtype) { 1470 case TOK_SUB_LEN: 1471 qt->len = tok_getnumber(hw, qtop, &tok[3]); 1472 break; 1473 1474 case TOK_SUB_CLASS: 1475 qt->class = tok_getnumber(hw, qtop, &tok[3]); 1476 break; 1477 1478 case TOK_SUB_ULP: 1479 qt->ulp = tok_getnumber(hw, qtop, &tok[3]); 1480 break; 1481 1482 case TOK_SUB_FILTER: 1483 if (tok[3].type == TOK_NUMBER_LIST) { 1484 uint32_t mask = 0; 1485 char *p = tok[3].string; 1486 1487 while ((p != NULL) && *p) { 1488 uint32_t v; 1489 1490 v = ocs_strtoul(p, 0, 0); 1491 if (v < 32) { 1492 mask |= (1U << v); 1493 } 1494 1495 p = ocs_strchr(p, ','); 1496 if (p != NULL) { 1497 p++; 1498 } 1499 } 1500 qt->filter_mask = mask; 1501 } else { 1502 qt->filter_mask = (1U << tok_getnumber(hw, qtop, &tok[3])); 1503 } 1504 break; 1505 default: 1506 break; 1507 } 1508 /* Advance current token index */ 1509 tokarray->iter_idx += 4; 1510 } else { 1511 break; 1512 } 1513 } 1514 qtop->entry_counts[qt->entry]++; 1515 break; 1516 1517 case TOK_ATTR_NAME: 1518 if ( ((tokarray->iter_idx + 5) <= tokarray->inuse_count) && 1519 (tok[1].type == TOK_COLON) && 1520 (tok[2].type == TOK_QUEUE) && 1521 (tok[3].type == TOK_EQUALS) && 1522 ((tok[4].type == TOK_NUMBER) || (tok[4].type == TOK_NUMBER_VALUE))) { 1523 qt->entry = subtype2qtop(tok[2].subtype); 1524 qt->set_default = TRUE; 1525 switch(tok[0].subtype) { 1526 case TOK_SUB_LEN: 1527 qt->len = tok_getnumber(hw, qtop, &tok[4]); 1528 break; 1529 case TOK_SUB_CLASS: 1530 qt->class = tok_getnumber(hw, qtop, &tok[4]); 1531 break; 1532 case TOK_SUB_ULP: 1533 qt->ulp = tok_getnumber(hw, qtop, &tok[4]); 1534 break; 1535 default: 1536 break; 1537 } 1538 qtop->inuse_count++; 1539 tokarray->iter_idx += 5; 1540 } else { 1541 tok_syntax(hw, tokarray); 1542 return -1; 1543 } 1544 break; 1545 1546 case TOK_NUMBER: 1547 case TOK_NUMBER_VALUE: { 1548 uint32_t rpt_count = 1; 1549 uint32_t i; 1550 1551 rpt_count = tok_getnumber(hw, qtop, tok); 1552 1553 if (tok[1].type == TOK_LPAREN) { 1554 uint32_t iter_idx_save; 1555 1556 tokarray->iter_idx += 2; 1557 1558 /* save token array iteration index */ 1559 iter_idx_save = tokarray->iter_idx; 1560 1561 for (i = 0; i < rpt_count; i++) { 1562 uint32_t rptcount_idx = qtop->rptcount_idx; 1563 1564 if (qtop->rptcount_idx < ARRAY_SIZE(qtop->rptcount)) { 1565 qtop->rptcount[qtop->rptcount_idx++] = i; 1566 } 1567 1568 /* restore token array iteration index */ 1569 tokarray->iter_idx = iter_idx_save; 1570 1571 /* parse, append to qtop */ 1572 parse_topology(hw, tokarray, qtop); 1573 1574 qtop->rptcount_idx = rptcount_idx; 1575 } 1576 } 1577 break; 1578 } 1579 1580 case TOK_RPAREN: 1581 tokarray->iter_idx++; 1582 return 0; 1583 1584 default: 1585 tok_syntax(hw, tokarray); 1586 return -1; 1587 } 1588 } 1589 return 0; 1590 } 1591 1592 /** 1593 * @brief Parse queue topology string 1594 * 1595 * The queue topology object is allocated, and filled with the results of parsing the 1596 * passed in queue topology string 1597 * 1598 * @param hw pointer to HW object 1599 * @param qtop_string input queue topology string 1600 * 1601 * @return pointer to allocated QTOP object, or NULL if there was an error 1602 */ 1603 ocs_hw_qtop_t * 1604 ocs_hw_qtop_parse(ocs_hw_t *hw, const char *qtop_string) 1605 { 1606 ocs_hw_qtop_t *qtop; 1607 tokarray_t tokarray; 1608 const char *s; 1609 #if HW_QTOP_DEBUG 1610 uint32_t i; 1611 ocs_hw_qtop_entry_t *qt; 1612 #endif 1613 1614 ocs_log_debug(hw->os, "queue topology: %s\n", qtop_string); 1615 1616 /* Allocate a token array */ 1617 tokarray.tokens = ocs_malloc(hw->os, MAX_TOKENS * sizeof(*tokarray.tokens), OCS_M_ZERO | OCS_M_NOWAIT); 1618 if (tokarray.tokens == NULL) { 1619 return NULL; 1620 } 1621 tokarray.alloc_count = MAX_TOKENS; 1622 tokarray.inuse_count = 0; 1623 tokarray.iter_idx = 0; 1624 1625 /* Parse the tokens */ 1626 for (s = qtop_string; (tokarray.inuse_count < tokarray.alloc_count) && 1627 ((s = tokenize(s, &tokarray.tokens[tokarray.inuse_count]))) != NULL; ) { 1628 tokarray.inuse_count++;; 1629 } 1630 1631 /* Allocate a queue topology structure */ 1632 qtop = ocs_malloc(hw->os, sizeof(*qtop), OCS_M_ZERO | OCS_M_NOWAIT); 1633 if (qtop == NULL) { 1634 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1635 ocs_log_err(hw->os, "malloc qtop failed\n"); 1636 return NULL; 1637 } 1638 qtop->os = hw->os; 1639 1640 /* Allocate queue topology entries */ 1641 qtop->entries = ocs_malloc(hw->os, OCS_HW_MAX_QTOP_ENTRIES*sizeof(*qtop->entries), OCS_M_ZERO | OCS_M_NOWAIT); 1642 if (qtop->entries == NULL) { 1643 ocs_log_err(hw->os, "malloc qtop entries failed\n"); 1644 ocs_free(hw->os, qtop, sizeof(*qtop)); 1645 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1646 return NULL; 1647 } 1648 qtop->alloc_count = OCS_HW_MAX_QTOP_ENTRIES; 1649 qtop->inuse_count = 0; 1650 1651 /* Parse the tokens */ 1652 parse_topology(hw, &tokarray, qtop); 1653 #if HW_QTOP_DEBUG 1654 for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) { 1655 ocs_log_debug(hw->os, "entry %s set_df %d len %4d class %d ulp %d\n", qtopentry2s(qt->entry), qt->set_default, qt->len, 1656 qt->class, qt->ulp); 1657 } 1658 #endif 1659 1660 /* Free the tokens array */ 1661 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1662 1663 return qtop; 1664 } 1665 1666 /** 1667 * @brief free queue topology object 1668 * 1669 * @param qtop pointer to QTOP object 1670 * 1671 * @return none 1672 */ 1673 void 1674 ocs_hw_qtop_free(ocs_hw_qtop_t *qtop) 1675 { 1676 if (qtop != NULL) { 1677 if (qtop->entries != NULL) { 1678 ocs_free(qtop->os, qtop->entries, qtop->alloc_count*sizeof(*qtop->entries)); 1679 } 1680 ocs_free(qtop->os, qtop, sizeof(*qtop)); 1681 } 1682 } 1683 1684 /* Uncomment this to turn on RQ debug */ 1685 // #define ENABLE_DEBUG_RQBUF 1686 1687 static int32_t ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id); 1688 static ocs_hw_sequence_t * ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex); 1689 static int32_t ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq); 1690 static ocs_hw_rtn_e ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq); 1691 1692 /** 1693 * @brief Process receive queue completions for RQ Pair mode. 1694 * 1695 * @par Description 1696 * RQ completions are processed. In RQ pair mode, a single header and single payload 1697 * buffer are received, and passed to the function that has registered for unsolicited 1698 * callbacks. 1699 * 1700 * @param hw Hardware context. 1701 * @param cq Pointer to HW completion queue. 1702 * @param cqe Completion queue entry. 1703 * 1704 * @return Returns 0 for success, or a negative error code value for failure. 1705 */ 1706 1707 int32_t 1708 ocs_hw_rqpair_process_rq(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1709 { 1710 uint16_t rq_id; 1711 uint32_t index; 1712 int32_t rqindex; 1713 int32_t rq_status; 1714 uint32_t h_len; 1715 uint32_t p_len; 1716 ocs_hw_sequence_t *seq; 1717 1718 rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index); 1719 if (0 != rq_status) { 1720 switch (rq_status) { 1721 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED: 1722 case SLI4_FC_ASYNC_RQ_DMA_FAILURE: 1723 /* just get RQ buffer then return to chip */ 1724 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1725 if (rqindex < 0) { 1726 ocs_log_test(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n", 1727 rq_status, rq_id); 1728 break; 1729 } 1730 1731 /* get RQ buffer */ 1732 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1733 1734 /* return to chip */ 1735 if (ocs_hw_rqpair_sequence_free(hw, seq)) { 1736 ocs_log_test(hw->os, "status=%#x, failed to return buffers to RQ\n", 1737 rq_status); 1738 break; 1739 } 1740 break; 1741 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED: 1742 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC: 1743 /* since RQ buffers were not consumed, cannot return them to chip */ 1744 /* fall through */ 1745 ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status); 1746 default: 1747 break; 1748 } 1749 return -1; 1750 } 1751 1752 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1753 if (rqindex < 0) { 1754 ocs_log_test(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id); 1755 return -1; 1756 } 1757 1758 OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++; 1759 rq->payload_use_count++;}) 1760 1761 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1762 ocs_hw_assert(seq != NULL); 1763 1764 seq->hw = hw; 1765 seq->auto_xrdy = 0; 1766 seq->out_of_xris = 0; 1767 seq->xri = 0; 1768 seq->hio = NULL; 1769 1770 sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len); 1771 seq->header->dma.len = h_len; 1772 seq->payload->dma.len = p_len; 1773 seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe); 1774 seq->hw_priv = cq->eq; 1775 1776 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1777 if (hw->config.bounce) { 1778 fc_header_t *hdr = seq->header->dma.virt; 1779 uint32_t s_id = fc_be24toh(hdr->s_id); 1780 uint32_t d_id = fc_be24toh(hdr->d_id); 1781 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1782 if (hw->callback.bounce != NULL) { 1783 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 1784 } 1785 } else { 1786 hw->callback.unsolicited(hw->args.unsolicited, seq); 1787 } 1788 1789 return 0; 1790 } 1791 1792 /** 1793 * @brief Process receive queue completions for RQ Pair mode - Auto xfer rdy 1794 * 1795 * @par Description 1796 * RQ completions are processed. In RQ pair mode, a single header and single payload 1797 * buffer are received, and passed to the function that has registered for unsolicited 1798 * callbacks. 1799 * 1800 * @param hw Hardware context. 1801 * @param cq Pointer to HW completion queue. 1802 * @param cqe Completion queue entry. 1803 * 1804 * @return Returns 0 for success, or a negative error code value for failure. 1805 */ 1806 1807 int32_t 1808 ocs_hw_rqpair_process_auto_xfr_rdy_cmd(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1809 { 1810 /* Seems silly to call a SLI function to decode - use the structure directly for performance */ 1811 sli4_fc_optimized_write_cmd_cqe_t *opt_wr = (sli4_fc_optimized_write_cmd_cqe_t*)cqe; 1812 uint16_t rq_id; 1813 uint32_t index; 1814 int32_t rqindex; 1815 int32_t rq_status; 1816 uint32_t h_len; 1817 uint32_t p_len; 1818 ocs_hw_sequence_t *seq; 1819 uint8_t axr_lock_taken = 0; 1820 #if defined(OCS_DISC_SPIN_DELAY) 1821 uint32_t delay = 0; 1822 char prop_buf[32]; 1823 #endif 1824 1825 rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index); 1826 if (0 != rq_status) { 1827 switch (rq_status) { 1828 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED: 1829 case SLI4_FC_ASYNC_RQ_DMA_FAILURE: 1830 /* just get RQ buffer then return to chip */ 1831 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1832 if (rqindex < 0) { 1833 ocs_log_err(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n", 1834 rq_status, rq_id); 1835 break; 1836 } 1837 1838 /* get RQ buffer */ 1839 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1840 1841 /* return to chip */ 1842 if (ocs_hw_rqpair_sequence_free(hw, seq)) { 1843 ocs_log_err(hw->os, "status=%#x, failed to return buffers to RQ\n", 1844 rq_status); 1845 break; 1846 } 1847 break; 1848 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED: 1849 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC: 1850 /* since RQ buffers were not consumed, cannot return them to chip */ 1851 ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status); 1852 /* fall through */ 1853 default: 1854 break; 1855 } 1856 return -1; 1857 } 1858 1859 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1860 if (rqindex < 0) { 1861 ocs_log_err(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id); 1862 return -1; 1863 } 1864 1865 OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++; 1866 rq->payload_use_count++;}) 1867 1868 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1869 ocs_hw_assert(seq != NULL); 1870 1871 seq->hw = hw; 1872 seq->auto_xrdy = opt_wr->agxr; 1873 seq->out_of_xris = opt_wr->oox; 1874 seq->xri = opt_wr->xri; 1875 seq->hio = NULL; 1876 1877 sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len); 1878 seq->header->dma.len = h_len; 1879 seq->payload->dma.len = p_len; 1880 seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe); 1881 seq->hw_priv = cq->eq; 1882 1883 if (seq->auto_xrdy) { 1884 fc_header_t *fc_hdr = seq->header->dma.virt; 1885 1886 seq->hio = ocs_hw_io_lookup(hw, seq->xri); 1887 ocs_lock(&seq->hio->axr_lock); 1888 axr_lock_taken = 1; 1889 1890 /* save the FCFI, src_id, dest_id and ox_id because we need it for the sequence object when the data comes. */ 1891 seq->hio->axr_buf->fcfi = seq->fcfi; 1892 seq->hio->axr_buf->hdr.ox_id = fc_hdr->ox_id; 1893 seq->hio->axr_buf->hdr.s_id = fc_hdr->s_id; 1894 seq->hio->axr_buf->hdr.d_id = fc_hdr->d_id; 1895 seq->hio->axr_buf->cmd_cqe = 1; 1896 1897 /* 1898 * Since auto xfer rdy is used for this IO, then clear the sequence 1899 * initiative bit in the header so that the upper layers wait for the 1900 * data. This should flow exactly like the first burst case. 1901 */ 1902 fc_hdr->f_ctl &= fc_htobe24(~FC_FCTL_SEQUENCE_INITIATIVE); 1903 1904 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */ 1905 if (seq->hio->type == OCS_HW_IO_TARGET_RSP) { 1906 seq->hio->axr_buf->call_axr_cmd = 1; 1907 seq->hio->axr_buf->cmd_seq = seq; 1908 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd; 1909 } 1910 } 1911 1912 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1913 if (hw->config.bounce) { 1914 fc_header_t *hdr = seq->header->dma.virt; 1915 uint32_t s_id = fc_be24toh(hdr->s_id); 1916 uint32_t d_id = fc_be24toh(hdr->d_id); 1917 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1918 if (hw->callback.bounce != NULL) { 1919 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 1920 } 1921 } else { 1922 hw->callback.unsolicited(hw->args.unsolicited, seq); 1923 } 1924 1925 if (seq->auto_xrdy) { 1926 /* If data cqe came before cmd cqe in out of order in case of AXR */ 1927 if(seq->hio->axr_buf->data_cqe == 1) { 1928 1929 #if defined(OCS_DISC_SPIN_DELAY) 1930 if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) { 1931 delay = ocs_strtoul(prop_buf, 0, 0); 1932 ocs_udelay(delay); 1933 } 1934 #endif 1935 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1936 if (hw->config.bounce) { 1937 fc_header_t *hdr = seq->header->dma.virt; 1938 uint32_t s_id = fc_be24toh(hdr->s_id); 1939 uint32_t d_id = fc_be24toh(hdr->d_id); 1940 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1941 if (hw->callback.bounce != NULL) { 1942 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, &seq->hio->axr_buf->seq, s_id, d_id, ox_id); 1943 } 1944 } else { 1945 hw->callback.unsolicited(hw->args.unsolicited, &seq->hio->axr_buf->seq); 1946 } 1947 } 1948 } 1949 1950 exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd: 1951 if(axr_lock_taken) { 1952 ocs_unlock(&seq->hio->axr_lock); 1953 } 1954 return 0; 1955 } 1956 1957 /** 1958 * @brief Process CQ completions for Auto xfer rdy data phases. 1959 * 1960 * @par Description 1961 * The data is DMA'd into the data buffer posted to the SGL prior to the XRI 1962 * being assigned to an IO. When the completion is received, All of the data 1963 * is in the single buffer. 1964 * 1965 * @param hw Hardware context. 1966 * @param cq Pointer to HW completion queue. 1967 * @param cqe Completion queue entry. 1968 * 1969 * @return Returns 0 for success, or a negative error code value for failure. 1970 */ 1971 1972 int32_t 1973 ocs_hw_rqpair_process_auto_xfr_rdy_data(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1974 { 1975 /* Seems silly to call a SLI function to decode - use the structure directly for performance */ 1976 sli4_fc_optimized_write_data_cqe_t *opt_wr = (sli4_fc_optimized_write_data_cqe_t*)cqe; 1977 ocs_hw_sequence_t *seq; 1978 ocs_hw_io_t *io; 1979 ocs_hw_auto_xfer_rdy_buffer_t *buf; 1980 #if defined(OCS_DISC_SPIN_DELAY) 1981 uint32_t delay = 0; 1982 char prop_buf[32]; 1983 #endif 1984 /* Look up the IO */ 1985 io = ocs_hw_io_lookup(hw, opt_wr->xri); 1986 ocs_lock(&io->axr_lock); 1987 buf = io->axr_buf; 1988 buf->data_cqe = 1; 1989 seq = &buf->seq; 1990 seq->hw = hw; 1991 seq->auto_xrdy = 1; 1992 seq->out_of_xris = 0; 1993 seq->xri = opt_wr->xri; 1994 seq->hio = io; 1995 seq->header = &buf->header; 1996 seq->payload = &buf->payload; 1997 1998 seq->header->dma.len = sizeof(fc_header_t); 1999 seq->payload->dma.len = opt_wr->total_data_placed; 2000 seq->fcfi = buf->fcfi; 2001 seq->hw_priv = cq->eq; 2002 2003 2004 if (opt_wr->status == SLI4_FC_WCQE_STATUS_SUCCESS) { 2005 seq->status = OCS_HW_UNSOL_SUCCESS; 2006 } else if (opt_wr->status == SLI4_FC_WCQE_STATUS_REMOTE_STOP) { 2007 seq->status = OCS_HW_UNSOL_ABTS_RCVD; 2008 } else { 2009 seq->status = OCS_HW_UNSOL_ERROR; 2010 } 2011 2012 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */ 2013 if(io->type == OCS_HW_IO_TARGET_RSP) { 2014 io->axr_buf->call_axr_data = 1; 2015 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data; 2016 } 2017 2018 if(!buf->cmd_cqe) { 2019 /* if data cqe came before cmd cqe, return here, cmd cqe will handle */ 2020 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data; 2021 } 2022 #if defined(OCS_DISC_SPIN_DELAY) 2023 if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) { 2024 delay = ocs_strtoul(prop_buf, 0, 0); 2025 ocs_udelay(delay); 2026 } 2027 #endif 2028 2029 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 2030 if (hw->config.bounce) { 2031 fc_header_t *hdr = seq->header->dma.virt; 2032 uint32_t s_id = fc_be24toh(hdr->s_id); 2033 uint32_t d_id = fc_be24toh(hdr->d_id); 2034 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 2035 if (hw->callback.bounce != NULL) { 2036 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 2037 } 2038 } else { 2039 hw->callback.unsolicited(hw->args.unsolicited, seq); 2040 } 2041 2042 exit_ocs_hw_rqpair_process_auto_xfr_rdy_data: 2043 ocs_unlock(&io->axr_lock); 2044 return 0; 2045 } 2046 2047 /** 2048 * @brief Return pointer to RQ buffer entry. 2049 * 2050 * @par Description 2051 * Returns a pointer to the RQ buffer entry given by @c rqindex and @c bufindex. 2052 * 2053 * @param hw Hardware context. 2054 * @param rqindex Index of the RQ that is being processed. 2055 * @param bufindex Index into the RQ that is being processed. 2056 * 2057 * @return Pointer to the sequence structure, or NULL otherwise. 2058 */ 2059 static ocs_hw_sequence_t * 2060 ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex) 2061 { 2062 sli4_queue_t *rq_hdr = &hw->rq[rqindex]; 2063 sli4_queue_t *rq_payload = &hw->rq[rqindex+1]; 2064 ocs_hw_sequence_t *seq = NULL; 2065 hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; 2066 2067 #if defined(ENABLE_DEBUG_RQBUF) 2068 uint64_t rqbuf_debug_value = 0xdead0000 | ((rq->id & 0xf) << 12) | (bufindex & 0xfff); 2069 #endif 2070 2071 if (bufindex >= rq_hdr->length) { 2072 ocs_log_err(hw->os, "RQ index %d bufindex %d exceed ring length %d for id %d\n", 2073 rqindex, bufindex, rq_hdr->length, rq_hdr->id); 2074 return NULL; 2075 } 2076 2077 sli_queue_lock(rq_hdr); 2078 sli_queue_lock(rq_payload); 2079 2080 #if defined(ENABLE_DEBUG_RQBUF) 2081 /* Put a debug value into the rq, to track which entries are still valid */ 2082 _sli_queue_poke(&hw->sli, rq_hdr, bufindex, (uint8_t *)&rqbuf_debug_value); 2083 _sli_queue_poke(&hw->sli, rq_payload, bufindex, (uint8_t *)&rqbuf_debug_value); 2084 #endif 2085 2086 seq = rq->rq_tracker[bufindex]; 2087 rq->rq_tracker[bufindex] = NULL; 2088 2089 if (seq == NULL ) { 2090 ocs_log_err(hw->os, "RQ buffer NULL, rqindex %d, bufindex %d, current q index = %d\n", 2091 rqindex, bufindex, rq_hdr->index); 2092 } 2093 2094 sli_queue_unlock(rq_payload); 2095 sli_queue_unlock(rq_hdr); 2096 return seq; 2097 } 2098 2099 /** 2100 * @brief Posts an RQ buffer to a queue and update the verification structures 2101 * 2102 * @param hw hardware context 2103 * @param seq Pointer to sequence object. 2104 * 2105 * @return Returns 0 on success, or a non-zero value otherwise. 2106 */ 2107 static int32_t 2108 ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2109 { 2110 sli4_queue_t *rq_hdr = &hw->rq[seq->header->rqindex]; 2111 sli4_queue_t *rq_payload = &hw->rq[seq->payload->rqindex]; 2112 uint32_t hw_rq_index = hw->hw_rq_lookup[seq->header->rqindex]; 2113 hw_rq_t *rq = hw->hw_rq[hw_rq_index]; 2114 uint32_t phys_hdr[2]; 2115 uint32_t phys_payload[2]; 2116 int32_t qindex_hdr; 2117 int32_t qindex_payload; 2118 2119 /* Update the RQ verification lookup tables */ 2120 phys_hdr[0] = ocs_addr32_hi(seq->header->dma.phys); 2121 phys_hdr[1] = ocs_addr32_lo(seq->header->dma.phys); 2122 phys_payload[0] = ocs_addr32_hi(seq->payload->dma.phys); 2123 phys_payload[1] = ocs_addr32_lo(seq->payload->dma.phys); 2124 2125 sli_queue_lock(rq_hdr); 2126 sli_queue_lock(rq_payload); 2127 2128 /* 2129 * Note: The header must be posted last for buffer pair mode because 2130 * posting on the header queue posts the payload queue as well. 2131 * We do not ring the payload queue independently in RQ pair mode. 2132 */ 2133 qindex_payload = _sli_queue_write(&hw->sli, rq_payload, (void *)phys_payload); 2134 qindex_hdr = _sli_queue_write(&hw->sli, rq_hdr, (void *)phys_hdr); 2135 if (qindex_hdr < 0 || 2136 qindex_payload < 0) { 2137 ocs_log_err(hw->os, "RQ_ID=%#x write failed\n", rq_hdr->id); 2138 sli_queue_unlock(rq_payload); 2139 sli_queue_unlock(rq_hdr); 2140 return OCS_HW_RTN_ERROR; 2141 } 2142 2143 /* ensure the indexes are the same */ 2144 ocs_hw_assert(qindex_hdr == qindex_payload); 2145 2146 /* Update the lookup table */ 2147 if (rq->rq_tracker[qindex_hdr] == NULL) { 2148 rq->rq_tracker[qindex_hdr] = seq; 2149 } else { 2150 ocs_log_test(hw->os, "expected rq_tracker[%d][%d] buffer to be NULL\n", 2151 hw_rq_index, qindex_hdr); 2152 } 2153 2154 sli_queue_unlock(rq_payload); 2155 sli_queue_unlock(rq_hdr); 2156 return OCS_HW_RTN_SUCCESS; 2157 } 2158 2159 /** 2160 * @brief Return RQ buffers (while in RQ pair mode). 2161 * 2162 * @par Description 2163 * The header and payload buffers are returned to the Receive Queue. 2164 * 2165 * @param hw Hardware context. 2166 * @param seq Header/payload sequence buffers. 2167 * 2168 * @return Returns OCS_HW_RTN_SUCCESS on success, or an error code value on failure. 2169 */ 2170 2171 ocs_hw_rtn_e 2172 ocs_hw_rqpair_sequence_free(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2173 { 2174 ocs_hw_rtn_e rc = OCS_HW_RTN_SUCCESS; 2175 2176 /* Check for auto xfer rdy dummy buffers and call the proper release function. */ 2177 if (seq->header->rqindex == OCS_HW_RQ_INDEX_DUMMY_HDR) { 2178 return ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(hw, seq); 2179 } 2180 2181 /* 2182 * Post the data buffer first. Because in RQ pair mode, ringing the 2183 * doorbell of the header ring will post the data buffer as well. 2184 */ 2185 if (ocs_hw_rqpair_put(hw, seq)) { 2186 ocs_log_err(hw->os, "error writing buffers\n"); 2187 return OCS_HW_RTN_ERROR; 2188 } 2189 2190 return rc; 2191 } 2192 2193 /** 2194 * @brief Find the RQ index of RQ_ID. 2195 * 2196 * @param hw Hardware context. 2197 * @param rq_id RQ ID to find. 2198 * 2199 * @return Returns the RQ index, or -1 if not found 2200 */ 2201 static inline int32_t 2202 ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id) 2203 { 2204 return ocs_hw_queue_hash_find(hw->rq_hash, rq_id); 2205 } 2206 2207 /** 2208 * @ingroup devInitShutdown 2209 * @brief Allocate auto xfer rdy buffers. 2210 * 2211 * @par Description 2212 * Allocates the auto xfer rdy buffers and places them on the free list. 2213 * 2214 * @param hw Hardware context allocated by the caller. 2215 * @param num_buffers Number of buffers to allocate. 2216 * 2217 * @return Returns 0 on success, or a non-zero value on failure. 2218 */ 2219 ocs_hw_rtn_e 2220 ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(ocs_hw_t *hw, uint32_t num_buffers) 2221 { 2222 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2223 uint32_t i; 2224 2225 hw->auto_xfer_rdy_buf_pool = ocs_pool_alloc(hw->os, sizeof(ocs_hw_auto_xfer_rdy_buffer_t), num_buffers, FALSE); 2226 if (hw->auto_xfer_rdy_buf_pool == NULL) { 2227 ocs_log_err(hw->os, "Failure to allocate auto xfer ready buffer pool\n"); 2228 return OCS_HW_RTN_NO_MEMORY; 2229 } 2230 2231 for (i = 0; i < num_buffers; i++) { 2232 /* allocate the wrapper object */ 2233 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i); 2234 ocs_hw_assert(buf != NULL); 2235 2236 /* allocate the auto xfer ready buffer */ 2237 if (ocs_dma_alloc(hw->os, &buf->payload.dma, hw->config.auto_xfer_rdy_size, OCS_MIN_DMA_ALIGNMENT)) { 2238 ocs_log_err(hw->os, "DMA allocation failed\n"); 2239 ocs_free(hw->os, buf, sizeof(*buf)); 2240 return OCS_HW_RTN_NO_MEMORY; 2241 } 2242 2243 /* build a fake data header in big endian */ 2244 buf->hdr.info = FC_RCTL_INFO_SOL_DATA; 2245 buf->hdr.r_ctl = FC_RCTL_FC4_DATA; 2246 buf->hdr.type = FC_TYPE_FCP; 2247 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER | 2248 FC_FCTL_FIRST_SEQUENCE | 2249 FC_FCTL_LAST_SEQUENCE | 2250 FC_FCTL_END_SEQUENCE | 2251 FC_FCTL_SEQUENCE_INITIATIVE); 2252 2253 /* build the fake header DMA object */ 2254 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR; 2255 buf->header.dma.virt = &buf->hdr; 2256 buf->header.dma.alloc = buf; 2257 buf->header.dma.size = sizeof(buf->hdr); 2258 buf->header.dma.len = sizeof(buf->hdr); 2259 2260 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA; 2261 } 2262 return OCS_HW_RTN_SUCCESS; 2263 } 2264 2265 /** 2266 * @ingroup devInitShutdown 2267 * @brief Post Auto xfer rdy buffers to the XRIs posted with DNRX. 2268 * 2269 * @par Description 2270 * When new buffers are freed, check existing XRIs waiting for buffers. 2271 * 2272 * @param hw Hardware context allocated by the caller. 2273 */ 2274 static void 2275 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(ocs_hw_t *hw) 2276 { 2277 ocs_hw_io_t *io; 2278 int32_t rc; 2279 2280 ocs_lock(&hw->io_lock); 2281 2282 while (!ocs_list_empty(&hw->io_port_dnrx)) { 2283 io = ocs_list_remove_head(&hw->io_port_dnrx); 2284 rc = ocs_hw_reque_xri(hw, io); 2285 if(rc) { 2286 break; 2287 } 2288 } 2289 2290 ocs_unlock(&hw->io_lock); 2291 } 2292 2293 /** 2294 * @brief Called when the POST_SGL_PAGE command completes. 2295 * 2296 * @par Description 2297 * Free the mailbox command buffer. 2298 * 2299 * @param hw Hardware context. 2300 * @param status Status field from the mbox completion. 2301 * @param mqe Mailbox response structure. 2302 * @param arg Pointer to a callback function that signals the caller that the command is done. 2303 * 2304 * @return Returns 0. 2305 */ 2306 static int32_t 2307 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb(ocs_hw_t *hw, int32_t status, uint8_t *mqe, void *arg) 2308 { 2309 if (status != 0) { 2310 ocs_log_debug(hw->os, "Status 0x%x\n", status); 2311 } 2312 2313 ocs_free(hw->os, mqe, SLI4_BMBX_SIZE); 2314 return 0; 2315 } 2316 2317 /** 2318 * @brief Prepares an XRI to move to the chip. 2319 * 2320 * @par Description 2321 * Puts the data SGL into the SGL list for the IO object and possibly registers 2322 * an SGL list for the XRI. Since both the POST_XRI and POST_SGL_PAGES commands are 2323 * mailbox commands, we don't need to wait for completion before preceding. 2324 * 2325 * @param hw Hardware context allocated by the caller. 2326 * @param io Pointer to the IO object. 2327 * 2328 * @return Returns OCS_HW_RTN_SUCCESS for success, or an error code value for failure. 2329 */ 2330 ocs_hw_rtn_e 2331 ocs_hw_rqpair_auto_xfer_rdy_move_to_port(ocs_hw_t *hw, ocs_hw_io_t *io) 2332 { 2333 /* We only need to preregister the SGL if it has not yet been done. */ 2334 if (!sli_get_sgl_preregister(&hw->sli)) { 2335 uint8_t *post_sgl; 2336 ocs_dma_t *psgls = &io->def_sgl; 2337 ocs_dma_t **sgls = &psgls; 2338 2339 /* non-local buffer required for mailbox queue */ 2340 post_sgl = ocs_malloc(hw->os, SLI4_BMBX_SIZE, OCS_M_NOWAIT); 2341 if (post_sgl == NULL) { 2342 ocs_log_err(hw->os, "no buffer for command\n"); 2343 return OCS_HW_RTN_NO_MEMORY; 2344 } 2345 if (sli_cmd_fcoe_post_sgl_pages(&hw->sli, post_sgl, SLI4_BMBX_SIZE, 2346 io->indicator, 1, sgls, NULL, NULL)) { 2347 if (ocs_hw_command(hw, post_sgl, OCS_CMD_NOWAIT, 2348 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb, NULL)) { 2349 ocs_free(hw->os, post_sgl, SLI4_BMBX_SIZE); 2350 ocs_log_err(hw->os, "SGL post failed\n"); 2351 return OCS_HW_RTN_ERROR; 2352 } 2353 } 2354 } 2355 2356 ocs_lock(&hw->io_lock); 2357 if (ocs_hw_rqpair_auto_xfer_rdy_buffer_post(hw, io, 0) != 0) { /* DNRX set - no buffer */ 2358 ocs_unlock(&hw->io_lock); 2359 return OCS_HW_RTN_ERROR; 2360 } 2361 ocs_unlock(&hw->io_lock); 2362 return OCS_HW_RTN_SUCCESS; 2363 } 2364 2365 /** 2366 * @brief Prepares an XRI to move back to the host. 2367 * 2368 * @par Description 2369 * Releases any attached buffer back to the pool. 2370 * 2371 * @param hw Hardware context allocated by the caller. 2372 * @param io Pointer to the IO object. 2373 */ 2374 void 2375 ocs_hw_rqpair_auto_xfer_rdy_move_to_host(ocs_hw_t *hw, ocs_hw_io_t *io) 2376 { 2377 if (io->axr_buf != NULL) { 2378 ocs_lock(&hw->io_lock); 2379 /* check list and remove if there */ 2380 if (ocs_list_on_list(&io->dnrx_link)) { 2381 ocs_list_remove(&hw->io_port_dnrx, io); 2382 io->auto_xfer_rdy_dnrx = 0; 2383 2384 /* release the count for waiting for a buffer */ 2385 ocs_hw_io_free(hw, io); 2386 } 2387 2388 ocs_pool_put(hw->auto_xfer_rdy_buf_pool, io->axr_buf); 2389 io->axr_buf = NULL; 2390 ocs_unlock(&hw->io_lock); 2391 2392 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw); 2393 } 2394 return; 2395 } 2396 2397 2398 /** 2399 * @brief Posts an auto xfer rdy buffer to an IO. 2400 * 2401 * @par Description 2402 * Puts the data SGL into the SGL list for the IO object 2403 * @n @name 2404 * @b Note: io_lock must be held. 2405 * 2406 * @param hw Hardware context allocated by the caller. 2407 * @param io Pointer to the IO object. 2408 * 2409 * @return Returns the value of DNRX bit in the TRSP and ABORT WQEs. 2410 */ 2411 uint8_t 2412 ocs_hw_rqpair_auto_xfer_rdy_buffer_post(ocs_hw_t *hw, ocs_hw_io_t *io, int reuse_buf) 2413 { 2414 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2415 sli4_sge_t *data; 2416 2417 if(!reuse_buf) { 2418 buf = ocs_pool_get(hw->auto_xfer_rdy_buf_pool); 2419 io->axr_buf = buf; 2420 } 2421 2422 data = io->def_sgl.virt; 2423 data[0].sge_type = SLI4_SGE_TYPE_SKIP; 2424 data[0].last = 0; 2425 2426 /* 2427 * Note: if we are doing DIF assists, then the SGE[1] must contain the 2428 * DI_SEED SGE. The host is responsible for programming: 2429 * SGE Type (Word 2, bits 30:27) 2430 * Replacement App Tag (Word 2 bits 15:0) 2431 * App Tag (Word 3 bits 15:0) 2432 * New Ref Tag (Word 3 bit 23) 2433 * Metadata Enable (Word 3 bit 20) 2434 * Auto-Increment RefTag (Word 3 bit 19) 2435 * Block Size (Word 3 bits 18:16) 2436 * The following fields are managed by the SLI Port: 2437 * Ref Tag Compare (Word 0) 2438 * Replacement Ref Tag (Word 1) - In not the LBA 2439 * NA (Word 2 bit 25) 2440 * Opcode RX (Word 3 bits 27:24) 2441 * Checksum Enable (Word 3 bit 22) 2442 * RefTag Enable (Word 3 bit 21) 2443 * 2444 * The first two SGLs are cleared by ocs_hw_io_init_sges(), so assume eveything is cleared. 2445 */ 2446 if (hw->config.auto_xfer_rdy_p_type) { 2447 sli4_diseed_sge_t *diseed = (sli4_diseed_sge_t*)&data[1]; 2448 2449 diseed->sge_type = SLI4_SGE_TYPE_DISEED; 2450 diseed->repl_app_tag = hw->config.auto_xfer_rdy_app_tag_value; 2451 diseed->app_tag_cmp = hw->config.auto_xfer_rdy_app_tag_value; 2452 diseed->check_app_tag = hw->config.auto_xfer_rdy_app_tag_valid; 2453 diseed->auto_incr_ref_tag = TRUE; /* Always the LBA */ 2454 diseed->dif_blk_size = hw->config.auto_xfer_rdy_blk_size_chip; 2455 } else { 2456 data[1].sge_type = SLI4_SGE_TYPE_SKIP; 2457 data[1].last = 0; 2458 } 2459 2460 data[2].sge_type = SLI4_SGE_TYPE_DATA; 2461 data[2].buffer_address_high = ocs_addr32_hi(io->axr_buf->payload.dma.phys); 2462 data[2].buffer_address_low = ocs_addr32_lo(io->axr_buf->payload.dma.phys); 2463 data[2].buffer_length = io->axr_buf->payload.dma.size; 2464 data[2].last = TRUE; 2465 data[3].sge_type = SLI4_SGE_TYPE_SKIP; 2466 2467 return 0; 2468 } 2469 2470 /** 2471 * @brief Return auto xfer ready buffers (while in RQ pair mode). 2472 * 2473 * @par Description 2474 * The header and payload buffers are returned to the auto xfer rdy pool. 2475 * 2476 * @param hw Hardware context. 2477 * @param seq Header/payload sequence buffers. 2478 * 2479 * @return Returns OCS_HW_RTN_SUCCESS for success, an error code value for failure. 2480 */ 2481 2482 static ocs_hw_rtn_e 2483 ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2484 { 2485 ocs_hw_auto_xfer_rdy_buffer_t *buf = seq->header->dma.alloc; 2486 2487 buf->data_cqe = 0; 2488 buf->cmd_cqe = 0; 2489 buf->fcfi = 0; 2490 buf->call_axr_cmd = 0; 2491 buf->call_axr_data = 0; 2492 2493 /* build a fake data header in big endian */ 2494 buf->hdr.info = FC_RCTL_INFO_SOL_DATA; 2495 buf->hdr.r_ctl = FC_RCTL_FC4_DATA; 2496 buf->hdr.type = FC_TYPE_FCP; 2497 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER | 2498 FC_FCTL_FIRST_SEQUENCE | 2499 FC_FCTL_LAST_SEQUENCE | 2500 FC_FCTL_END_SEQUENCE | 2501 FC_FCTL_SEQUENCE_INITIATIVE); 2502 2503 /* build the fake header DMA object */ 2504 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR; 2505 buf->header.dma.virt = &buf->hdr; 2506 buf->header.dma.alloc = buf; 2507 buf->header.dma.size = sizeof(buf->hdr); 2508 buf->header.dma.len = sizeof(buf->hdr); 2509 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA; 2510 2511 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw); 2512 2513 return OCS_HW_RTN_SUCCESS; 2514 } 2515 2516 /** 2517 * @ingroup devInitShutdown 2518 * @brief Free auto xfer rdy buffers. 2519 * 2520 * @par Description 2521 * Frees the auto xfer rdy buffers. 2522 * 2523 * @param hw Hardware context allocated by the caller. 2524 * 2525 * @return Returns 0 on success, or a non-zero value on failure. 2526 */ 2527 static void 2528 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(ocs_hw_t *hw) 2529 { 2530 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2531 uint32_t i; 2532 2533 if (hw->auto_xfer_rdy_buf_pool != NULL) { 2534 ocs_lock(&hw->io_lock); 2535 for (i = 0; i < ocs_pool_get_count(hw->auto_xfer_rdy_buf_pool); i++) { 2536 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i); 2537 if (buf != NULL) { 2538 ocs_dma_free(hw->os, &buf->payload.dma); 2539 } 2540 } 2541 ocs_unlock(&hw->io_lock); 2542 2543 ocs_pool_free(hw->auto_xfer_rdy_buf_pool); 2544 hw->auto_xfer_rdy_buf_pool = NULL; 2545 } 2546 } 2547 2548 /** 2549 * @ingroup devInitShutdown 2550 * @brief Configure the rq_pair function from ocs_hw_init(). 2551 * 2552 * @par Description 2553 * Allocates the buffers to auto xfer rdy and posts initial XRIs for this feature. 2554 * 2555 * @param hw Hardware context allocated by the caller. 2556 * 2557 * @return Returns 0 on success, or a non-zero value on failure. 2558 */ 2559 ocs_hw_rtn_e 2560 ocs_hw_rqpair_init(ocs_hw_t *hw) 2561 { 2562 ocs_hw_rtn_e rc; 2563 uint32_t xris_posted; 2564 2565 ocs_log_debug(hw->os, "RQ Pair mode\n"); 2566 2567 /* 2568 * If we get this far, the auto XFR_RDY feature was enabled successfully, otherwise ocs_hw_init() would 2569 * return with an error. So allocate the buffers based on the initial XRI pool required to support this 2570 * feature. 2571 */ 2572 if (sli_get_auto_xfer_rdy_capable(&hw->sli) && 2573 hw->config.auto_xfer_rdy_size > 0) { 2574 if (hw->auto_xfer_rdy_buf_pool == NULL) { 2575 /* 2576 * Allocate one more buffer than XRIs so that when all the XRIs are in use, we still have 2577 * one to post back for the case where the response phase is started in the context of 2578 * the data completion. 2579 */ 2580 rc = ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(hw, hw->config.auto_xfer_rdy_xri_cnt + 1); 2581 if (rc != OCS_HW_RTN_SUCCESS) { 2582 return rc; 2583 } 2584 } else { 2585 ocs_pool_reset(hw->auto_xfer_rdy_buf_pool); 2586 } 2587 2588 /* Post the auto XFR_RDY XRIs */ 2589 xris_posted = ocs_hw_xri_move_to_port_owned(hw, hw->config.auto_xfer_rdy_xri_cnt); 2590 if (xris_posted != hw->config.auto_xfer_rdy_xri_cnt) { 2591 ocs_log_err(hw->os, "post_xri failed, only posted %d XRIs\n", xris_posted); 2592 return OCS_HW_RTN_ERROR; 2593 } 2594 } 2595 2596 return 0; 2597 } 2598 2599 /** 2600 * @ingroup devInitShutdown 2601 * @brief Tear down the rq_pair function from ocs_hw_teardown(). 2602 * 2603 * @par Description 2604 * Frees the buffers to auto xfer rdy. 2605 * 2606 * @param hw Hardware context allocated by the caller. 2607 */ 2608 void 2609 ocs_hw_rqpair_teardown(ocs_hw_t *hw) 2610 { 2611 /* We need to free any auto xfer ready buffers */ 2612 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(hw); 2613 } 2614