1 /*- 2 * Copyright (c) 2017 Broadcom. All rights reserved. 3 * The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright notice, 12 * this list of conditions and the following disclaimer in the documentation 13 * and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the copyright holder nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 34 /** 35 * @file 36 * 37 */ 38 39 #include "ocs_os.h" 40 #include "ocs_hw.h" 41 #include "ocs_hw_queues.h" 42 43 #define HW_QTOP_DEBUG 0 44 45 /** 46 * @brief Initialize queues 47 * 48 * Given the parsed queue topology spec, the SLI queues are created and 49 * initialized 50 * 51 * @param hw pointer to HW object 52 * @param qtop pointer to queue topology 53 * 54 * @return returns 0 for success, an error code value for failure. 55 */ 56 ocs_hw_rtn_e 57 ocs_hw_init_queues(ocs_hw_t *hw, ocs_hw_qtop_t *qtop) 58 { 59 uint32_t i, j; 60 uint32_t default_lengths[QTOP_LAST], len; 61 uint32_t rqset_len = 0, rqset_ulp = 0, rqset_count = 0; 62 uint8_t rqset_filter_mask = 0; 63 hw_eq_t *eqs[hw->config.n_rq]; 64 hw_cq_t *cqs[hw->config.n_rq]; 65 hw_rq_t *rqs[hw->config.n_rq]; 66 ocs_hw_qtop_entry_t *qt, *next_qt; 67 ocs_hw_mrq_t mrq; 68 bool use_mrq = FALSE; 69 70 hw_eq_t *eq = NULL; 71 hw_cq_t *cq = NULL; 72 hw_wq_t *wq = NULL; 73 hw_rq_t *rq = NULL; 74 hw_mq_t *mq = NULL; 75 76 mrq.num_pairs = 0; 77 default_lengths[QTOP_EQ] = 1024; 78 default_lengths[QTOP_CQ] = hw->num_qentries[SLI_QTYPE_CQ]; 79 default_lengths[QTOP_WQ] = hw->num_qentries[SLI_QTYPE_WQ]; 80 default_lengths[QTOP_RQ] = hw->num_qentries[SLI_QTYPE_RQ]; 81 default_lengths[QTOP_MQ] = OCS_HW_MQ_DEPTH; 82 83 ocs_hw_verify(hw != NULL, OCS_HW_RTN_INVALID_ARG); 84 85 hw->eq_count = 0; 86 hw->cq_count = 0; 87 hw->mq_count = 0; 88 hw->wq_count = 0; 89 hw->rq_count = 0; 90 hw->hw_rq_count = 0; 91 ocs_list_init(&hw->eq_list, hw_eq_t, link); 92 93 /* If MRQ is requested, Check if it is supported by SLI. */ 94 if ((hw->config.n_rq > 1 ) && !hw->sli.config.features.flag.mrqp) { 95 ocs_log_err(hw->os, "MRQ topology not supported by SLI4.\n"); 96 return OCS_HW_RTN_ERROR; 97 } 98 99 if (hw->config.n_rq > 1) 100 use_mrq = TRUE; 101 102 /* Allocate class WQ pools */ 103 for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) { 104 hw->wq_class_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 105 if (hw->wq_class_array[i] == NULL) { 106 ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n"); 107 return OCS_HW_RTN_NO_MEMORY; 108 } 109 } 110 111 /* Allocate per CPU WQ pools */ 112 for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) { 113 hw->wq_cpu_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 114 if (hw->wq_cpu_array[i] == NULL) { 115 ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n"); 116 return OCS_HW_RTN_NO_MEMORY; 117 } 118 } 119 120 ocs_hw_assert(qtop != NULL); 121 122 for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) { 123 if (i == qtop->inuse_count - 1) 124 next_qt = NULL; 125 else 126 next_qt = qt + 1; 127 128 switch(qt->entry) { 129 case QTOP_EQ: 130 len = (qt->len) ? qt->len : default_lengths[QTOP_EQ]; 131 132 if (qt->set_default) { 133 default_lengths[QTOP_EQ] = len; 134 break; 135 } 136 137 eq = hw_new_eq(hw, len); 138 if (eq == NULL) { 139 hw_queue_teardown(hw); 140 return OCS_HW_RTN_NO_MEMORY; 141 } 142 break; 143 144 case QTOP_CQ: 145 len = (qt->len) ? qt->len : default_lengths[QTOP_CQ]; 146 147 if (qt->set_default) { 148 default_lengths[QTOP_CQ] = len; 149 break; 150 } 151 152 if (!eq || !next_qt) { 153 goto fail; 154 } 155 156 /* If this CQ is for MRQ, then delay the creation */ 157 if (!use_mrq || next_qt->entry != QTOP_RQ) { 158 cq = hw_new_cq(eq, len); 159 if (cq == NULL) { 160 goto fail; 161 } 162 } 163 break; 164 165 case QTOP_WQ: { 166 len = (qt->len) ? qt->len : default_lengths[QTOP_WQ]; 167 if (qt->set_default) { 168 default_lengths[QTOP_WQ] = len; 169 break; 170 } 171 172 if ((hw->ulp_start + qt->ulp) > hw->ulp_max) { 173 ocs_log_err(hw->os, "invalid ULP %d for WQ\n", qt->ulp); 174 hw_queue_teardown(hw); 175 return OCS_HW_RTN_NO_MEMORY; 176 } 177 178 if (cq == NULL) 179 goto fail; 180 181 wq = hw_new_wq(cq, len, qt->class, hw->ulp_start + qt->ulp); 182 if (wq == NULL) { 183 goto fail; 184 } 185 186 /* Place this WQ on the EQ WQ array */ 187 if (ocs_varray_add(eq->wq_array, wq)) { 188 ocs_log_err(hw->os, "QTOP_WQ: EQ ocs_varray_add failed\n"); 189 hw_queue_teardown(hw); 190 return OCS_HW_RTN_ERROR; 191 } 192 193 /* Place this WQ on the HW class array */ 194 if (qt->class < ARRAY_SIZE(hw->wq_class_array)) { 195 if (ocs_varray_add(hw->wq_class_array[qt->class], wq)) { 196 ocs_log_err(hw->os, "HW wq_class_array ocs_varray_add failed\n"); 197 hw_queue_teardown(hw); 198 return OCS_HW_RTN_ERROR; 199 } 200 } else { 201 ocs_log_err(hw->os, "Invalid class value: %d\n", qt->class); 202 hw_queue_teardown(hw); 203 return OCS_HW_RTN_ERROR; 204 } 205 206 /* 207 * Place this WQ on the per CPU list, asumming that EQs are mapped to cpu given 208 * by the EQ instance modulo number of CPUs 209 */ 210 if (ocs_varray_add(hw->wq_cpu_array[eq->instance % ocs_get_num_cpus()], wq)) { 211 ocs_log_err(hw->os, "HW wq_cpu_array ocs_varray_add failed\n"); 212 hw_queue_teardown(hw); 213 return OCS_HW_RTN_ERROR; 214 } 215 216 break; 217 } 218 case QTOP_RQ: { 219 len = (qt->len) ? qt->len : default_lengths[QTOP_RQ]; 220 if (qt->set_default) { 221 default_lengths[QTOP_RQ] = len; 222 break; 223 } 224 225 if ((hw->ulp_start + qt->ulp) > hw->ulp_max) { 226 ocs_log_err(hw->os, "invalid ULP %d for RQ\n", qt->ulp); 227 hw_queue_teardown(hw); 228 return OCS_HW_RTN_NO_MEMORY; 229 } 230 231 if (use_mrq) { 232 mrq.rq_cfg[mrq.num_pairs].len = len; 233 mrq.rq_cfg[mrq.num_pairs].ulp = hw->ulp_start + qt->ulp; 234 mrq.rq_cfg[mrq.num_pairs].filter_mask = qt->filter_mask; 235 mrq.rq_cfg[mrq.num_pairs].eq = eq; 236 mrq.num_pairs ++; 237 } else { 238 rq = hw_new_rq(cq, len, hw->ulp_start + qt->ulp); 239 if (rq == NULL) { 240 hw_queue_teardown(hw); 241 return OCS_HW_RTN_NO_MEMORY; 242 } 243 rq->filter_mask = qt->filter_mask; 244 } 245 break; 246 } 247 248 case QTOP_MQ: 249 len = (qt->len) ? qt->len : default_lengths[QTOP_MQ]; 250 if (qt->set_default) { 251 default_lengths[QTOP_MQ] = len; 252 break; 253 } 254 255 if (cq == NULL) 256 goto fail; 257 258 mq = hw_new_mq(cq, len); 259 if (mq == NULL) { 260 goto fail; 261 } 262 break; 263 264 default: 265 ocs_hw_assert(0); 266 break; 267 } 268 } 269 270 if (mrq.num_pairs) { 271 /* First create normal RQs. */ 272 for (i = 0; i < mrq.num_pairs; i++) { 273 for (j = 0; j < mrq.num_pairs; j++) { 274 if ((i != j) && (mrq.rq_cfg[i].filter_mask == mrq.rq_cfg[j].filter_mask)) { 275 /* This should be created using set */ 276 if (rqset_filter_mask && (rqset_filter_mask != mrq.rq_cfg[i].filter_mask)) { 277 ocs_log_crit(hw->os, "Cant create morethan one RQ Set\n"); 278 hw_queue_teardown(hw); 279 return OCS_HW_RTN_ERROR; 280 } else if (!rqset_filter_mask){ 281 rqset_filter_mask = mrq.rq_cfg[i].filter_mask; 282 rqset_len = mrq.rq_cfg[i].len; 283 rqset_ulp = mrq.rq_cfg[i].ulp; 284 } 285 eqs[rqset_count] = mrq.rq_cfg[i].eq; 286 rqset_count++; 287 break; 288 } 289 } 290 if (j == mrq.num_pairs) { 291 /* Normal RQ */ 292 cq = hw_new_cq(mrq.rq_cfg[i].eq, default_lengths[QTOP_CQ]); 293 if (cq == NULL) { 294 hw_queue_teardown(hw); 295 return OCS_HW_RTN_NO_MEMORY; 296 } 297 298 rq = hw_new_rq(cq, mrq.rq_cfg[i].len, mrq.rq_cfg[i].ulp); 299 if (rq == NULL) { 300 hw_queue_teardown(hw); 301 return OCS_HW_RTN_NO_MEMORY; 302 } 303 rq->filter_mask = mrq.rq_cfg[i].filter_mask; 304 } 305 } 306 307 /* Now create RQ Set */ 308 if (rqset_count) { 309 if (rqset_count > OCE_HW_MAX_NUM_MRQ_PAIRS) { 310 ocs_log_crit(hw->os, 311 "Max Supported MRQ pairs = %d\n", 312 OCE_HW_MAX_NUM_MRQ_PAIRS); 313 hw_queue_teardown(hw); 314 return OCS_HW_RTN_ERROR; 315 } 316 317 /* Create CQ set */ 318 if (hw_new_cq_set(eqs, cqs, rqset_count, default_lengths[QTOP_CQ])) { 319 hw_queue_teardown(hw); 320 return OCS_HW_RTN_ERROR; 321 } 322 323 /* Create RQ set */ 324 if (hw_new_rq_set(cqs, rqs, rqset_count, rqset_len, rqset_ulp)) { 325 hw_queue_teardown(hw); 326 return OCS_HW_RTN_ERROR; 327 } 328 329 for (i = 0; i < rqset_count ; i++) { 330 rqs[i]->filter_mask = rqset_filter_mask; 331 rqs[i]->is_mrq = TRUE; 332 rqs[i]->base_mrq_id = rqs[0]->hdr->id; 333 } 334 335 hw->hw_mrq_count = rqset_count; 336 } 337 } 338 339 return OCS_HW_RTN_SUCCESS; 340 fail: 341 hw_queue_teardown(hw); 342 return OCS_HW_RTN_NO_MEMORY; 343 344 } 345 346 /** 347 * @brief Allocate a new EQ object 348 * 349 * A new EQ object is instantiated 350 * 351 * @param hw pointer to HW object 352 * @param entry_count number of entries in the EQ 353 * 354 * @return pointer to allocated EQ object 355 */ 356 hw_eq_t* 357 hw_new_eq(ocs_hw_t *hw, uint32_t entry_count) 358 { 359 hw_eq_t *eq = ocs_malloc(hw->os, sizeof(*eq), OCS_M_ZERO | OCS_M_NOWAIT); 360 361 if (eq != NULL) { 362 eq->type = SLI_QTYPE_EQ; 363 eq->hw = hw; 364 eq->entry_count = entry_count; 365 eq->instance = hw->eq_count++; 366 eq->queue = &hw->eq[eq->instance]; 367 ocs_list_init(&eq->cq_list, hw_cq_t, link); 368 369 eq->wq_array = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 370 if (eq->wq_array == NULL) { 371 ocs_free(hw->os, eq, sizeof(*eq)); 372 eq = NULL; 373 } else { 374 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_EQ, eq->queue, entry_count, NULL, 0)) { 375 ocs_log_err(hw->os, "EQ[%d] allocation failure\n", eq->instance); 376 ocs_free(hw->os, eq, sizeof(*eq)); 377 eq = NULL; 378 } else { 379 sli_eq_modify_delay(&hw->sli, eq->queue, 1, 0, 8); 380 hw->hw_eq[eq->instance] = eq; 381 ocs_list_add_tail(&hw->eq_list, eq); 382 ocs_log_debug(hw->os, "create eq[%2d] id %3d len %4d\n", eq->instance, eq->queue->id, 383 eq->entry_count); 384 } 385 } 386 } 387 return eq; 388 } 389 390 /** 391 * @brief Allocate a new CQ object 392 * 393 * A new CQ object is instantiated 394 * 395 * @param eq pointer to parent EQ object 396 * @param entry_count number of entries in the CQ 397 * 398 * @return pointer to allocated CQ object 399 */ 400 hw_cq_t* 401 hw_new_cq(hw_eq_t *eq, uint32_t entry_count) 402 { 403 ocs_hw_t *hw = eq->hw; 404 hw_cq_t *cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT); 405 406 if (cq != NULL) { 407 cq->eq = eq; 408 cq->type = SLI_QTYPE_CQ; 409 cq->instance = eq->hw->cq_count++; 410 cq->entry_count = entry_count; 411 cq->queue = &hw->cq[cq->instance]; 412 413 ocs_list_init(&cq->q_list, hw_q_t, link); 414 415 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_CQ, cq->queue, cq->entry_count, eq->queue, 0)) { 416 ocs_log_err(hw->os, "CQ[%d] allocation failure len=%d\n", 417 eq->instance, 418 eq->entry_count); 419 ocs_free(hw->os, cq, sizeof(*cq)); 420 cq = NULL; 421 } else { 422 hw->hw_cq[cq->instance] = cq; 423 ocs_list_add_tail(&eq->cq_list, cq); 424 ocs_log_debug(hw->os, "create cq[%2d] id %3d len %4d\n", cq->instance, cq->queue->id, 425 cq->entry_count); 426 } 427 } 428 return cq; 429 } 430 431 /** 432 * @brief Allocate a new CQ Set of objects. 433 * 434 * @param eqs pointer to a set of EQ objects. 435 * @param cqs pointer to a set of CQ objects to be returned. 436 * @param num_cqs number of CQ queues in the set. 437 * @param entry_count number of entries in the CQ. 438 * 439 * @return 0 on success and -1 on failure. 440 */ 441 uint32_t 442 hw_new_cq_set(hw_eq_t *eqs[], hw_cq_t *cqs[], uint32_t num_cqs, uint32_t entry_count) 443 { 444 uint32_t i; 445 ocs_hw_t *hw = eqs[0]->hw; 446 sli4_t *sli4 = &hw->sli; 447 hw_cq_t *cq = NULL; 448 sli4_queue_t *qs[SLI_MAX_CQ_SET_COUNT], *assocs[SLI_MAX_CQ_SET_COUNT]; 449 450 /* Initialise CQS pointers to NULL */ 451 for (i = 0; i < num_cqs; i++) { 452 cqs[i] = NULL; 453 } 454 455 for (i = 0; i < num_cqs; i++) { 456 cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT); 457 if (cq == NULL) 458 goto error; 459 460 cqs[i] = cq; 461 cq->eq = eqs[i]; 462 cq->type = SLI_QTYPE_CQ; 463 cq->instance = hw->cq_count++; 464 cq->entry_count = entry_count; 465 cq->queue = &hw->cq[cq->instance]; 466 qs[i] = cq->queue; 467 assocs[i] = eqs[i]->queue; 468 ocs_list_init(&cq->q_list, hw_q_t, link); 469 } 470 471 if (sli_cq_alloc_set(sli4, qs, num_cqs, entry_count, assocs)) { 472 ocs_log_err(NULL, "Failed to create CQ Set. \n"); 473 goto error; 474 } 475 476 for (i = 0; i < num_cqs; i++) { 477 hw->hw_cq[cqs[i]->instance] = cqs[i]; 478 ocs_list_add_tail(&cqs[i]->eq->cq_list, cqs[i]); 479 } 480 481 return 0; 482 483 error: 484 for (i = 0; i < num_cqs; i++) { 485 if (cqs[i]) { 486 ocs_free(hw->os, cqs[i], sizeof(*cqs[i])); 487 cqs[i] = NULL; 488 } 489 } 490 return -1; 491 } 492 493 /** 494 * @brief Allocate a new MQ object 495 * 496 * A new MQ object is instantiated 497 * 498 * @param cq pointer to parent CQ object 499 * @param entry_count number of entries in the MQ 500 * 501 * @return pointer to allocated MQ object 502 */ 503 hw_mq_t* 504 hw_new_mq(hw_cq_t *cq, uint32_t entry_count) 505 { 506 ocs_hw_t *hw = cq->eq->hw; 507 hw_mq_t *mq = ocs_malloc(hw->os, sizeof(*mq), OCS_M_ZERO | OCS_M_NOWAIT); 508 509 if (mq != NULL) { 510 mq->cq = cq; 511 mq->type = SLI_QTYPE_MQ; 512 mq->instance = cq->eq->hw->mq_count++; 513 mq->entry_count = entry_count; 514 mq->entry_size = OCS_HW_MQ_DEPTH; 515 mq->queue = &hw->mq[mq->instance]; 516 517 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_MQ, 518 mq->queue, 519 mq->entry_size, 520 cq->queue, 0)) { 521 ocs_log_err(hw->os, "MQ allocation failure\n"); 522 ocs_free(hw->os, mq, sizeof(*mq)); 523 mq = NULL; 524 } else { 525 hw->hw_mq[mq->instance] = mq; 526 ocs_list_add_tail(&cq->q_list, mq); 527 ocs_log_debug(hw->os, "create mq[%2d] id %3d len %4d\n", mq->instance, mq->queue->id, 528 mq->entry_count); 529 } 530 } 531 return mq; 532 } 533 534 /** 535 * @brief Allocate a new WQ object 536 * 537 * A new WQ object is instantiated 538 * 539 * @param cq pointer to parent CQ object 540 * @param entry_count number of entries in the WQ 541 * @param class WQ class 542 * @param ulp index of chute 543 * 544 * @return pointer to allocated WQ object 545 */ 546 hw_wq_t* 547 hw_new_wq(hw_cq_t *cq, uint32_t entry_count, uint32_t class, uint32_t ulp) 548 { 549 ocs_hw_t *hw = cq->eq->hw; 550 hw_wq_t *wq = ocs_malloc(hw->os, sizeof(*wq), OCS_M_ZERO | OCS_M_NOWAIT); 551 552 if (wq != NULL) { 553 wq->hw = cq->eq->hw; 554 wq->cq = cq; 555 wq->type = SLI_QTYPE_WQ; 556 wq->instance = cq->eq->hw->wq_count++; 557 wq->entry_count = entry_count; 558 wq->queue = &hw->wq[wq->instance]; 559 wq->ulp = ulp; 560 wq->wqec_set_count = OCS_HW_WQEC_SET_COUNT; 561 wq->wqec_count = wq->wqec_set_count; 562 wq->free_count = wq->entry_count - 1; 563 wq->class = class; 564 ocs_list_init(&wq->pending_list, ocs_hw_wqe_t, link); 565 566 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_WQ, wq->queue, wq->entry_count, cq->queue, ulp)) { 567 ocs_log_err(hw->os, "WQ allocation failure\n"); 568 ocs_free(hw->os, wq, sizeof(*wq)); 569 wq = NULL; 570 } else { 571 hw->hw_wq[wq->instance] = wq; 572 ocs_list_add_tail(&cq->q_list, wq); 573 ocs_log_debug(hw->os, "create wq[%2d] id %3d len %4d cls %d ulp %d\n", wq->instance, wq->queue->id, 574 wq->entry_count, wq->class, wq->ulp); 575 } 576 } 577 return wq; 578 } 579 580 /** 581 * @brief Allocate a hw_rq_t object 582 * 583 * Allocate an RQ object, which encapsulates 2 SLI queues (for rq pair) 584 * 585 * @param cq pointer to parent CQ object 586 * @param entry_count number of entries in the RQs 587 * @param ulp ULP index for this RQ 588 * 589 * @return pointer to newly allocated hw_rq_t 590 */ 591 hw_rq_t* 592 hw_new_rq(hw_cq_t *cq, uint32_t entry_count, uint32_t ulp) 593 { 594 ocs_hw_t *hw = cq->eq->hw; 595 hw_rq_t *rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT); 596 uint32_t max_hw_rq; 597 598 ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq); 599 600 if (rq != NULL) { 601 rq->instance = hw->hw_rq_count++; 602 rq->cq = cq; 603 rq->type = SLI_QTYPE_RQ; 604 rq->ulp = ulp; 605 606 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR)); 607 608 /* Create the header RQ */ 609 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq)); 610 rq->hdr = &hw->rq[hw->rq_count]; 611 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE; 612 613 if (sli_fc_rq_alloc(&hw->sli, rq->hdr, 614 rq->entry_count, 615 rq->hdr_entry_size, 616 cq->queue, 617 ulp, TRUE)) { 618 ocs_log_err(hw->os, "RQ allocation failure - header\n"); 619 ocs_free(hw->os, rq, sizeof(*rq)); 620 return NULL; 621 } 622 hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */ 623 hw->rq_count++; 624 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d hdr size %4d ulp %d\n", 625 rq->instance, rq->hdr->id, rq->entry_count, rq->hdr_entry_size, rq->ulp); 626 627 /* Create the default data RQ */ 628 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq)); 629 rq->data = &hw->rq[hw->rq_count]; 630 rq->data_entry_size = hw->config.rq_default_buffer_size; 631 632 if (sli_fc_rq_alloc(&hw->sli, rq->data, 633 rq->entry_count, 634 rq->data_entry_size, 635 cq->queue, 636 ulp, FALSE)) { 637 ocs_log_err(hw->os, "RQ allocation failure - first burst\n"); 638 ocs_free(hw->os, rq, sizeof(*rq)); 639 return NULL; 640 } 641 hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */ 642 hw->rq_count++; 643 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d data size %4d ulp %d\n", rq->instance, 644 rq->data->id, rq->entry_count, rq->data_entry_size, rq->ulp); 645 646 hw->hw_rq[rq->instance] = rq; 647 ocs_list_add_tail(&cq->q_list, rq); 648 649 rq->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) * 650 rq->entry_count, OCS_M_ZERO | OCS_M_NOWAIT); 651 if (rq->rq_tracker == NULL) { 652 ocs_log_err(hw->os, "RQ tracker buf allocation failure\n"); 653 return NULL; 654 } 655 } 656 return rq; 657 } 658 659 /** 660 * @brief Allocate a hw_rq_t object SET 661 * 662 * Allocate an RQ object SET, where each element in set 663 * encapsulates 2 SLI queues (for rq pair) 664 * 665 * @param cqs pointers to be associated with RQs. 666 * @param rqs RQ pointers to be returned on success. 667 * @param num_rq_pairs number of rq pairs in the Set. 668 * @param entry_count number of entries in the RQs 669 * @param ulp ULP index for this RQ 670 * 671 * @return 0 in success and -1 on failure. 672 */ 673 uint32_t 674 hw_new_rq_set(hw_cq_t *cqs[], hw_rq_t *rqs[], uint32_t num_rq_pairs, uint32_t entry_count, uint32_t ulp) 675 { 676 ocs_hw_t *hw = cqs[0]->eq->hw; 677 hw_rq_t *rq = NULL; 678 sli4_queue_t *qs[SLI_MAX_RQ_SET_COUNT * 2] = { NULL }; 679 uint32_t max_hw_rq, i, q_count; 680 681 ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq); 682 683 /* Initialise RQS pointers */ 684 for (i = 0; i < num_rq_pairs; i++) { 685 rqs[i] = NULL; 686 } 687 688 for (i = 0, q_count = 0; i < num_rq_pairs; i++, q_count += 2) { 689 rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT); 690 if (rq == NULL) 691 goto error; 692 693 rqs[i] = rq; 694 rq->instance = hw->hw_rq_count++; 695 rq->cq = cqs[i]; 696 rq->type = SLI_QTYPE_RQ; 697 rq->ulp = ulp; 698 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR)); 699 700 /* Header RQ */ 701 rq->hdr = &hw->rq[hw->rq_count]; 702 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE; 703 hw->hw_rq_lookup[hw->rq_count] = rq->instance; 704 hw->rq_count++; 705 qs[q_count] = rq->hdr; 706 707 /* Data RQ */ 708 rq->data = &hw->rq[hw->rq_count]; 709 rq->data_entry_size = hw->config.rq_default_buffer_size; 710 hw->hw_rq_lookup[hw->rq_count] = rq->instance; 711 hw->rq_count++; 712 qs[q_count + 1] = rq->data; 713 714 rq->rq_tracker = NULL; 715 } 716 717 if (sli_fc_rq_set_alloc(&hw->sli, num_rq_pairs, qs, 718 cqs[0]->queue->id, 719 rqs[0]->entry_count, 720 rqs[0]->hdr_entry_size, 721 rqs[0]->data_entry_size, 722 ulp)) { 723 ocs_log_err(hw->os, "RQ Set allocation failure for base CQ=%d\n", cqs[0]->queue->id); 724 goto error; 725 } 726 727 for (i = 0; i < num_rq_pairs; i++) { 728 hw->hw_rq[rqs[i]->instance] = rqs[i]; 729 ocs_list_add_tail(&cqs[i]->q_list, rqs[i]); 730 rqs[i]->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) * 731 rqs[i]->entry_count, OCS_M_ZERO | OCS_M_NOWAIT); 732 if (rqs[i]->rq_tracker == NULL) { 733 ocs_log_err(hw->os, "RQ tracker buf allocation failure\n"); 734 goto error; 735 } 736 } 737 738 return 0; 739 740 error: 741 for (i = 0; i < num_rq_pairs; i++) { 742 if (rqs[i] != NULL) { 743 if (rqs[i]->rq_tracker != NULL) { 744 ocs_free(hw->os, rqs[i]->rq_tracker, 745 sizeof(ocs_hw_sequence_t*) * 746 rqs[i]->entry_count); 747 } 748 ocs_free(hw->os, rqs[i], sizeof(*rqs[i])); 749 } 750 } 751 752 return -1; 753 } 754 755 /** 756 * @brief Free an EQ object 757 * 758 * The EQ object and any child queue objects are freed 759 * 760 * @param eq pointer to EQ object 761 * 762 * @return none 763 */ 764 void 765 hw_del_eq(hw_eq_t *eq) 766 { 767 if (eq != NULL) { 768 hw_cq_t *cq; 769 hw_cq_t *cq_next; 770 771 ocs_list_foreach_safe(&eq->cq_list, cq, cq_next) { 772 hw_del_cq(cq); 773 } 774 ocs_varray_free(eq->wq_array); 775 ocs_list_remove(&eq->hw->eq_list, eq); 776 eq->hw->hw_eq[eq->instance] = NULL; 777 ocs_free(eq->hw->os, eq, sizeof(*eq)); 778 } 779 } 780 781 /** 782 * @brief Free a CQ object 783 * 784 * The CQ object and any child queue objects are freed 785 * 786 * @param cq pointer to CQ object 787 * 788 * @return none 789 */ 790 void 791 hw_del_cq(hw_cq_t *cq) 792 { 793 if (cq != NULL) { 794 hw_q_t *q; 795 hw_q_t *q_next; 796 797 ocs_list_foreach_safe(&cq->q_list, q, q_next) { 798 switch(q->type) { 799 case SLI_QTYPE_MQ: 800 hw_del_mq((hw_mq_t*) q); 801 break; 802 case SLI_QTYPE_WQ: 803 hw_del_wq((hw_wq_t*) q); 804 break; 805 case SLI_QTYPE_RQ: 806 hw_del_rq((hw_rq_t*) q); 807 break; 808 default: 809 break; 810 } 811 } 812 ocs_list_remove(&cq->eq->cq_list, cq); 813 cq->eq->hw->hw_cq[cq->instance] = NULL; 814 ocs_free(cq->eq->hw->os, cq, sizeof(*cq)); 815 } 816 } 817 818 /** 819 * @brief Free a MQ object 820 * 821 * The MQ object is freed 822 * 823 * @param mq pointer to MQ object 824 * 825 * @return none 826 */ 827 void 828 hw_del_mq(hw_mq_t *mq) 829 { 830 if (mq != NULL) { 831 ocs_list_remove(&mq->cq->q_list, mq); 832 mq->cq->eq->hw->hw_mq[mq->instance] = NULL; 833 ocs_free(mq->cq->eq->hw->os, mq, sizeof(*mq)); 834 } 835 } 836 837 /** 838 * @brief Free a WQ object 839 * 840 * The WQ object is freed 841 * 842 * @param wq pointer to WQ object 843 * 844 * @return none 845 */ 846 void 847 hw_del_wq(hw_wq_t *wq) 848 { 849 if (wq != NULL) { 850 ocs_list_remove(&wq->cq->q_list, wq); 851 wq->cq->eq->hw->hw_wq[wq->instance] = NULL; 852 ocs_free(wq->cq->eq->hw->os, wq, sizeof(*wq)); 853 } 854 } 855 856 /** 857 * @brief Free an RQ object 858 * 859 * The RQ object is freed 860 * 861 * @param rq pointer to RQ object 862 * 863 * @return none 864 */ 865 void 866 hw_del_rq(hw_rq_t *rq) 867 { 868 869 if (rq != NULL) { 870 ocs_hw_t *hw = rq->cq->eq->hw; 871 /* Free RQ tracker */ 872 if (rq->rq_tracker != NULL) { 873 ocs_free(hw->os, rq->rq_tracker, sizeof(ocs_hw_sequence_t*) * rq->entry_count); 874 rq->rq_tracker = NULL; 875 } 876 ocs_list_remove(&rq->cq->q_list, rq); 877 hw->hw_rq[rq->instance] = NULL; 878 ocs_free(hw->os, rq, sizeof(*rq)); 879 } 880 } 881 882 /** 883 * @brief Display HW queue objects 884 * 885 * The HW queue objects are displayed using ocs_log 886 * 887 * @param hw pointer to HW object 888 * 889 * @return none 890 */ 891 void 892 hw_queue_dump(ocs_hw_t *hw) 893 { 894 hw_eq_t *eq; 895 hw_cq_t *cq; 896 hw_q_t *q; 897 hw_mq_t *mq; 898 hw_wq_t *wq; 899 hw_rq_t *rq; 900 901 ocs_list_foreach(&hw->eq_list, eq) { 902 ocs_printf("eq[%d] id %2d\n", eq->instance, eq->queue->id); 903 ocs_list_foreach(&eq->cq_list, cq) { 904 ocs_printf(" cq[%d] id %2d current\n", cq->instance, cq->queue->id); 905 ocs_list_foreach(&cq->q_list, q) { 906 switch(q->type) { 907 case SLI_QTYPE_MQ: 908 mq = (hw_mq_t *) q; 909 ocs_printf(" mq[%d] id %2d\n", mq->instance, mq->queue->id); 910 break; 911 case SLI_QTYPE_WQ: 912 wq = (hw_wq_t *) q; 913 ocs_printf(" wq[%d] id %2d\n", wq->instance, wq->queue->id); 914 break; 915 case SLI_QTYPE_RQ: 916 rq = (hw_rq_t *) q; 917 ocs_printf(" rq[%d] hdr id %2d\n", rq->instance, rq->hdr->id); 918 break; 919 default: 920 break; 921 } 922 } 923 } 924 } 925 } 926 927 /** 928 * @brief Teardown HW queue objects 929 * 930 * The HW queue objects are freed 931 * 932 * @param hw pointer to HW object 933 * 934 * @return none 935 */ 936 void 937 hw_queue_teardown(ocs_hw_t *hw) 938 { 939 uint32_t i; 940 hw_eq_t *eq; 941 hw_eq_t *eq_next; 942 943 if (ocs_list_valid(&hw->eq_list)) { 944 ocs_list_foreach_safe(&hw->eq_list, eq, eq_next) { 945 hw_del_eq(eq); 946 } 947 } 948 for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) { 949 ocs_varray_free(hw->wq_cpu_array[i]); 950 hw->wq_cpu_array[i] = NULL; 951 } 952 for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) { 953 ocs_varray_free(hw->wq_class_array[i]); 954 hw->wq_class_array[i] = NULL; 955 } 956 } 957 958 /** 959 * @brief Allocate a WQ to an IO object 960 * 961 * The next work queue index is used to assign a WQ to an IO. 962 * 963 * If wq_steering is OCS_HW_WQ_STEERING_CLASS, a WQ from io->wq_class is 964 * selected. 965 * 966 * If wq_steering is OCS_HW_WQ_STEERING_REQUEST, then a WQ from the EQ that 967 * the IO request came in on is selected. 968 * 969 * If wq_steering is OCS_HW_WQ_STEERING_CPU, then a WQ associated with the 970 * CPU the request is made on is selected. 971 * 972 * @param hw pointer to HW object 973 * @param io pointer to IO object 974 * 975 * @return Return pointer to next WQ 976 */ 977 hw_wq_t * 978 ocs_hw_queue_next_wq(ocs_hw_t *hw, ocs_hw_io_t *io) 979 { 980 hw_eq_t *eq; 981 hw_wq_t *wq = NULL; 982 983 switch(io->wq_steering) { 984 case OCS_HW_WQ_STEERING_CLASS: 985 if (likely(io->wq_class < ARRAY_SIZE(hw->wq_class_array))) { 986 wq = ocs_varray_iter_next(hw->wq_class_array[io->wq_class]); 987 } 988 break; 989 case OCS_HW_WQ_STEERING_REQUEST: 990 eq = io->eq; 991 if (likely(eq != NULL)) { 992 wq = ocs_varray_iter_next(eq->wq_array); 993 } 994 break; 995 case OCS_HW_WQ_STEERING_CPU: { 996 uint32_t cpuidx = ocs_thread_getcpu(); 997 998 if (likely(cpuidx < ARRAY_SIZE(hw->wq_cpu_array))) { 999 wq = ocs_varray_iter_next(hw->wq_cpu_array[cpuidx]); 1000 } 1001 break; 1002 } 1003 } 1004 1005 if (unlikely(wq == NULL)) { 1006 wq = hw->hw_wq[0]; 1007 } 1008 1009 return wq; 1010 } 1011 1012 /** 1013 * @brief Return count of EQs for a queue topology object 1014 * 1015 * The EQ count for in the HWs queue topology (hw->qtop) object is returned 1016 * 1017 * @param hw pointer to HW object 1018 * 1019 * @return count of EQs 1020 */ 1021 uint32_t 1022 ocs_hw_qtop_eq_count(ocs_hw_t *hw) 1023 { 1024 return hw->qtop->entry_counts[QTOP_EQ]; 1025 } 1026 1027 #define TOKEN_LEN 32 1028 1029 /** 1030 * @brief return string given a QTOP entry 1031 * 1032 * @param entry QTOP entry 1033 * 1034 * @return returns string or "unknown" 1035 */ 1036 #if HW_QTOP_DEBUG 1037 static char * 1038 qtopentry2s(ocs_hw_qtop_entry_e entry) { 1039 switch(entry) { 1040 #define P(x) case x: return #x; 1041 P(QTOP_EQ) 1042 P(QTOP_CQ) 1043 P(QTOP_WQ) 1044 P(QTOP_RQ) 1045 P(QTOP_MQ) 1046 P(QTOP_THREAD_START) 1047 P(QTOP_THREAD_END) 1048 P(QTOP_LAST) 1049 #undef P 1050 } 1051 return "unknown"; 1052 } 1053 #endif 1054 1055 /** 1056 * @brief Declare token types 1057 */ 1058 typedef enum { 1059 TOK_LPAREN = 1, 1060 TOK_RPAREN, 1061 TOK_COLON, 1062 TOK_EQUALS, 1063 TOK_QUEUE, 1064 TOK_ATTR_NAME, 1065 TOK_NUMBER, 1066 TOK_NUMBER_VALUE, 1067 TOK_NUMBER_LIST, 1068 } tok_type_e; 1069 1070 /** 1071 * @brief Declare token sub-types 1072 */ 1073 typedef enum { 1074 TOK_SUB_EQ = 100, 1075 TOK_SUB_CQ, 1076 TOK_SUB_RQ, 1077 TOK_SUB_MQ, 1078 TOK_SUB_WQ, 1079 TOK_SUB_LEN, 1080 TOK_SUB_CLASS, 1081 TOK_SUB_ULP, 1082 TOK_SUB_FILTER, 1083 } tok_subtype_e; 1084 1085 /** 1086 * @brief convert queue subtype to QTOP entry 1087 * 1088 * @param q queue subtype 1089 * 1090 * @return QTOP entry or 0 1091 */ 1092 static ocs_hw_qtop_entry_e 1093 subtype2qtop(tok_subtype_e q) 1094 { 1095 switch(q) { 1096 case TOK_SUB_EQ: return QTOP_EQ; 1097 case TOK_SUB_CQ: return QTOP_CQ; 1098 case TOK_SUB_RQ: return QTOP_RQ; 1099 case TOK_SUB_MQ: return QTOP_MQ; 1100 case TOK_SUB_WQ: return QTOP_WQ; 1101 default: 1102 break; 1103 } 1104 return 0; 1105 } 1106 1107 /** 1108 * @brief Declare token object 1109 */ 1110 typedef struct { 1111 tok_type_e type; 1112 tok_subtype_e subtype; 1113 char string[TOKEN_LEN]; 1114 } tok_t; 1115 1116 /** 1117 * @brief Declare token array object 1118 */ 1119 typedef struct { 1120 tok_t *tokens; /* Pointer to array of tokens */ 1121 uint32_t alloc_count; /* Number of tokens in the array */ 1122 uint32_t inuse_count; /* Number of tokens posted to array */ 1123 uint32_t iter_idx; /* Iterator index */ 1124 } tokarray_t; 1125 1126 /** 1127 * @brief Declare token match structure 1128 */ 1129 typedef struct { 1130 char *s; 1131 tok_type_e type; 1132 tok_subtype_e subtype; 1133 } tokmatch_t; 1134 1135 /** 1136 * @brief test if character is ID start character 1137 * 1138 * @param c character to test 1139 * 1140 * @return TRUE if character is an ID start character 1141 */ 1142 static int32_t 1143 idstart(int c) 1144 { 1145 return isalpha(c) || (c == '_') || (c == '$'); 1146 } 1147 1148 /** 1149 * @brief test if character is an ID character 1150 * 1151 * @param c character to test 1152 * 1153 * @return TRUE if character is an ID character 1154 */ 1155 static int32_t 1156 idchar(int c) 1157 { 1158 return idstart(c) || ocs_isdigit(c); 1159 } 1160 1161 /** 1162 * @brief Declare single character matches 1163 */ 1164 static tokmatch_t cmatches[] = { 1165 {"(", TOK_LPAREN}, 1166 {")", TOK_RPAREN}, 1167 {":", TOK_COLON}, 1168 {"=", TOK_EQUALS}, 1169 }; 1170 1171 /** 1172 * @brief Declare identifier match strings 1173 */ 1174 static tokmatch_t smatches[] = { 1175 {"eq", TOK_QUEUE, TOK_SUB_EQ}, 1176 {"cq", TOK_QUEUE, TOK_SUB_CQ}, 1177 {"rq", TOK_QUEUE, TOK_SUB_RQ}, 1178 {"mq", TOK_QUEUE, TOK_SUB_MQ}, 1179 {"wq", TOK_QUEUE, TOK_SUB_WQ}, 1180 {"len", TOK_ATTR_NAME, TOK_SUB_LEN}, 1181 {"class", TOK_ATTR_NAME, TOK_SUB_CLASS}, 1182 {"ulp", TOK_ATTR_NAME, TOK_SUB_ULP}, 1183 {"filter", TOK_ATTR_NAME, TOK_SUB_FILTER}, 1184 }; 1185 1186 /** 1187 * @brief Scan string and return next token 1188 * 1189 * The string is scanned and the next token is returned 1190 * 1191 * @param s input string to scan 1192 * @param tok pointer to place scanned token 1193 * 1194 * @return pointer to input string following scanned token, or NULL 1195 */ 1196 static const char * 1197 tokenize(const char *s, tok_t *tok) 1198 { 1199 uint32_t i; 1200 1201 memset(tok, 0, sizeof(*tok)); 1202 1203 /* Skip over whitespace */ 1204 while (*s && ocs_isspace(*s)) { 1205 s++; 1206 } 1207 1208 /* Return if nothing left in this string */ 1209 if (*s == 0) { 1210 return NULL; 1211 } 1212 1213 /* Look for single character matches */ 1214 for (i = 0; i < ARRAY_SIZE(cmatches); i++) { 1215 if (cmatches[i].s[0] == *s) { 1216 tok->type = cmatches[i].type; 1217 tok->subtype = cmatches[i].subtype; 1218 tok->string[0] = *s++; 1219 return s; 1220 } 1221 } 1222 1223 /* Scan for a hex number or decimal */ 1224 if ((s[0] == '0') && ((s[1] == 'x') || (s[1] == 'X'))) { 1225 char *p = tok->string; 1226 1227 tok->type = TOK_NUMBER; 1228 1229 *p++ = *s++; 1230 *p++ = *s++; 1231 while ((*s == '.') || ocs_isxdigit(*s)) { 1232 if ((p - tok->string) < (int32_t)sizeof(tok->string)) { 1233 *p++ = *s; 1234 } 1235 if (*s == ',') { 1236 tok->type = TOK_NUMBER_LIST; 1237 } 1238 s++; 1239 } 1240 *p = 0; 1241 return s; 1242 } else if (ocs_isdigit(*s)) { 1243 char *p = tok->string; 1244 1245 tok->type = TOK_NUMBER; 1246 while ((*s == ',') || ocs_isdigit(*s)) { 1247 if ((p - tok->string) < (int32_t)sizeof(tok->string)) { 1248 *p++ = *s; 1249 } 1250 if (*s == ',') { 1251 tok->type = TOK_NUMBER_LIST; 1252 } 1253 s++; 1254 } 1255 *p = 0; 1256 return s; 1257 } 1258 1259 /* Scan for an ID */ 1260 if (idstart(*s)) { 1261 char *p = tok->string; 1262 1263 for (*p++ = *s++; idchar(*s); s++) { 1264 if ((p - tok->string) < TOKEN_LEN) { 1265 *p++ = *s; 1266 } 1267 } 1268 1269 /* See if this is a $ number value */ 1270 if (tok->string[0] == '$') { 1271 tok->type = TOK_NUMBER_VALUE; 1272 } else { 1273 /* Look for a string match */ 1274 for (i = 0; i < ARRAY_SIZE(smatches); i++) { 1275 if (strcmp(smatches[i].s, tok->string) == 0) { 1276 tok->type = smatches[i].type; 1277 tok->subtype = smatches[i].subtype; 1278 return s; 1279 } 1280 } 1281 } 1282 } 1283 return s; 1284 } 1285 1286 /** 1287 * @brief convert token type to string 1288 * 1289 * @param type token type 1290 * 1291 * @return string, or "unknown" 1292 */ 1293 static const char * 1294 token_type2s(tok_type_e type) 1295 { 1296 switch(type) { 1297 #define P(x) case x: return #x; 1298 P(TOK_LPAREN) 1299 P(TOK_RPAREN) 1300 P(TOK_COLON) 1301 P(TOK_EQUALS) 1302 P(TOK_QUEUE) 1303 P(TOK_ATTR_NAME) 1304 P(TOK_NUMBER) 1305 P(TOK_NUMBER_VALUE) 1306 P(TOK_NUMBER_LIST) 1307 #undef P 1308 } 1309 return "unknown"; 1310 } 1311 1312 /** 1313 * @brief convert token sub-type to string 1314 * 1315 * @param subtype token sub-type 1316 * 1317 * @return string, or "unknown" 1318 */ 1319 static const char * 1320 token_subtype2s(tok_subtype_e subtype) 1321 { 1322 switch(subtype) { 1323 #define P(x) case x: return #x; 1324 P(TOK_SUB_EQ) 1325 P(TOK_SUB_CQ) 1326 P(TOK_SUB_RQ) 1327 P(TOK_SUB_MQ) 1328 P(TOK_SUB_WQ) 1329 P(TOK_SUB_LEN) 1330 P(TOK_SUB_CLASS) 1331 P(TOK_SUB_ULP) 1332 P(TOK_SUB_FILTER) 1333 #undef P 1334 } 1335 return ""; 1336 } 1337 1338 /** 1339 * @brief Generate syntax error message 1340 * 1341 * A syntax error message is found, the input tokens are dumped up to and including 1342 * the token that failed as indicated by the current iterator index. 1343 * 1344 * @param hw pointer to HW object 1345 * @param tokarray pointer to token array object 1346 * 1347 * @return none 1348 */ 1349 static void 1350 tok_syntax(ocs_hw_t *hw, tokarray_t *tokarray) 1351 { 1352 uint32_t i; 1353 tok_t *tok; 1354 1355 ocs_log_test(hw->os, "Syntax error:\n"); 1356 1357 for (i = 0, tok = tokarray->tokens; (i <= tokarray->inuse_count); i++, tok++) { 1358 ocs_log_test(hw->os, "%s [%2d] %-16s %-16s %s\n", (i == tokarray->iter_idx) ? ">>>" : " ", i, 1359 token_type2s(tok->type), token_subtype2s(tok->subtype), tok->string); 1360 } 1361 } 1362 1363 /** 1364 * @brief parse a number 1365 * 1366 * Parses tokens of type TOK_NUMBER and TOK_NUMBER_VALUE, returning a numeric value 1367 * 1368 * @param hw pointer to HW object 1369 * @param qtop pointer to QTOP object 1370 * @param tok pointer to token to parse 1371 * 1372 * @return numeric value 1373 */ 1374 static uint32_t 1375 tok_getnumber(ocs_hw_t *hw, ocs_hw_qtop_t *qtop, tok_t *tok) 1376 { 1377 uint32_t rval = 0; 1378 uint32_t num_cpus = ocs_get_num_cpus(); 1379 1380 switch(tok->type) { 1381 case TOK_NUMBER_VALUE: 1382 if (ocs_strcmp(tok->string, "$ncpu") == 0) { 1383 rval = num_cpus; 1384 } else if (ocs_strcmp(tok->string, "$ncpu1") == 0) { 1385 rval = num_cpus - 1; 1386 } else if (ocs_strcmp(tok->string, "$nwq") == 0) { 1387 if (hw != NULL) { 1388 rval = hw->config.n_wq; 1389 } 1390 } else if (ocs_strcmp(tok->string, "$maxmrq") == 0) { 1391 rval = MIN(num_cpus, OCS_HW_MAX_MRQS); 1392 } else if (ocs_strcmp(tok->string, "$nulp") == 0) { 1393 rval = hw->ulp_max - hw->ulp_start + 1; 1394 } else if ((qtop->rptcount_idx > 0) && ocs_strcmp(tok->string, "$rpt0") == 0) { 1395 rval = qtop->rptcount[qtop->rptcount_idx-1]; 1396 } else if ((qtop->rptcount_idx > 1) && ocs_strcmp(tok->string, "$rpt1") == 0) { 1397 rval = qtop->rptcount[qtop->rptcount_idx-2]; 1398 } else if ((qtop->rptcount_idx > 2) && ocs_strcmp(tok->string, "$rpt2") == 0) { 1399 rval = qtop->rptcount[qtop->rptcount_idx-3]; 1400 } else if ((qtop->rptcount_idx > 3) && ocs_strcmp(tok->string, "$rpt3") == 0) { 1401 rval = qtop->rptcount[qtop->rptcount_idx-4]; 1402 } else { 1403 rval = ocs_strtoul(tok->string, 0, 0); 1404 } 1405 break; 1406 case TOK_NUMBER: 1407 rval = ocs_strtoul(tok->string, 0, 0); 1408 break; 1409 default: 1410 break; 1411 } 1412 return rval; 1413 } 1414 1415 /** 1416 * @brief parse an array of tokens 1417 * 1418 * The tokens are semantically parsed, to generate QTOP entries. 1419 * 1420 * @param hw pointer to HW object 1421 * @param tokarray array array of tokens 1422 * @param qtop ouptut QTOP object 1423 * 1424 * @return returns 0 for success, a negative error code value for failure. 1425 */ 1426 static int32_t 1427 parse_topology(ocs_hw_t *hw, tokarray_t *tokarray, ocs_hw_qtop_t *qtop) 1428 { 1429 ocs_hw_qtop_entry_t *qt = qtop->entries + qtop->inuse_count; 1430 tok_t *tok; 1431 1432 for (; (tokarray->iter_idx < tokarray->inuse_count) && 1433 ((tok = &tokarray->tokens[tokarray->iter_idx]) != NULL); ) { 1434 if (qtop->inuse_count >= qtop->alloc_count) { 1435 return -1; 1436 } 1437 1438 qt = qtop->entries + qtop->inuse_count; 1439 1440 switch (tok[0].type) 1441 { 1442 case TOK_QUEUE: 1443 qt->entry = subtype2qtop(tok[0].subtype); 1444 qt->set_default = FALSE; 1445 qt->len = 0; 1446 qt->class = 0; 1447 qtop->inuse_count++; 1448 1449 tokarray->iter_idx++; /* Advance current token index */ 1450 1451 /* Parse for queue attributes, possibly multiple instances */ 1452 while ((tokarray->iter_idx + 4) <= tokarray->inuse_count) { 1453 tok = &tokarray->tokens[tokarray->iter_idx]; 1454 if( (tok[0].type == TOK_COLON) && 1455 (tok[1].type == TOK_ATTR_NAME) && 1456 (tok[2].type == TOK_EQUALS) && 1457 ((tok[3].type == TOK_NUMBER) || 1458 (tok[3].type == TOK_NUMBER_VALUE) || 1459 (tok[3].type == TOK_NUMBER_LIST))) { 1460 switch (tok[1].subtype) { 1461 case TOK_SUB_LEN: 1462 qt->len = tok_getnumber(hw, qtop, &tok[3]); 1463 break; 1464 1465 case TOK_SUB_CLASS: 1466 qt->class = tok_getnumber(hw, qtop, &tok[3]); 1467 break; 1468 1469 case TOK_SUB_ULP: 1470 qt->ulp = tok_getnumber(hw, qtop, &tok[3]); 1471 break; 1472 1473 case TOK_SUB_FILTER: 1474 if (tok[3].type == TOK_NUMBER_LIST) { 1475 uint32_t mask = 0; 1476 char *p = tok[3].string; 1477 1478 while ((p != NULL) && *p) { 1479 uint32_t v; 1480 1481 v = ocs_strtoul(p, 0, 0); 1482 if (v < 32) { 1483 mask |= (1U << v); 1484 } 1485 1486 p = ocs_strchr(p, ','); 1487 if (p != NULL) { 1488 p++; 1489 } 1490 } 1491 qt->filter_mask = mask; 1492 } else { 1493 qt->filter_mask = (1U << tok_getnumber(hw, qtop, &tok[3])); 1494 } 1495 break; 1496 default: 1497 break; 1498 } 1499 /* Advance current token index */ 1500 tokarray->iter_idx += 4; 1501 } else { 1502 break; 1503 } 1504 } 1505 qtop->entry_counts[qt->entry]++; 1506 break; 1507 1508 case TOK_ATTR_NAME: 1509 if ( ((tokarray->iter_idx + 5) <= tokarray->inuse_count) && 1510 (tok[1].type == TOK_COLON) && 1511 (tok[2].type == TOK_QUEUE) && 1512 (tok[3].type == TOK_EQUALS) && 1513 ((tok[4].type == TOK_NUMBER) || (tok[4].type == TOK_NUMBER_VALUE))) { 1514 qt->entry = subtype2qtop(tok[2].subtype); 1515 qt->set_default = TRUE; 1516 switch(tok[0].subtype) { 1517 case TOK_SUB_LEN: 1518 qt->len = tok_getnumber(hw, qtop, &tok[4]); 1519 break; 1520 case TOK_SUB_CLASS: 1521 qt->class = tok_getnumber(hw, qtop, &tok[4]); 1522 break; 1523 case TOK_SUB_ULP: 1524 qt->ulp = tok_getnumber(hw, qtop, &tok[4]); 1525 break; 1526 default: 1527 break; 1528 } 1529 qtop->inuse_count++; 1530 tokarray->iter_idx += 5; 1531 } else { 1532 tok_syntax(hw, tokarray); 1533 return -1; 1534 } 1535 break; 1536 1537 case TOK_NUMBER: 1538 case TOK_NUMBER_VALUE: { 1539 uint32_t rpt_count = 1; 1540 uint32_t i; 1541 1542 rpt_count = tok_getnumber(hw, qtop, tok); 1543 1544 if (tok[1].type == TOK_LPAREN) { 1545 uint32_t iter_idx_save; 1546 1547 tokarray->iter_idx += 2; 1548 1549 /* save token array iteration index */ 1550 iter_idx_save = tokarray->iter_idx; 1551 1552 for (i = 0; i < rpt_count; i++) { 1553 uint32_t rptcount_idx = qtop->rptcount_idx; 1554 1555 if (qtop->rptcount_idx < ARRAY_SIZE(qtop->rptcount)) { 1556 qtop->rptcount[qtop->rptcount_idx++] = i; 1557 } 1558 1559 /* restore token array iteration index */ 1560 tokarray->iter_idx = iter_idx_save; 1561 1562 /* parse, append to qtop */ 1563 parse_topology(hw, tokarray, qtop); 1564 1565 qtop->rptcount_idx = rptcount_idx; 1566 } 1567 } 1568 break; 1569 } 1570 1571 case TOK_RPAREN: 1572 tokarray->iter_idx++; 1573 return 0; 1574 1575 default: 1576 tok_syntax(hw, tokarray); 1577 return -1; 1578 } 1579 } 1580 return 0; 1581 } 1582 1583 /** 1584 * @brief Parse queue topology string 1585 * 1586 * The queue topology object is allocated, and filled with the results of parsing the 1587 * passed in queue topology string 1588 * 1589 * @param hw pointer to HW object 1590 * @param qtop_string input queue topology string 1591 * 1592 * @return pointer to allocated QTOP object, or NULL if there was an error 1593 */ 1594 ocs_hw_qtop_t * 1595 ocs_hw_qtop_parse(ocs_hw_t *hw, const char *qtop_string) 1596 { 1597 ocs_hw_qtop_t *qtop; 1598 tokarray_t tokarray; 1599 const char *s; 1600 #if HW_QTOP_DEBUG 1601 uint32_t i; 1602 ocs_hw_qtop_entry_t *qt; 1603 #endif 1604 1605 ocs_log_debug(hw->os, "queue topology: %s\n", qtop_string); 1606 1607 /* Allocate a token array */ 1608 tokarray.tokens = ocs_malloc(hw->os, MAX_TOKENS * sizeof(*tokarray.tokens), OCS_M_ZERO | OCS_M_NOWAIT); 1609 if (tokarray.tokens == NULL) { 1610 return NULL; 1611 } 1612 tokarray.alloc_count = MAX_TOKENS; 1613 tokarray.inuse_count = 0; 1614 tokarray.iter_idx = 0; 1615 1616 /* Parse the tokens */ 1617 for (s = qtop_string; (tokarray.inuse_count < tokarray.alloc_count) && 1618 ((s = tokenize(s, &tokarray.tokens[tokarray.inuse_count]))) != NULL; ) { 1619 tokarray.inuse_count++; 1620 } 1621 1622 /* Allocate a queue topology structure */ 1623 qtop = ocs_malloc(hw->os, sizeof(*qtop), OCS_M_ZERO | OCS_M_NOWAIT); 1624 if (qtop == NULL) { 1625 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1626 ocs_log_err(hw->os, "malloc qtop failed\n"); 1627 return NULL; 1628 } 1629 qtop->os = hw->os; 1630 1631 /* Allocate queue topology entries */ 1632 qtop->entries = ocs_malloc(hw->os, OCS_HW_MAX_QTOP_ENTRIES*sizeof(*qtop->entries), OCS_M_ZERO | OCS_M_NOWAIT); 1633 if (qtop->entries == NULL) { 1634 ocs_log_err(hw->os, "malloc qtop entries failed\n"); 1635 ocs_free(hw->os, qtop, sizeof(*qtop)); 1636 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1637 return NULL; 1638 } 1639 qtop->alloc_count = OCS_HW_MAX_QTOP_ENTRIES; 1640 qtop->inuse_count = 0; 1641 1642 /* Parse the tokens */ 1643 parse_topology(hw, &tokarray, qtop); 1644 #if HW_QTOP_DEBUG 1645 for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) { 1646 ocs_log_debug(hw->os, "entry %s set_df %d len %4d class %d ulp %d\n", qtopentry2s(qt->entry), qt->set_default, qt->len, 1647 qt->class, qt->ulp); 1648 } 1649 #endif 1650 1651 /* Free the tokens array */ 1652 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1653 1654 return qtop; 1655 } 1656 1657 /** 1658 * @brief free queue topology object 1659 * 1660 * @param qtop pointer to QTOP object 1661 * 1662 * @return none 1663 */ 1664 void 1665 ocs_hw_qtop_free(ocs_hw_qtop_t *qtop) 1666 { 1667 if (qtop != NULL) { 1668 if (qtop->entries != NULL) { 1669 ocs_free(qtop->os, qtop->entries, qtop->alloc_count*sizeof(*qtop->entries)); 1670 } 1671 ocs_free(qtop->os, qtop, sizeof(*qtop)); 1672 } 1673 } 1674 1675 /* Uncomment this to turn on RQ debug */ 1676 // #define ENABLE_DEBUG_RQBUF 1677 1678 static int32_t ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id); 1679 static ocs_hw_sequence_t * ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex); 1680 static int32_t ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq); 1681 static ocs_hw_rtn_e ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq); 1682 1683 /** 1684 * @brief Process receive queue completions for RQ Pair mode. 1685 * 1686 * @par Description 1687 * RQ completions are processed. In RQ pair mode, a single header and single payload 1688 * buffer are received, and passed to the function that has registered for unsolicited 1689 * callbacks. 1690 * 1691 * @param hw Hardware context. 1692 * @param cq Pointer to HW completion queue. 1693 * @param cqe Completion queue entry. 1694 * 1695 * @return Returns 0 for success, or a negative error code value for failure. 1696 */ 1697 1698 int32_t 1699 ocs_hw_rqpair_process_rq(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1700 { 1701 uint16_t rq_id; 1702 uint32_t index; 1703 int32_t rqindex; 1704 int32_t rq_status; 1705 uint32_t h_len; 1706 uint32_t p_len; 1707 ocs_hw_sequence_t *seq; 1708 1709 rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index); 1710 if (0 != rq_status) { 1711 switch (rq_status) { 1712 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED: 1713 case SLI4_FC_ASYNC_RQ_DMA_FAILURE: 1714 /* just get RQ buffer then return to chip */ 1715 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1716 if (rqindex < 0) { 1717 ocs_log_test(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n", 1718 rq_status, rq_id); 1719 break; 1720 } 1721 1722 /* get RQ buffer */ 1723 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1724 1725 /* return to chip */ 1726 if (ocs_hw_rqpair_sequence_free(hw, seq)) { 1727 ocs_log_test(hw->os, "status=%#x, failed to return buffers to RQ\n", 1728 rq_status); 1729 break; 1730 } 1731 break; 1732 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED: 1733 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC: 1734 /* since RQ buffers were not consumed, cannot return them to chip */ 1735 /* fall through */ 1736 ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status); 1737 default: 1738 break; 1739 } 1740 return -1; 1741 } 1742 1743 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1744 if (rqindex < 0) { 1745 ocs_log_test(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id); 1746 return -1; 1747 } 1748 1749 OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++; 1750 rq->payload_use_count++;}) 1751 1752 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1753 ocs_hw_assert(seq != NULL); 1754 1755 seq->hw = hw; 1756 seq->auto_xrdy = 0; 1757 seq->out_of_xris = 0; 1758 seq->xri = 0; 1759 seq->hio = NULL; 1760 1761 sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len); 1762 seq->header->dma.len = h_len; 1763 seq->payload->dma.len = p_len; 1764 seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe); 1765 seq->hw_priv = cq->eq; 1766 1767 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1768 if (hw->config.bounce) { 1769 fc_header_t *hdr = seq->header->dma.virt; 1770 uint32_t s_id = fc_be24toh(hdr->s_id); 1771 uint32_t d_id = fc_be24toh(hdr->d_id); 1772 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1773 if (hw->callback.bounce != NULL) { 1774 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 1775 } 1776 } else { 1777 hw->callback.unsolicited(hw->args.unsolicited, seq); 1778 } 1779 1780 return 0; 1781 } 1782 1783 /** 1784 * @brief Process receive queue completions for RQ Pair mode - Auto xfer rdy 1785 * 1786 * @par Description 1787 * RQ completions are processed. In RQ pair mode, a single header and single payload 1788 * buffer are received, and passed to the function that has registered for unsolicited 1789 * callbacks. 1790 * 1791 * @param hw Hardware context. 1792 * @param cq Pointer to HW completion queue. 1793 * @param cqe Completion queue entry. 1794 * 1795 * @return Returns 0 for success, or a negative error code value for failure. 1796 */ 1797 1798 int32_t 1799 ocs_hw_rqpair_process_auto_xfr_rdy_cmd(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1800 { 1801 /* Seems silly to call a SLI function to decode - use the structure directly for performance */ 1802 sli4_fc_optimized_write_cmd_cqe_t *opt_wr = (sli4_fc_optimized_write_cmd_cqe_t*)cqe; 1803 uint16_t rq_id; 1804 uint32_t index; 1805 int32_t rqindex; 1806 int32_t rq_status; 1807 uint32_t h_len; 1808 uint32_t p_len; 1809 ocs_hw_sequence_t *seq; 1810 uint8_t axr_lock_taken = 0; 1811 #if defined(OCS_DISC_SPIN_DELAY) 1812 uint32_t delay = 0; 1813 char prop_buf[32]; 1814 #endif 1815 1816 rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index); 1817 if (0 != rq_status) { 1818 switch (rq_status) { 1819 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED: 1820 case SLI4_FC_ASYNC_RQ_DMA_FAILURE: 1821 /* just get RQ buffer then return to chip */ 1822 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1823 if (rqindex < 0) { 1824 ocs_log_err(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n", 1825 rq_status, rq_id); 1826 break; 1827 } 1828 1829 /* get RQ buffer */ 1830 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1831 1832 /* return to chip */ 1833 if (ocs_hw_rqpair_sequence_free(hw, seq)) { 1834 ocs_log_err(hw->os, "status=%#x, failed to return buffers to RQ\n", 1835 rq_status); 1836 break; 1837 } 1838 break; 1839 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED: 1840 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC: 1841 /* since RQ buffers were not consumed, cannot return them to chip */ 1842 ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status); 1843 /* fall through */ 1844 default: 1845 break; 1846 } 1847 return -1; 1848 } 1849 1850 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1851 if (rqindex < 0) { 1852 ocs_log_err(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id); 1853 return -1; 1854 } 1855 1856 OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++; 1857 rq->payload_use_count++;}) 1858 1859 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1860 ocs_hw_assert(seq != NULL); 1861 1862 seq->hw = hw; 1863 seq->auto_xrdy = opt_wr->agxr; 1864 seq->out_of_xris = opt_wr->oox; 1865 seq->xri = opt_wr->xri; 1866 seq->hio = NULL; 1867 1868 sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len); 1869 seq->header->dma.len = h_len; 1870 seq->payload->dma.len = p_len; 1871 seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe); 1872 seq->hw_priv = cq->eq; 1873 1874 if (seq->auto_xrdy) { 1875 fc_header_t *fc_hdr = seq->header->dma.virt; 1876 1877 seq->hio = ocs_hw_io_lookup(hw, seq->xri); 1878 ocs_lock(&seq->hio->axr_lock); 1879 axr_lock_taken = 1; 1880 1881 /* save the FCFI, src_id, dest_id and ox_id because we need it for the sequence object when the data comes. */ 1882 seq->hio->axr_buf->fcfi = seq->fcfi; 1883 seq->hio->axr_buf->hdr.ox_id = fc_hdr->ox_id; 1884 seq->hio->axr_buf->hdr.s_id = fc_hdr->s_id; 1885 seq->hio->axr_buf->hdr.d_id = fc_hdr->d_id; 1886 seq->hio->axr_buf->cmd_cqe = 1; 1887 1888 /* 1889 * Since auto xfer rdy is used for this IO, then clear the sequence 1890 * initiative bit in the header so that the upper layers wait for the 1891 * data. This should flow exactly like the first burst case. 1892 */ 1893 fc_hdr->f_ctl &= fc_htobe24(~FC_FCTL_SEQUENCE_INITIATIVE); 1894 1895 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */ 1896 if (seq->hio->type == OCS_HW_IO_TARGET_RSP) { 1897 seq->hio->axr_buf->call_axr_cmd = 1; 1898 seq->hio->axr_buf->cmd_seq = seq; 1899 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd; 1900 } 1901 } 1902 1903 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1904 if (hw->config.bounce) { 1905 fc_header_t *hdr = seq->header->dma.virt; 1906 uint32_t s_id = fc_be24toh(hdr->s_id); 1907 uint32_t d_id = fc_be24toh(hdr->d_id); 1908 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1909 if (hw->callback.bounce != NULL) { 1910 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 1911 } 1912 } else { 1913 hw->callback.unsolicited(hw->args.unsolicited, seq); 1914 } 1915 1916 if (seq->auto_xrdy) { 1917 /* If data cqe came before cmd cqe in out of order in case of AXR */ 1918 if(seq->hio->axr_buf->data_cqe == 1) { 1919 #if defined(OCS_DISC_SPIN_DELAY) 1920 if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) { 1921 delay = ocs_strtoul(prop_buf, 0, 0); 1922 ocs_udelay(delay); 1923 } 1924 #endif 1925 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1926 if (hw->config.bounce) { 1927 fc_header_t *hdr = seq->header->dma.virt; 1928 uint32_t s_id = fc_be24toh(hdr->s_id); 1929 uint32_t d_id = fc_be24toh(hdr->d_id); 1930 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1931 if (hw->callback.bounce != NULL) { 1932 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, &seq->hio->axr_buf->seq, s_id, d_id, ox_id); 1933 } 1934 } else { 1935 hw->callback.unsolicited(hw->args.unsolicited, &seq->hio->axr_buf->seq); 1936 } 1937 } 1938 } 1939 1940 exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd: 1941 if(axr_lock_taken) { 1942 ocs_unlock(&seq->hio->axr_lock); 1943 } 1944 return 0; 1945 } 1946 1947 /** 1948 * @brief Process CQ completions for Auto xfer rdy data phases. 1949 * 1950 * @par Description 1951 * The data is DMA'd into the data buffer posted to the SGL prior to the XRI 1952 * being assigned to an IO. When the completion is received, All of the data 1953 * is in the single buffer. 1954 * 1955 * @param hw Hardware context. 1956 * @param cq Pointer to HW completion queue. 1957 * @param cqe Completion queue entry. 1958 * 1959 * @return Returns 0 for success, or a negative error code value for failure. 1960 */ 1961 1962 int32_t 1963 ocs_hw_rqpair_process_auto_xfr_rdy_data(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1964 { 1965 /* Seems silly to call a SLI function to decode - use the structure directly for performance */ 1966 sli4_fc_optimized_write_data_cqe_t *opt_wr = (sli4_fc_optimized_write_data_cqe_t*)cqe; 1967 ocs_hw_sequence_t *seq; 1968 ocs_hw_io_t *io; 1969 ocs_hw_auto_xfer_rdy_buffer_t *buf; 1970 #if defined(OCS_DISC_SPIN_DELAY) 1971 uint32_t delay = 0; 1972 char prop_buf[32]; 1973 #endif 1974 /* Look up the IO */ 1975 io = ocs_hw_io_lookup(hw, opt_wr->xri); 1976 ocs_lock(&io->axr_lock); 1977 buf = io->axr_buf; 1978 buf->data_cqe = 1; 1979 seq = &buf->seq; 1980 seq->hw = hw; 1981 seq->auto_xrdy = 1; 1982 seq->out_of_xris = 0; 1983 seq->xri = opt_wr->xri; 1984 seq->hio = io; 1985 seq->header = &buf->header; 1986 seq->payload = &buf->payload; 1987 1988 seq->header->dma.len = sizeof(fc_header_t); 1989 seq->payload->dma.len = opt_wr->total_data_placed; 1990 seq->fcfi = buf->fcfi; 1991 seq->hw_priv = cq->eq; 1992 1993 if (opt_wr->status == SLI4_FC_WCQE_STATUS_SUCCESS) { 1994 seq->status = OCS_HW_UNSOL_SUCCESS; 1995 } else if (opt_wr->status == SLI4_FC_WCQE_STATUS_REMOTE_STOP) { 1996 seq->status = OCS_HW_UNSOL_ABTS_RCVD; 1997 } else { 1998 seq->status = OCS_HW_UNSOL_ERROR; 1999 } 2000 2001 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */ 2002 if(io->type == OCS_HW_IO_TARGET_RSP) { 2003 io->axr_buf->call_axr_data = 1; 2004 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data; 2005 } 2006 2007 if(!buf->cmd_cqe) { 2008 /* if data cqe came before cmd cqe, return here, cmd cqe will handle */ 2009 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data; 2010 } 2011 #if defined(OCS_DISC_SPIN_DELAY) 2012 if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) { 2013 delay = ocs_strtoul(prop_buf, 0, 0); 2014 ocs_udelay(delay); 2015 } 2016 #endif 2017 2018 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 2019 if (hw->config.bounce) { 2020 fc_header_t *hdr = seq->header->dma.virt; 2021 uint32_t s_id = fc_be24toh(hdr->s_id); 2022 uint32_t d_id = fc_be24toh(hdr->d_id); 2023 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 2024 if (hw->callback.bounce != NULL) { 2025 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 2026 } 2027 } else { 2028 hw->callback.unsolicited(hw->args.unsolicited, seq); 2029 } 2030 2031 exit_ocs_hw_rqpair_process_auto_xfr_rdy_data: 2032 ocs_unlock(&io->axr_lock); 2033 return 0; 2034 } 2035 2036 /** 2037 * @brief Return pointer to RQ buffer entry. 2038 * 2039 * @par Description 2040 * Returns a pointer to the RQ buffer entry given by @c rqindex and @c bufindex. 2041 * 2042 * @param hw Hardware context. 2043 * @param rqindex Index of the RQ that is being processed. 2044 * @param bufindex Index into the RQ that is being processed. 2045 * 2046 * @return Pointer to the sequence structure, or NULL otherwise. 2047 */ 2048 static ocs_hw_sequence_t * 2049 ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex) 2050 { 2051 sli4_queue_t *rq_hdr = &hw->rq[rqindex]; 2052 sli4_queue_t *rq_payload = &hw->rq[rqindex+1]; 2053 ocs_hw_sequence_t *seq = NULL; 2054 hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; 2055 2056 #if defined(ENABLE_DEBUG_RQBUF) 2057 uint64_t rqbuf_debug_value = 0xdead0000 | ((rq->id & 0xf) << 12) | (bufindex & 0xfff); 2058 #endif 2059 2060 if (bufindex >= rq_hdr->length) { 2061 ocs_log_err(hw->os, "RQ index %d bufindex %d exceed ring length %d for id %d\n", 2062 rqindex, bufindex, rq_hdr->length, rq_hdr->id); 2063 return NULL; 2064 } 2065 2066 sli_queue_lock(rq_hdr); 2067 sli_queue_lock(rq_payload); 2068 2069 #if defined(ENABLE_DEBUG_RQBUF) 2070 /* Put a debug value into the rq, to track which entries are still valid */ 2071 _sli_queue_poke(&hw->sli, rq_hdr, bufindex, (uint8_t *)&rqbuf_debug_value); 2072 _sli_queue_poke(&hw->sli, rq_payload, bufindex, (uint8_t *)&rqbuf_debug_value); 2073 #endif 2074 2075 seq = rq->rq_tracker[bufindex]; 2076 rq->rq_tracker[bufindex] = NULL; 2077 2078 if (seq == NULL ) { 2079 ocs_log_err(hw->os, "RQ buffer NULL, rqindex %d, bufindex %d, current q index = %d\n", 2080 rqindex, bufindex, rq_hdr->index); 2081 } 2082 2083 sli_queue_unlock(rq_payload); 2084 sli_queue_unlock(rq_hdr); 2085 return seq; 2086 } 2087 2088 /** 2089 * @brief Posts an RQ buffer to a queue and update the verification structures 2090 * 2091 * @param hw hardware context 2092 * @param seq Pointer to sequence object. 2093 * 2094 * @return Returns 0 on success, or a non-zero value otherwise. 2095 */ 2096 static int32_t 2097 ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2098 { 2099 sli4_queue_t *rq_hdr = &hw->rq[seq->header->rqindex]; 2100 sli4_queue_t *rq_payload = &hw->rq[seq->payload->rqindex]; 2101 uint32_t hw_rq_index = hw->hw_rq_lookup[seq->header->rqindex]; 2102 hw_rq_t *rq = hw->hw_rq[hw_rq_index]; 2103 uint32_t phys_hdr[2]; 2104 uint32_t phys_payload[2]; 2105 int32_t qindex_hdr; 2106 int32_t qindex_payload; 2107 2108 /* Update the RQ verification lookup tables */ 2109 phys_hdr[0] = ocs_addr32_hi(seq->header->dma.phys); 2110 phys_hdr[1] = ocs_addr32_lo(seq->header->dma.phys); 2111 phys_payload[0] = ocs_addr32_hi(seq->payload->dma.phys); 2112 phys_payload[1] = ocs_addr32_lo(seq->payload->dma.phys); 2113 2114 sli_queue_lock(rq_hdr); 2115 sli_queue_lock(rq_payload); 2116 2117 /* 2118 * Note: The header must be posted last for buffer pair mode because 2119 * posting on the header queue posts the payload queue as well. 2120 * We do not ring the payload queue independently in RQ pair mode. 2121 */ 2122 qindex_payload = _sli_queue_write(&hw->sli, rq_payload, (void *)phys_payload); 2123 qindex_hdr = _sli_queue_write(&hw->sli, rq_hdr, (void *)phys_hdr); 2124 if (qindex_hdr < 0 || 2125 qindex_payload < 0) { 2126 ocs_log_err(hw->os, "RQ_ID=%#x write failed\n", rq_hdr->id); 2127 sli_queue_unlock(rq_payload); 2128 sli_queue_unlock(rq_hdr); 2129 return OCS_HW_RTN_ERROR; 2130 } 2131 2132 /* ensure the indexes are the same */ 2133 ocs_hw_assert(qindex_hdr == qindex_payload); 2134 2135 /* Update the lookup table */ 2136 if (rq->rq_tracker[qindex_hdr] == NULL) { 2137 rq->rq_tracker[qindex_hdr] = seq; 2138 } else { 2139 ocs_log_test(hw->os, "expected rq_tracker[%d][%d] buffer to be NULL\n", 2140 hw_rq_index, qindex_hdr); 2141 } 2142 2143 sli_queue_unlock(rq_payload); 2144 sli_queue_unlock(rq_hdr); 2145 return OCS_HW_RTN_SUCCESS; 2146 } 2147 2148 /** 2149 * @brief Return RQ buffers (while in RQ pair mode). 2150 * 2151 * @par Description 2152 * The header and payload buffers are returned to the Receive Queue. 2153 * 2154 * @param hw Hardware context. 2155 * @param seq Header/payload sequence buffers. 2156 * 2157 * @return Returns OCS_HW_RTN_SUCCESS on success, or an error code value on failure. 2158 */ 2159 2160 ocs_hw_rtn_e 2161 ocs_hw_rqpair_sequence_free(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2162 { 2163 ocs_hw_rtn_e rc = OCS_HW_RTN_SUCCESS; 2164 2165 /* Check for auto xfer rdy dummy buffers and call the proper release function. */ 2166 if (seq->header->rqindex == OCS_HW_RQ_INDEX_DUMMY_HDR) { 2167 return ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(hw, seq); 2168 } 2169 2170 /* 2171 * Post the data buffer first. Because in RQ pair mode, ringing the 2172 * doorbell of the header ring will post the data buffer as well. 2173 */ 2174 if (ocs_hw_rqpair_put(hw, seq)) { 2175 ocs_log_err(hw->os, "error writing buffers\n"); 2176 return OCS_HW_RTN_ERROR; 2177 } 2178 2179 return rc; 2180 } 2181 2182 /** 2183 * @brief Find the RQ index of RQ_ID. 2184 * 2185 * @param hw Hardware context. 2186 * @param rq_id RQ ID to find. 2187 * 2188 * @return Returns the RQ index, or -1 if not found 2189 */ 2190 static inline int32_t 2191 ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id) 2192 { 2193 return ocs_hw_queue_hash_find(hw->rq_hash, rq_id); 2194 } 2195 2196 /** 2197 * @ingroup devInitShutdown 2198 * @brief Allocate auto xfer rdy buffers. 2199 * 2200 * @par Description 2201 * Allocates the auto xfer rdy buffers and places them on the free list. 2202 * 2203 * @param hw Hardware context allocated by the caller. 2204 * @param num_buffers Number of buffers to allocate. 2205 * 2206 * @return Returns 0 on success, or a non-zero value on failure. 2207 */ 2208 ocs_hw_rtn_e 2209 ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(ocs_hw_t *hw, uint32_t num_buffers) 2210 { 2211 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2212 uint32_t i; 2213 2214 hw->auto_xfer_rdy_buf_pool = ocs_pool_alloc(hw->os, sizeof(ocs_hw_auto_xfer_rdy_buffer_t), num_buffers, FALSE); 2215 if (hw->auto_xfer_rdy_buf_pool == NULL) { 2216 ocs_log_err(hw->os, "Failure to allocate auto xfer ready buffer pool\n"); 2217 return OCS_HW_RTN_NO_MEMORY; 2218 } 2219 2220 for (i = 0; i < num_buffers; i++) { 2221 /* allocate the wrapper object */ 2222 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i); 2223 ocs_hw_assert(buf != NULL); 2224 2225 /* allocate the auto xfer ready buffer */ 2226 if (ocs_dma_alloc(hw->os, &buf->payload.dma, hw->config.auto_xfer_rdy_size, OCS_MIN_DMA_ALIGNMENT)) { 2227 ocs_log_err(hw->os, "DMA allocation failed\n"); 2228 ocs_free(hw->os, buf, sizeof(*buf)); 2229 return OCS_HW_RTN_NO_MEMORY; 2230 } 2231 2232 /* build a fake data header in big endian */ 2233 buf->hdr.info = FC_RCTL_INFO_SOL_DATA; 2234 buf->hdr.r_ctl = FC_RCTL_FC4_DATA; 2235 buf->hdr.type = FC_TYPE_FCP; 2236 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER | 2237 FC_FCTL_FIRST_SEQUENCE | 2238 FC_FCTL_LAST_SEQUENCE | 2239 FC_FCTL_END_SEQUENCE | 2240 FC_FCTL_SEQUENCE_INITIATIVE); 2241 2242 /* build the fake header DMA object */ 2243 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR; 2244 buf->header.dma.virt = &buf->hdr; 2245 buf->header.dma.alloc = buf; 2246 buf->header.dma.size = sizeof(buf->hdr); 2247 buf->header.dma.len = sizeof(buf->hdr); 2248 2249 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA; 2250 } 2251 return OCS_HW_RTN_SUCCESS; 2252 } 2253 2254 /** 2255 * @ingroup devInitShutdown 2256 * @brief Post Auto xfer rdy buffers to the XRIs posted with DNRX. 2257 * 2258 * @par Description 2259 * When new buffers are freed, check existing XRIs waiting for buffers. 2260 * 2261 * @param hw Hardware context allocated by the caller. 2262 */ 2263 static void 2264 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(ocs_hw_t *hw) 2265 { 2266 ocs_hw_io_t *io; 2267 int32_t rc; 2268 2269 ocs_lock(&hw->io_lock); 2270 2271 while (!ocs_list_empty(&hw->io_port_dnrx)) { 2272 io = ocs_list_remove_head(&hw->io_port_dnrx); 2273 rc = ocs_hw_reque_xri(hw, io); 2274 if(rc) { 2275 break; 2276 } 2277 } 2278 2279 ocs_unlock(&hw->io_lock); 2280 } 2281 2282 /** 2283 * @brief Called when the POST_SGL_PAGE command completes. 2284 * 2285 * @par Description 2286 * Free the mailbox command buffer. 2287 * 2288 * @param hw Hardware context. 2289 * @param status Status field from the mbox completion. 2290 * @param mqe Mailbox response structure. 2291 * @param arg Pointer to a callback function that signals the caller that the command is done. 2292 * 2293 * @return Returns 0. 2294 */ 2295 static int32_t 2296 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb(ocs_hw_t *hw, int32_t status, uint8_t *mqe, void *arg) 2297 { 2298 if (status != 0) { 2299 ocs_log_debug(hw->os, "Status 0x%x\n", status); 2300 } 2301 2302 ocs_free(hw->os, mqe, SLI4_BMBX_SIZE); 2303 return 0; 2304 } 2305 2306 /** 2307 * @brief Prepares an XRI to move to the chip. 2308 * 2309 * @par Description 2310 * Puts the data SGL into the SGL list for the IO object and possibly registers 2311 * an SGL list for the XRI. Since both the POST_XRI and POST_SGL_PAGES commands are 2312 * mailbox commands, we don't need to wait for completion before preceding. 2313 * 2314 * @param hw Hardware context allocated by the caller. 2315 * @param io Pointer to the IO object. 2316 * 2317 * @return Returns OCS_HW_RTN_SUCCESS for success, or an error code value for failure. 2318 */ 2319 ocs_hw_rtn_e 2320 ocs_hw_rqpair_auto_xfer_rdy_move_to_port(ocs_hw_t *hw, ocs_hw_io_t *io) 2321 { 2322 /* We only need to preregister the SGL if it has not yet been done. */ 2323 if (!sli_get_sgl_preregister(&hw->sli)) { 2324 uint8_t *post_sgl; 2325 ocs_dma_t *psgls = &io->def_sgl; 2326 ocs_dma_t **sgls = &psgls; 2327 2328 /* non-local buffer required for mailbox queue */ 2329 post_sgl = ocs_malloc(hw->os, SLI4_BMBX_SIZE, OCS_M_NOWAIT); 2330 if (post_sgl == NULL) { 2331 ocs_log_err(hw->os, "no buffer for command\n"); 2332 return OCS_HW_RTN_NO_MEMORY; 2333 } 2334 if (sli_cmd_fcoe_post_sgl_pages(&hw->sli, post_sgl, SLI4_BMBX_SIZE, 2335 io->indicator, 1, sgls, NULL, NULL)) { 2336 if (ocs_hw_command(hw, post_sgl, OCS_CMD_NOWAIT, 2337 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb, NULL)) { 2338 ocs_free(hw->os, post_sgl, SLI4_BMBX_SIZE); 2339 ocs_log_err(hw->os, "SGL post failed\n"); 2340 return OCS_HW_RTN_ERROR; 2341 } 2342 } 2343 } 2344 2345 ocs_lock(&hw->io_lock); 2346 if (ocs_hw_rqpair_auto_xfer_rdy_buffer_post(hw, io, 0) != 0) { /* DNRX set - no buffer */ 2347 ocs_unlock(&hw->io_lock); 2348 return OCS_HW_RTN_ERROR; 2349 } 2350 ocs_unlock(&hw->io_lock); 2351 return OCS_HW_RTN_SUCCESS; 2352 } 2353 2354 /** 2355 * @brief Prepares an XRI to move back to the host. 2356 * 2357 * @par Description 2358 * Releases any attached buffer back to the pool. 2359 * 2360 * @param hw Hardware context allocated by the caller. 2361 * @param io Pointer to the IO object. 2362 */ 2363 void 2364 ocs_hw_rqpair_auto_xfer_rdy_move_to_host(ocs_hw_t *hw, ocs_hw_io_t *io) 2365 { 2366 if (io->axr_buf != NULL) { 2367 ocs_lock(&hw->io_lock); 2368 /* check list and remove if there */ 2369 if (ocs_list_on_list(&io->dnrx_link)) { 2370 ocs_list_remove(&hw->io_port_dnrx, io); 2371 io->auto_xfer_rdy_dnrx = 0; 2372 2373 /* release the count for waiting for a buffer */ 2374 ocs_hw_io_free(hw, io); 2375 } 2376 2377 ocs_pool_put(hw->auto_xfer_rdy_buf_pool, io->axr_buf); 2378 io->axr_buf = NULL; 2379 ocs_unlock(&hw->io_lock); 2380 2381 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw); 2382 } 2383 return; 2384 } 2385 2386 /** 2387 * @brief Posts an auto xfer rdy buffer to an IO. 2388 * 2389 * @par Description 2390 * Puts the data SGL into the SGL list for the IO object 2391 * @n @name 2392 * @b Note: io_lock must be held. 2393 * 2394 * @param hw Hardware context allocated by the caller. 2395 * @param io Pointer to the IO object. 2396 * 2397 * @return Returns the value of DNRX bit in the TRSP and ABORT WQEs. 2398 */ 2399 uint8_t 2400 ocs_hw_rqpair_auto_xfer_rdy_buffer_post(ocs_hw_t *hw, ocs_hw_io_t *io, int reuse_buf) 2401 { 2402 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2403 sli4_sge_t *data; 2404 2405 if(!reuse_buf) { 2406 buf = ocs_pool_get(hw->auto_xfer_rdy_buf_pool); 2407 io->axr_buf = buf; 2408 } 2409 2410 data = io->def_sgl.virt; 2411 data[0].sge_type = SLI4_SGE_TYPE_SKIP; 2412 data[0].last = 0; 2413 2414 /* 2415 * Note: if we are doing DIF assists, then the SGE[1] must contain the 2416 * DI_SEED SGE. The host is responsible for programming: 2417 * SGE Type (Word 2, bits 30:27) 2418 * Replacement App Tag (Word 2 bits 15:0) 2419 * App Tag (Word 3 bits 15:0) 2420 * New Ref Tag (Word 3 bit 23) 2421 * Metadata Enable (Word 3 bit 20) 2422 * Auto-Increment RefTag (Word 3 bit 19) 2423 * Block Size (Word 3 bits 18:16) 2424 * The following fields are managed by the SLI Port: 2425 * Ref Tag Compare (Word 0) 2426 * Replacement Ref Tag (Word 1) - In not the LBA 2427 * NA (Word 2 bit 25) 2428 * Opcode RX (Word 3 bits 27:24) 2429 * Checksum Enable (Word 3 bit 22) 2430 * RefTag Enable (Word 3 bit 21) 2431 * 2432 * The first two SGLs are cleared by ocs_hw_io_init_sges(), so assume eveything is cleared. 2433 */ 2434 if (hw->config.auto_xfer_rdy_p_type) { 2435 sli4_diseed_sge_t *diseed = (sli4_diseed_sge_t*)&data[1]; 2436 2437 diseed->sge_type = SLI4_SGE_TYPE_DISEED; 2438 diseed->repl_app_tag = hw->config.auto_xfer_rdy_app_tag_value; 2439 diseed->app_tag_cmp = hw->config.auto_xfer_rdy_app_tag_value; 2440 diseed->check_app_tag = hw->config.auto_xfer_rdy_app_tag_valid; 2441 diseed->auto_incr_ref_tag = TRUE; /* Always the LBA */ 2442 diseed->dif_blk_size = hw->config.auto_xfer_rdy_blk_size_chip; 2443 } else { 2444 data[1].sge_type = SLI4_SGE_TYPE_SKIP; 2445 data[1].last = 0; 2446 } 2447 2448 data[2].sge_type = SLI4_SGE_TYPE_DATA; 2449 data[2].buffer_address_high = ocs_addr32_hi(io->axr_buf->payload.dma.phys); 2450 data[2].buffer_address_low = ocs_addr32_lo(io->axr_buf->payload.dma.phys); 2451 data[2].buffer_length = io->axr_buf->payload.dma.size; 2452 data[2].last = TRUE; 2453 data[3].sge_type = SLI4_SGE_TYPE_SKIP; 2454 2455 return 0; 2456 } 2457 2458 /** 2459 * @brief Return auto xfer ready buffers (while in RQ pair mode). 2460 * 2461 * @par Description 2462 * The header and payload buffers are returned to the auto xfer rdy pool. 2463 * 2464 * @param hw Hardware context. 2465 * @param seq Header/payload sequence buffers. 2466 * 2467 * @return Returns OCS_HW_RTN_SUCCESS for success, an error code value for failure. 2468 */ 2469 2470 static ocs_hw_rtn_e 2471 ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2472 { 2473 ocs_hw_auto_xfer_rdy_buffer_t *buf = seq->header->dma.alloc; 2474 2475 buf->data_cqe = 0; 2476 buf->cmd_cqe = 0; 2477 buf->fcfi = 0; 2478 buf->call_axr_cmd = 0; 2479 buf->call_axr_data = 0; 2480 2481 /* build a fake data header in big endian */ 2482 buf->hdr.info = FC_RCTL_INFO_SOL_DATA; 2483 buf->hdr.r_ctl = FC_RCTL_FC4_DATA; 2484 buf->hdr.type = FC_TYPE_FCP; 2485 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER | 2486 FC_FCTL_FIRST_SEQUENCE | 2487 FC_FCTL_LAST_SEQUENCE | 2488 FC_FCTL_END_SEQUENCE | 2489 FC_FCTL_SEQUENCE_INITIATIVE); 2490 2491 /* build the fake header DMA object */ 2492 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR; 2493 buf->header.dma.virt = &buf->hdr; 2494 buf->header.dma.alloc = buf; 2495 buf->header.dma.size = sizeof(buf->hdr); 2496 buf->header.dma.len = sizeof(buf->hdr); 2497 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA; 2498 2499 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw); 2500 2501 return OCS_HW_RTN_SUCCESS; 2502 } 2503 2504 /** 2505 * @ingroup devInitShutdown 2506 * @brief Free auto xfer rdy buffers. 2507 * 2508 * @par Description 2509 * Frees the auto xfer rdy buffers. 2510 * 2511 * @param hw Hardware context allocated by the caller. 2512 * 2513 * @return Returns 0 on success, or a non-zero value on failure. 2514 */ 2515 static void 2516 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(ocs_hw_t *hw) 2517 { 2518 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2519 uint32_t i; 2520 2521 if (hw->auto_xfer_rdy_buf_pool != NULL) { 2522 ocs_lock(&hw->io_lock); 2523 for (i = 0; i < ocs_pool_get_count(hw->auto_xfer_rdy_buf_pool); i++) { 2524 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i); 2525 if (buf != NULL) { 2526 ocs_dma_free(hw->os, &buf->payload.dma); 2527 } 2528 } 2529 ocs_unlock(&hw->io_lock); 2530 2531 ocs_pool_free(hw->auto_xfer_rdy_buf_pool); 2532 hw->auto_xfer_rdy_buf_pool = NULL; 2533 } 2534 } 2535 2536 /** 2537 * @ingroup devInitShutdown 2538 * @brief Configure the rq_pair function from ocs_hw_init(). 2539 * 2540 * @par Description 2541 * Allocates the buffers to auto xfer rdy and posts initial XRIs for this feature. 2542 * 2543 * @param hw Hardware context allocated by the caller. 2544 * 2545 * @return Returns 0 on success, or a non-zero value on failure. 2546 */ 2547 ocs_hw_rtn_e 2548 ocs_hw_rqpair_init(ocs_hw_t *hw) 2549 { 2550 ocs_hw_rtn_e rc; 2551 uint32_t xris_posted; 2552 2553 ocs_log_debug(hw->os, "RQ Pair mode\n"); 2554 2555 /* 2556 * If we get this far, the auto XFR_RDY feature was enabled successfully, otherwise ocs_hw_init() would 2557 * return with an error. So allocate the buffers based on the initial XRI pool required to support this 2558 * feature. 2559 */ 2560 if (sli_get_auto_xfer_rdy_capable(&hw->sli) && 2561 hw->config.auto_xfer_rdy_size > 0) { 2562 if (hw->auto_xfer_rdy_buf_pool == NULL) { 2563 /* 2564 * Allocate one more buffer than XRIs so that when all the XRIs are in use, we still have 2565 * one to post back for the case where the response phase is started in the context of 2566 * the data completion. 2567 */ 2568 rc = ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(hw, hw->config.auto_xfer_rdy_xri_cnt + 1); 2569 if (rc != OCS_HW_RTN_SUCCESS) { 2570 return rc; 2571 } 2572 } else { 2573 ocs_pool_reset(hw->auto_xfer_rdy_buf_pool); 2574 } 2575 2576 /* Post the auto XFR_RDY XRIs */ 2577 xris_posted = ocs_hw_xri_move_to_port_owned(hw, hw->config.auto_xfer_rdy_xri_cnt); 2578 if (xris_posted != hw->config.auto_xfer_rdy_xri_cnt) { 2579 ocs_log_err(hw->os, "post_xri failed, only posted %d XRIs\n", xris_posted); 2580 return OCS_HW_RTN_ERROR; 2581 } 2582 } 2583 2584 return 0; 2585 } 2586 2587 /** 2588 * @ingroup devInitShutdown 2589 * @brief Tear down the rq_pair function from ocs_hw_teardown(). 2590 * 2591 * @par Description 2592 * Frees the buffers to auto xfer rdy. 2593 * 2594 * @param hw Hardware context allocated by the caller. 2595 */ 2596 void 2597 ocs_hw_rqpair_teardown(ocs_hw_t *hw) 2598 { 2599 /* We need to free any auto xfer ready buffers */ 2600 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(hw); 2601 } 2602