1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause 3 * 4 * Copyright © 2014-2016 Freescale Semiconductor, Inc. 5 * Copyright © 2016-2019 NXP 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * 3. Neither the name of the copyright holder nor the names of its 19 * contributors may be used to endorse or promote products derived from this 20 * software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 * 34 * Original source file obtained from: 35 * drivers/soc/fsl/dpio/qbman-portal.c 36 * 37 * Commit: 4c86114194e644b6da9107d75910635c9e87179e 38 * Repository: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git 39 */ 40 41 /* 42 * Copyright © 2021-2022 Dmitry Salychev 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 /* 68 * DPAA2 QBMan software portal. 69 */ 70 71 #include <sys/param.h> 72 #include <sys/kernel.h> 73 #include <sys/bus.h> 74 #include <sys/rman.h> 75 #include <sys/module.h> 76 #include <sys/malloc.h> 77 #include <sys/mutex.h> 78 #include <sys/time.h> 79 #include <sys/types.h> 80 #include <sys/systm.h> 81 #include <sys/condvar.h> 82 #include <sys/lock.h> 83 84 #include <machine/bus.h> 85 #include <machine/resource.h> 86 #include <machine/atomic.h> 87 88 #include "pcib_if.h" 89 #include "pci_if.h" 90 91 #include "dpaa2_swp.h" 92 #include "dpaa2_mc.h" 93 #include "dpaa2_bp.h" 94 95 #define CMD_SPIN_TIMEOUT 100u /* us */ 96 #define CMD_SPIN_ATTEMPTS 2000u /* 200 ms max. */ 97 98 #define CMD_VERB_MASK 0x7Fu 99 100 /* Shifts in the VERB byte of the enqueue command descriptor. */ 101 #define ENQ_CMD_ORP_ENABLE_SHIFT 2 102 #define ENQ_CMD_IRQ_ON_DISPATCH_SHIFT 3 103 #define ENQ_CMD_TARGET_TYPE_SHIFT 4 104 #define ENQ_CMD_DCA_EN_SHIFT 7 105 /* VERB byte options of the enqueue command descriptor. */ 106 #define ENQ_CMD_EMPTY 0u 107 #define ENQ_CMD_RESPONSE_ALWAYS 1u 108 #define ENQ_CMD_REJECTS_TO_FQ 2u 109 110 #define ENQ_DESC_FD_OFFSET 32u 111 112 #define ENQ_DCA_IDXMASK 0x0Fu 113 #define ENQ_FLAG_DCA (1ull << 31) 114 115 /* QBMan portal command codes. */ 116 #define CMDID_SWP_MC_ACQUIRE 0x30 117 #define CMDID_SWP_BP_QUERY 0x32 118 #define CMDID_SWP_WQCHAN_CONFIGURE 0x46 119 120 /* QBMan portal command result codes. */ 121 #define QBMAN_CMD_RC_OK 0xF0 122 123 /* SDQCR attribute codes */ 124 #define QB_SDQCR_FC_SHIFT 29u 125 #define QB_SDQCR_FC_MASK 0x1u 126 #define QB_SDQCR_DCT_SHIFT 24u 127 #define QB_SDQCR_DCT_MASK 0x3u 128 #define QB_SDQCR_TOK_SHIFT 16u 129 #define QB_SDQCR_TOK_MASK 0xFFu 130 #define QB_SDQCR_SRC_SHIFT 0u 131 #define QB_SDQCR_SRC_MASK 0xFFFFu 132 133 /* Shifts in the VERB byte of the volatile dequeue command. */ 134 #define QB_VDQCR_VERB_DCT0_SHIFT 0 135 #define QB_VDQCR_VERB_DCT1_SHIFT 1 136 #define QB_VDQCR_VERB_DT0_SHIFT 2 137 #define QB_VDQCR_VERB_DT1_SHIFT 3 138 #define QB_VDQCR_VERB_RLS_SHIFT 4 139 #define QB_VDQCR_VERB_WAE_SHIFT 5 140 #define QB_VDQCR_VERB_RAD_SHIFT 6 141 142 /* Maximum timeout period for the DQRR interrupt. */ 143 #define DQRR_MAX_ITP 4096u 144 #define DQRR_PI_MASK 0x0Fu 145 146 /* Release Array Allocation register helpers. */ 147 #define RAR_IDX(rar) ((rar) & 0x7u) 148 #define RAR_VB(rar) ((rar) & 0x80u) 149 #define RAR_SUCCESS(rar) ((rar) & 0x100u) 150 151 MALLOC_DEFINE(M_DPAA2_SWP, "dpaa2_swp", "DPAA2 QBMan Software Portal"); 152 153 enum qbman_sdqcr_dct { 154 qbman_sdqcr_dct_null = 0, 155 qbman_sdqcr_dct_prio_ics, 156 qbman_sdqcr_dct_active_ics, 157 qbman_sdqcr_dct_active 158 }; 159 160 enum qbman_sdqcr_fc { 161 qbman_sdqcr_fc_one = 0, 162 qbman_sdqcr_fc_up_to_3 = 1 163 }; 164 165 /* Routines to execute software portal commands. */ 166 static int dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *, 167 struct dpaa2_swp_cmd *, struct dpaa2_swp_rsp *, uint8_t); 168 static int dpaa2_swp_exec_br_command(struct dpaa2_swp *, struct dpaa2_swp_cmd *, 169 uint32_t); 170 static int dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *, 171 struct dpaa2_swp_cmd *); 172 173 /* Management Commands helpers. */ 174 static int dpaa2_swp_send_mgmt_command(struct dpaa2_swp *, 175 struct dpaa2_swp_cmd *, uint8_t); 176 static int dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *, 177 struct dpaa2_swp_rsp *); 178 179 /* Helper subroutines. */ 180 static int dpaa2_swp_cyc_diff(uint8_t, uint8_t, uint8_t); 181 182 int 183 dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc, 184 uint16_t flags) 185 { 186 struct dpaa2_swp *p; 187 uint32_t reg, mask_size, eqcr_pi; /* EQCR producer index */ 188 189 if (!swp || !desc) 190 return (DPAA2_SWP_STAT_EINVAL); 191 192 p = malloc(sizeof(struct dpaa2_swp), M_DPAA2_SWP, 193 flags & DPAA2_SWP_NOWAIT_ALLOC 194 ? (M_NOWAIT | M_ZERO) 195 : (M_WAITOK | M_ZERO)); 196 if (!p) 197 return (DPAA2_SWP_STAT_NO_MEMORY); 198 199 mtx_init(&p->lock, "swp_sleep_lock", NULL, MTX_DEF); 200 201 p->cfg.mem_backed = false; 202 p->cfg.writes_cinh = true; 203 204 p->desc = desc; 205 p->flags = flags; 206 p->mc.valid_bit = DPAA2_SWP_VALID_BIT; 207 p->mr.valid_bit = DPAA2_SWP_VALID_BIT; 208 209 /* FIXME: Memory-backed mode doesn't work now. Why? */ 210 p->cena_res = desc->cena_res; 211 p->cena_map = desc->cena_map; 212 p->cinh_res = desc->cinh_res; 213 p->cinh_map = desc->cinh_map; 214 215 /* Static Dequeue Command Register configuration. */ 216 p->sdq = 0; 217 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; 218 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; 219 p->sdq |= DPAA2_SWP_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; 220 221 /* Volatile Dequeue Command configuration. */ 222 p->vdq.valid_bit = DPAA2_SWP_VALID_BIT; 223 224 /* Dequeue Response Ring configuration */ 225 p->dqrr.next_idx = 0; 226 p->dqrr.valid_bit = DPAA2_SWP_VALID_BIT; 227 if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_4100) { 228 p->dqrr.ring_size = 4; 229 p->dqrr.reset_bug = 1; 230 } else { 231 p->dqrr.ring_size = 8; 232 p->dqrr.reset_bug = 0; 233 } 234 235 if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_5000) { 236 reg = dpaa2_swp_set_cfg( 237 p->dqrr.ring_size, /* max. entries QMan writes to DQRR */ 238 1, /* writes enabled in the CINH memory only */ 239 0, /* EQCR_CI stashing threshold */ 240 3, /* RPM: RCR in array mode */ 241 2, /* DCM: Discrete consumption ack */ 242 2, /* EPM: EQCR in ring mode (FIFO) */ 243 1, /* mem stashing drop enable enable */ 244 1, /* mem stashing priority enable */ 245 1, /* mem stashing enable */ 246 1, /* dequeue stashing priority enable */ 247 0, /* dequeue stashing enable enable */ 248 0 /* EQCR_CI stashing priority enable */ 249 ); 250 reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */ 251 } else { 252 bus_set_region_4(p->cena_map, 0, 0, 253 rman_get_size(p->cena_res) / 4); 254 255 reg = dpaa2_swp_set_cfg( 256 p->dqrr.ring_size, /* max. entries QMan writes to DQRR */ /* DQRR_MF */ 257 1, /* writes enabled in the CINH memory only */ /* WN */ 258 0, /* EQCR_CI stashing is disabled */ /* EST */ 259 3, /* RPM: RCR in array mode */ /* RPM */ 260 2, /* DCM: Discrete consumption ack */ /* DCM */ 261 2, /* EPM: EQCR in ring mode (FIFO) */ /* EPM */ 262 1, /* Dequeued frame data, annotation, and FQ context stashing drop enable */ /* SD */ 263 1, /* Dequeued frame data, annotation, and FQ context stashing priority */ /* SP */ 264 1, /* Dequeued frame data, annotation, and FQ context stashing enable */ /* SE */ 265 1, /* Dequeue response ring (DQRR) entry stashing priority */ /* DP */ 266 0, /* Dequeue response ring (DQRR) entry, or cacheable portal area, stashing enable. */ /* DE */ 267 0 /* EQCR_CI stashing priority */ /* EP */ 268 ); 269 /* TODO: Switch to memory-backed mode. */ 270 reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */ 271 } 272 dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_CFG, reg); 273 reg = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_CFG); 274 if (!reg) { 275 free(p, M_DPAA2_SWP); 276 return (DPAA2_SWP_STAT_PORTAL_DISABLED); 277 } 278 279 /* 280 * Static Dequeue Command Register needs to be initialized to 0 when no 281 * channels are being dequeued from or else the QMan HW will indicate an 282 * error. The values that were calculated above will be applied when 283 * dequeues from a specific channel are enabled. 284 */ 285 dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_SDQCR, 0); 286 287 p->eqcr.pi_ring_size = 8; 288 /* if ((desc->swp_version & DPAA2_SWP_REV_MASK) >= DPAA2_SWP_REV_5000) */ 289 /* p->eqcr.pi_ring_size = 32; */ 290 291 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) 292 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1; 293 294 eqcr_pi = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_PI); 295 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask; 296 p->eqcr.pi_vb = eqcr_pi & DPAA2_SWP_VALID_BIT; 297 p->eqcr.ci = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_CI) 298 & p->eqcr.pi_ci_mask; 299 p->eqcr.available = p->eqcr.pi_ring_size; 300 301 /* Initialize the portal with an IRQ threshold and timeout of 0us. */ 302 dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 0); 303 304 *swp = p; 305 306 return (0); 307 } 308 309 void 310 dpaa2_swp_free_portal(struct dpaa2_swp *swp) 311 { 312 uint16_t flags; 313 314 KASSERT(swp != NULL, ("%s: swp is NULL", __func__)); 315 316 DPAA2_SWP_LOCK(swp, &flags); 317 swp->flags |= DPAA2_SWP_DESTROYED; 318 DPAA2_SWP_UNLOCK(swp); 319 320 /* Let threads stop using this portal. */ 321 DELAY(DPAA2_SWP_TIMEOUT); 322 323 mtx_destroy(&swp->lock); 324 free(swp, M_DPAA2_SWP); 325 } 326 327 uint32_t 328 dpaa2_swp_set_cfg(uint8_t max_fill, uint8_t wn, uint8_t est, uint8_t rpm, 329 uint8_t dcm, uint8_t epm, int sd, int sp, int se, int dp, int de, int ep) 330 { 331 return ( 332 max_fill << DPAA2_SWP_CFG_DQRR_MF_SHIFT | 333 est << DPAA2_SWP_CFG_EST_SHIFT | 334 wn << DPAA2_SWP_CFG_WN_SHIFT | 335 rpm << DPAA2_SWP_CFG_RPM_SHIFT | 336 dcm << DPAA2_SWP_CFG_DCM_SHIFT | 337 epm << DPAA2_SWP_CFG_EPM_SHIFT | 338 sd << DPAA2_SWP_CFG_SD_SHIFT | 339 sp << DPAA2_SWP_CFG_SP_SHIFT | 340 se << DPAA2_SWP_CFG_SE_SHIFT | 341 dp << DPAA2_SWP_CFG_DP_SHIFT | 342 de << DPAA2_SWP_CFG_DE_SHIFT | 343 ep << DPAA2_SWP_CFG_EP_SHIFT 344 ); 345 } 346 347 /* Read/write registers of a software portal. */ 348 349 void 350 dpaa2_swp_write_reg(struct dpaa2_swp *swp, uint32_t o, uint32_t v) 351 { 352 bus_write_4(swp->cinh_map, o, v); 353 } 354 355 uint32_t 356 dpaa2_swp_read_reg(struct dpaa2_swp *swp, uint32_t o) 357 { 358 return (bus_read_4(swp->cinh_map, o)); 359 } 360 361 /* Helper routines. */ 362 363 /** 364 * @brief Set enqueue descriptor without Order Point Record ID. 365 * 366 * ed: Enqueue descriptor. 367 * resp_always: Enqueue with response always (1); FD from a rejected enqueue 368 * will be returned on a FQ (0). 369 */ 370 void 371 dpaa2_swp_set_ed_norp(struct dpaa2_eq_desc *ed, bool resp_always) 372 { 373 ed->verb &= ~(1 << ENQ_CMD_ORP_ENABLE_SHIFT); 374 if (resp_always) 375 ed->verb |= ENQ_CMD_RESPONSE_ALWAYS; 376 else 377 ed->verb |= ENQ_CMD_REJECTS_TO_FQ; 378 } 379 380 /** 381 * @brief Set FQ of the enqueue descriptor. 382 */ 383 void 384 dpaa2_swp_set_ed_fq(struct dpaa2_eq_desc *ed, uint32_t fqid) 385 { 386 ed->verb &= ~(1 << ENQ_CMD_TARGET_TYPE_SHIFT); 387 ed->tgtid = fqid; 388 } 389 390 /** 391 * @brief Enable interrupts for a software portal. 392 */ 393 void 394 dpaa2_swp_set_intr_trigger(struct dpaa2_swp *swp, uint32_t mask) 395 { 396 if (swp != NULL) 397 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_IER, mask); 398 } 399 400 /** 401 * @brief Return the value in the SWP_IER register. 402 */ 403 uint32_t 404 dpaa2_swp_get_intr_trigger(struct dpaa2_swp *swp) 405 { 406 if (swp != NULL) 407 return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_IER); 408 return (0); 409 } 410 411 /** 412 * @brief Return the value in the SWP_ISR register. 413 */ 414 uint32_t 415 dpaa2_swp_read_intr_status(struct dpaa2_swp *swp) 416 { 417 if (swp != NULL) 418 return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_ISR); 419 return (0); 420 } 421 422 /** 423 * @brief Clear SWP_ISR register according to the given mask. 424 */ 425 void 426 dpaa2_swp_clear_intr_status(struct dpaa2_swp *swp, uint32_t mask) 427 { 428 if (swp != NULL) 429 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ISR, mask); 430 } 431 432 /** 433 * @brief Enable or disable push dequeue. 434 * 435 * swp: the software portal object 436 * chan_idx: the channel index (0 to 15) 437 * en: enable or disable push dequeue 438 */ 439 void 440 dpaa2_swp_set_push_dequeue(struct dpaa2_swp *swp, uint8_t chan_idx, bool en) 441 { 442 uint16_t dqsrc; 443 444 if (swp != NULL) { 445 if (chan_idx > 15u) { 446 device_printf(swp->desc->dpio_dev, "channel index " 447 "should be <= 15: chan_idx=%d\n", chan_idx); 448 return; 449 } 450 451 if (en) 452 swp->sdq |= 1 << chan_idx; 453 else 454 swp->sdq &= ~(1 << chan_idx); 455 /* 456 * Read make the complete src map. If no channels are enabled 457 * the SDQCR must be 0 or else QMan will assert errors. 458 */ 459 dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) & 460 DPAA2_SDQCR_SRC_MASK; 461 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_SDQCR, dqsrc != 0 462 ? swp->sdq : 0); 463 } 464 } 465 466 /** 467 * @brief Set new IRQ coalescing values. 468 * 469 * swp: The software portal object. 470 * threshold: Threshold for DQRR interrupt generation. The DQRR interrupt 471 * asserts when the ring contains greater than "threshold" entries. 472 * holdoff: DQRR interrupt holdoff (timeout) period in us. 473 */ 474 int dpaa2_swp_set_irq_coalescing(struct dpaa2_swp *swp, uint32_t threshold, 475 uint32_t holdoff) 476 { 477 uint32_t itp; /* Interrupt Timeout Period */ 478 479 if (swp == NULL) 480 return (EINVAL); 481 482 /* 483 * Convert "holdoff" value from us to 256 QBMAN clock cycles 484 * increments. This depends on the QBMAN internal frequency. 485 */ 486 itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio; 487 if (itp > DQRR_MAX_ITP) 488 itp = DQRR_MAX_ITP; 489 if (threshold >= swp->dqrr.ring_size) 490 threshold = swp->dqrr.ring_size - 1; 491 492 swp->dqrr.irq_threshold = threshold; 493 swp->dqrr.irq_itp = itp; 494 495 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_DQRR_ITR, threshold); 496 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ITPR, itp); 497 498 return (0); 499 } 500 501 /* 502 * Software portal commands. 503 */ 504 505 /** 506 * @brief Configure the channel data availability notification (CDAN) 507 * in a particular WQ channel. 508 */ 509 int 510 dpaa2_swp_conf_wq_channel(struct dpaa2_swp *swp, uint16_t chan_id, 511 uint8_t we_mask, bool cdan_en, uint64_t ctx) 512 { 513 /* NOTE: 64 bytes command. */ 514 struct __packed { 515 uint8_t verb; 516 uint8_t result; /* in response only! */ 517 uint16_t chan_id; 518 uint8_t we; 519 uint8_t ctrl; 520 uint16_t _reserved2; 521 uint64_t ctx; 522 uint8_t _reserved3[48]; 523 } cmd = {0}; 524 struct __packed { 525 uint8_t verb; 526 uint8_t result; 527 uint16_t chan_id; 528 uint8_t _reserved[60]; 529 } rsp; 530 int error; 531 532 if (swp == NULL) 533 return (EINVAL); 534 535 cmd.chan_id = chan_id; 536 cmd.we = we_mask; 537 cmd.ctrl = cdan_en ? 1u : 0u; 538 cmd.ctx = ctx; 539 540 error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd, 541 (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_WQCHAN_CONFIGURE); 542 if (error) 543 return (error); 544 545 if (rsp.result != QBMAN_CMD_RC_OK) { 546 device_printf(swp->desc->dpio_dev, "WQ channel configuration " 547 "error: channel_id=%d, result=0x%02x\n", chan_id, 548 rsp.result); 549 return (EIO); 550 } 551 552 return (0); 553 } 554 555 /** 556 * @brief Query current configuration/state of the buffer pool. 557 */ 558 int 559 dpaa2_swp_query_bp(struct dpaa2_swp *swp, uint16_t bpid, 560 struct dpaa2_bp_conf *conf) 561 { 562 /* NOTE: 64 bytes command. */ 563 struct __packed { 564 uint8_t verb; 565 uint8_t _reserved1; 566 uint16_t bpid; 567 uint8_t _reserved2[60]; 568 } cmd = {0}; 569 struct __packed { 570 uint8_t verb; 571 uint8_t result; 572 uint32_t _reserved1; 573 uint8_t bdi; 574 uint8_t state; 575 uint32_t fill; 576 /* TODO: Support the other fields as well. */ 577 uint8_t _reserved2[52]; 578 } rsp; 579 int error; 580 581 if (swp == NULL || conf == NULL) 582 return (EINVAL); 583 584 cmd.bpid = bpid; 585 586 error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd, 587 (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_BP_QUERY); 588 if (error) 589 return (error); 590 591 if (rsp.result != QBMAN_CMD_RC_OK) { 592 device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, " 593 "result=0x%02x\n", bpid, rsp.result); 594 return (EIO); 595 } 596 597 conf->bdi = rsp.bdi; 598 conf->state = rsp.state; 599 conf->free_bufn = rsp.fill; 600 601 return (0); 602 } 603 604 int 605 dpaa2_swp_release_bufs(struct dpaa2_swp *swp, uint16_t bpid, bus_addr_t *buf, 606 uint32_t buf_num) 607 { 608 /* NOTE: 64 bytes command. */ 609 struct __packed { 610 uint8_t verb; 611 uint8_t _reserved1; 612 uint16_t bpid; 613 uint32_t _reserved2; 614 uint64_t buf[DPAA2_SWP_BUFS_PER_CMD]; 615 } cmd = {0}; 616 int error; 617 618 if (swp == NULL || buf == NULL || buf_num == 0u || 619 buf_num > DPAA2_SWP_BUFS_PER_CMD) 620 return (EINVAL); 621 622 for (uint32_t i = 0; i < buf_num; i++) 623 cmd.buf[i] = buf[i]; 624 cmd.bpid = bpid; 625 cmd.verb |= 1 << 5; /* Switch release buffer command to valid. */ 626 627 error = dpaa2_swp_exec_br_command(swp, (struct dpaa2_swp_cmd *) &cmd, 628 buf_num); 629 if (error) { 630 device_printf(swp->desc->dpio_dev, "buffers release command " 631 "failed\n"); 632 return (error); 633 } 634 635 return (0); 636 } 637 638 int 639 dpaa2_swp_dqrr_next_locked(struct dpaa2_swp *swp, struct dpaa2_dq *dq, 640 uint32_t *idx) 641 { 642 struct resource_map *map = swp->cinh_map; 643 struct dpaa2_swp_rsp *rsp = (struct dpaa2_swp_rsp *) dq; 644 uint32_t verb, pi; /* producer index */ 645 uint32_t offset = swp->cfg.mem_backed 646 ? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx) 647 : DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx); 648 649 if (swp == NULL || dq == NULL) 650 return (EINVAL); 651 652 /* 653 * Before using valid-bit to detect if something is there, we have to 654 * handle the case of the DQRR reset bug... 655 */ 656 if (swp->dqrr.reset_bug) { 657 /* 658 * We pick up new entries by cache-inhibited producer index, 659 * which means that a non-coherent mapping would require us to 660 * invalidate and read *only* once that PI has indicated that 661 * there's an entry here. The first trip around the DQRR ring 662 * will be much less efficient than all subsequent trips around 663 * it... 664 */ 665 pi = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_DQPI) & DQRR_PI_MASK; 666 667 /* There are new entries if pi != next_idx */ 668 if (pi == swp->dqrr.next_idx) 669 return (ENOENT); 670 671 /* 672 * If next_idx is/was the last ring index, and 'pi' is 673 * different, we can disable the workaround as all the ring 674 * entries have now been DMA'd to so valid-bit checking is 675 * repaired. 676 * 677 * NOTE: This logic needs to be based on next_idx (which 678 * increments one at a time), rather than on pi (which 679 * can burst and wrap-around between our snapshots of it). 680 */ 681 if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1)) 682 swp->dqrr.reset_bug = 0; 683 } 684 685 verb = bus_read_4(map, offset); 686 if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit) 687 return (ENOENT); 688 689 /* Read dequeue response message. */ 690 for (int i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++) 691 rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t)); 692 693 /* Return index of the current entry (if requested). */ 694 if (idx != NULL) 695 *idx = swp->dqrr.next_idx; 696 697 /* 698 * There's something there. Move "next_idx" attention to the next ring 699 * entry before returning what we found. 700 */ 701 swp->dqrr.next_idx++; 702 swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */ 703 if (swp->dqrr.next_idx == 0u) 704 swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT; 705 706 return (0); 707 } 708 709 int 710 dpaa2_swp_pull(struct dpaa2_swp *swp, uint16_t chan_id, struct dpaa2_buf *buf, 711 uint32_t frames_n) 712 { 713 /* NOTE: 64 bytes command. */ 714 struct __packed { 715 uint8_t verb; 716 uint8_t numf; 717 uint8_t tok; 718 uint8_t _reserved; 719 uint32_t dq_src; 720 uint64_t rsp_addr; 721 uint64_t _reserved1[6]; 722 } cmd = {0}; 723 struct dpaa2_dq *msg; 724 uint16_t flags; 725 int i, error; 726 727 KASSERT(swp != NULL, ("%s: swp is NULL", __func__)); 728 KASSERT(frames_n != 0u, ("%s: cannot pull zero frames", __func__)); 729 KASSERT(frames_n <= 16u, ("%s: too much frames to pull", __func__)); 730 KASSERT(buf->type == DPAA2_BUF_STORE, ("%s: not channel storage " 731 "buffer", __func__)); 732 733 cmd.numf = frames_n - 1; 734 cmd.tok = DPAA2_SWP_VDQCR_TOKEN; 735 cmd.dq_src = chan_id; 736 cmd.rsp_addr = (uint64_t) buf->store.paddr; 737 738 /* Dequeue command type */ 739 cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT); 740 cmd.verb |= (1 << QB_VDQCR_VERB_DCT1_SHIFT); 741 /* Dequeue from a specific software portal channel (ID's in DQ_SRC). */ 742 cmd.verb &= ~(1 << QB_VDQCR_VERB_DT0_SHIFT); 743 cmd.verb &= ~(1 << QB_VDQCR_VERB_DT1_SHIFT); 744 /* Write the response to this command into memory (at the RSP_ADDR). */ 745 cmd.verb |= (1 << QB_VDQCR_VERB_RLS_SHIFT); 746 /* Response writes won't attempt to allocate into a cache. */ 747 cmd.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); 748 /* Allow the FQ to remain active in the portal after dequeue. */ 749 cmd.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT); 750 751 DPAA2_SWP_LOCK(swp, &flags); 752 if (flags & DPAA2_SWP_DESTROYED) { 753 /* Terminate operation if portal is destroyed. */ 754 DPAA2_SWP_UNLOCK(swp); 755 return (ENOENT); 756 } 757 758 error = dpaa2_swp_exec_vdc_command_locked(swp, 759 (struct dpaa2_swp_cmd *) &cmd); 760 if (error != 0) { 761 DPAA2_SWP_UNLOCK(swp); 762 return (error); 763 } 764 765 /* Let's sync before reading VDQ response from QBMan. */ 766 bus_dmamap_sync(buf->store.dmat, buf->store.dmap, BUS_DMASYNC_POSTREAD); 767 768 /* Read VDQ response from QBMan. */ 769 msg = (struct dpaa2_dq *) buf->store.vaddr; 770 for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) { 771 if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) && 772 (msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) { 773 /* Reset token. */ 774 msg->fdr.desc.tok = 0; 775 break; 776 } 777 DELAY(CMD_SPIN_TIMEOUT); 778 } 779 DPAA2_SWP_UNLOCK(swp); 780 781 /* Return an error on expired timeout. */ 782 return (i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0); 783 } 784 785 /** 786 * @brief Issue a command to enqueue a frame using one enqueue descriptor. 787 * 788 * swp: Software portal used to send this command to. 789 * ed: Enqueue command descriptor. 790 * fd: Frame descriptor to enqueue. 791 */ 792 int 793 dpaa2_swp_enq(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed, 794 struct dpaa2_fd *fd) 795 { 796 uint32_t flags = 0; 797 int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1); 798 799 return (rc >= 0 ? 0 : EBUSY); 800 } 801 802 /** 803 * @brief Issue a command to enqueue frames using one enqueue descriptor. 804 * 805 * swp: Software portal used to send this command to. 806 * ed: Enqueue command descriptor. 807 * fd: Frame descriptor to enqueue. 808 * flags: Table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL. 809 * frames_n: Number of FDs to enqueue. 810 * 811 * NOTE: Enqueue command (64 bytes): 32 (eq. descriptor) + 32 (frame descriptor). 812 */ 813 int 814 dpaa2_swp_enq_mult(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed, 815 struct dpaa2_fd *fd, uint32_t *flags, int frames_n) 816 { 817 const uint8_t *ed_pdat8 = (const uint8_t *) ed; 818 const uint32_t *ed_pdat32 = (const uint32_t *) ed; 819 const uint64_t *ed_pdat64 = (const uint64_t *) ed; 820 const uint64_t *fd_pdat64 = (const uint64_t *) fd; 821 struct resource_map *map; 822 uint32_t eqcr_ci, eqcr_pi; /* EQCR consumer/producer index */ 823 uint32_t half_mask, full_mask, val, ci_offset; 824 uint16_t swp_flags; 825 int num_enq = 0; 826 827 if (swp == NULL || ed == NULL || fd == NULL || flags == NULL || 828 frames_n == 0) 829 return (EINVAL); 830 831 DPAA2_SWP_LOCK(swp, &swp_flags); 832 if (swp_flags & DPAA2_SWP_DESTROYED) { 833 /* Terminate operation if portal is destroyed. */ 834 DPAA2_SWP_UNLOCK(swp); 835 return (ENOENT); 836 } 837 838 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; 839 ci_offset = swp->cfg.mem_backed 840 ? DPAA2_SWP_CENA_EQCR_CI_MEMBACK 841 : DPAA2_SWP_CENA_EQCR_CI; 842 843 half_mask = swp->eqcr.pi_ci_mask >> 1; 844 full_mask = swp->eqcr.pi_ci_mask; 845 846 if (swp->eqcr.available == 0) { 847 val = dpaa2_swp_read_reg(swp, ci_offset); 848 eqcr_ci = swp->eqcr.ci; 849 swp->eqcr.ci = val & full_mask; 850 851 swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size, 852 eqcr_ci, swp->eqcr.ci); 853 854 if (swp->eqcr.available == 0) { 855 DPAA2_SWP_UNLOCK(swp); 856 return (0); 857 } 858 } 859 860 eqcr_pi = swp->eqcr.pi; 861 num_enq = swp->eqcr.available < frames_n 862 ? swp->eqcr.available : frames_n; 863 swp->eqcr.available -= num_enq; 864 865 KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size, 866 ("%s: unexpected num_enq=%d", __func__, num_enq)); 867 KASSERT(swp->eqcr.available >= 0 && 868 swp->eqcr.available <= swp->eqcr.pi_ring_size, 869 ("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available)); 870 871 /* Fill in the EQCR ring. */ 872 for (int i = 0; i < num_enq; i++) { 873 /* Write enq. desc. without the VERB, DCA, SEQNUM and OPRID. */ 874 for (int j = 1; j <= 3; j++) 875 bus_write_8(map, 876 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 877 sizeof(uint64_t) * j, ed_pdat64[j]); 878 /* Write OPRID. */ 879 bus_write_4(map, 880 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + sizeof(uint32_t), 881 ed_pdat32[1]); 882 /* Write DCA and SEQNUM without VERB byte. */ 883 for (int j = 1; j <= 3; j++) 884 bus_write_1(map, 885 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 886 sizeof(uint8_t) * j, ed_pdat8[j]); 887 888 /* Write frame descriptor. */ 889 for (int j = 0; j <= 3; j++) 890 bus_write_8(map, 891 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 892 ENQ_DESC_FD_OFFSET + 893 sizeof(uint64_t) * j, fd_pdat64[j]); 894 eqcr_pi++; 895 } 896 897 wmb(); 898 899 /* Write the VERB byte of enqueue descriptor. */ 900 eqcr_pi = swp->eqcr.pi; 901 for (int i = 0; i < num_enq; i++) { 902 bus_write_1(map, 903 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask), 904 ed_pdat8[0] | swp->eqcr.pi_vb); 905 906 if (flags && (flags[i] & ENQ_FLAG_DCA)) { 907 /* Update DCA byte. */ 908 bus_write_1(map, 909 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 1, 910 (1 << ENQ_CMD_DCA_EN_SHIFT) | 911 (flags[i] & ENQ_DCA_IDXMASK)); 912 } 913 eqcr_pi++; 914 if (!(eqcr_pi & half_mask)) 915 swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT; 916 } 917 swp->eqcr.pi = eqcr_pi & full_mask; 918 919 DPAA2_SWP_UNLOCK(swp); 920 921 return (num_enq); 922 } 923 924 static int 925 dpaa2_swp_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last) 926 { 927 /* 'first' is included, 'last' is excluded */ 928 return ((first <= last) 929 ? (last - first) : ((2 * ringsize) - (first - last))); 930 } 931 932 /** 933 * @brief Execute Buffer Release Command (BRC). 934 */ 935 static int 936 dpaa2_swp_exec_br_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd, 937 uint32_t buf_num) 938 { 939 struct __packed with_verb { 940 uint8_t verb; 941 uint8_t _reserved[63]; 942 } *c; 943 const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params; 944 const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params; 945 struct resource_map *map; 946 uint32_t offset, rar; /* Release Array Allocation register */ 947 uint16_t flags; 948 949 if (!swp || !cmd) 950 return (EINVAL); 951 952 DPAA2_SWP_LOCK(swp, &flags); 953 if (flags & DPAA2_SWP_DESTROYED) { 954 /* Terminate operation if portal is destroyed. */ 955 DPAA2_SWP_UNLOCK(swp); 956 return (ENOENT); 957 } 958 959 rar = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_RAR); 960 if (!RAR_SUCCESS(rar)) { 961 DPAA2_SWP_UNLOCK(swp); 962 return (EBUSY); 963 } 964 965 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; 966 offset = swp->cfg.mem_backed 967 ? DPAA2_SWP_CENA_RCR_MEM(RAR_IDX(rar)) 968 : DPAA2_SWP_CENA_RCR(RAR_IDX(rar)); 969 c = (struct with_verb *) cmd; 970 971 /* Write command bytes (without VERB byte). */ 972 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++) 973 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]); 974 bus_write_4(map, offset + 4, cmd_pdat32[1]); 975 for (uint32_t i = 1; i <= 3; i++) 976 bus_write_1(map, offset + i, cmd_pdat8[i]); 977 978 /* Write VERB byte and trigger command execution. */ 979 if (swp->cfg.mem_backed) { 980 bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num); 981 wmb(); 982 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_RCR_AM_RT + 983 RAR_IDX(rar) * 4, DPAA2_SWP_RT_MODE); 984 } else { 985 wmb(); 986 bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num); 987 } 988 989 DPAA2_SWP_UNLOCK(swp); 990 991 return (0); 992 } 993 994 /** 995 * @brief Execute Volatile Dequeue Command (VDC). 996 * 997 * This command will be executed by QBMan only once in order to deliver requested 998 * number of frames (1-16 or 1-32 depending on QBMan version) to the driver via 999 * DQRR or arbitrary DMA-mapped memory. 1000 * 1001 * NOTE: There is a counterpart to the volatile dequeue command called static 1002 * dequeue command (SDQC) which is executed periodically all the time the 1003 * command is present in the SDQCR register. 1004 */ 1005 static int 1006 dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *swp, 1007 struct dpaa2_swp_cmd *cmd) 1008 { 1009 struct __packed with_verb { 1010 uint8_t verb; 1011 uint8_t _reserved[63]; 1012 } *c; 1013 const uint8_t *p8 = (const uint8_t *) cmd->params; 1014 const uint32_t *p32 = (const uint32_t *) cmd->params; 1015 struct resource_map *map; 1016 uint32_t offset; 1017 1018 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; 1019 offset = swp->cfg.mem_backed 1020 ? DPAA2_SWP_CENA_VDQCR_MEM : DPAA2_SWP_CENA_VDQCR; 1021 c = (struct with_verb *) cmd; 1022 1023 /* Write command bytes (without VERB byte). */ 1024 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++) 1025 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]); 1026 bus_write_4(map, offset + 4, p32[1]); 1027 for (uint32_t i = 1; i <= 3; i++) 1028 bus_write_1(map, offset + i, p8[i]); 1029 1030 /* Write VERB byte and trigger command execution. */ 1031 if (swp->cfg.mem_backed) { 1032 bus_write_1(map, offset, c->verb | swp->vdq.valid_bit); 1033 swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT; 1034 wmb(); 1035 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_VDQCR_RT, 1036 DPAA2_SWP_RT_MODE); 1037 } else { 1038 wmb(); 1039 bus_write_1(map, offset, c->verb | swp->vdq.valid_bit); 1040 swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT; 1041 } 1042 1043 return (0); 1044 } 1045 1046 /** 1047 * @brief Execute a QBMan management command. 1048 */ 1049 static int 1050 dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd, 1051 struct dpaa2_swp_rsp *rsp, uint8_t cmdid) 1052 { 1053 #if (defined(_KERNEL) && defined(INVARIANTS)) 1054 struct __packed with_verb { 1055 uint8_t verb; 1056 uint8_t _reserved[63]; 1057 } *r; 1058 #endif 1059 uint16_t flags; 1060 int error; 1061 1062 if (swp == NULL || cmd == NULL || rsp == NULL) 1063 return (EINVAL); 1064 1065 DPAA2_SWP_LOCK(swp, &flags); 1066 if (flags & DPAA2_SWP_DESTROYED) { 1067 /* Terminate operation if portal is destroyed. */ 1068 DPAA2_SWP_UNLOCK(swp); 1069 return (ENOENT); 1070 } 1071 1072 /* 1073 * Send a command to QBMan using Management Command register and wait 1074 * for response from the Management Response registers. 1075 */ 1076 dpaa2_swp_send_mgmt_command(swp, cmd, cmdid); 1077 error = dpaa2_swp_wait_for_mgmt_response(swp, rsp); 1078 if (error) { 1079 DPAA2_SWP_UNLOCK(swp); 1080 return (error); 1081 } 1082 DPAA2_SWP_UNLOCK(swp); 1083 1084 #if (defined(_KERNEL) && defined(INVARIANTS)) 1085 r = (struct with_verb *) rsp; 1086 KASSERT((r->verb & CMD_VERB_MASK) == cmdid, 1087 ("wrong VERB byte in response: resp=0x%02x, expected=0x%02x", 1088 r->verb, cmdid)); 1089 #endif 1090 1091 return (0); 1092 } 1093 1094 static int 1095 dpaa2_swp_send_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd, 1096 uint8_t cmdid) 1097 { 1098 const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params; 1099 const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params; 1100 struct resource_map *map; 1101 uint32_t offset; 1102 1103 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; 1104 offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR; 1105 1106 /* Write command bytes (without VERB byte). */ 1107 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++) 1108 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]); 1109 bus_write_4(map, offset + 4, cmd_pdat32[1]); 1110 for (uint32_t i = 1; i <= 3; i++) 1111 bus_write_1(map, offset + i, cmd_pdat8[i]); 1112 1113 /* Write VERB byte and trigger command execution. */ 1114 if (swp->cfg.mem_backed) { 1115 bus_write_1(map, offset, cmdid | swp->mr.valid_bit); 1116 wmb(); 1117 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_CR_RT, 1118 DPAA2_SWP_RT_MODE); 1119 } else { 1120 wmb(); 1121 bus_write_1(map, offset, cmdid | swp->mc.valid_bit); 1122 } 1123 1124 return (0); 1125 } 1126 1127 static int 1128 dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *swp, struct dpaa2_swp_rsp *rsp) 1129 { 1130 struct resource_map *map = swp->cfg.mem_backed 1131 ? swp->cena_map : swp->cinh_map; 1132 /* Management command response to be read from the only RR or RR0/RR1. */ 1133 const uint32_t offset = swp->cfg.mem_backed 1134 ? DPAA2_SWP_CENA_RR_MEM 1135 : DPAA2_SWP_CENA_RR(swp->mc.valid_bit); 1136 uint32_t i, verb, ret; 1137 int rc; 1138 1139 /* Wait for a command response from QBMan. */ 1140 for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) { 1141 if (swp->cfg.mem_backed) { 1142 verb = (uint32_t) (bus_read_4(map, offset) & 0xFFu); 1143 if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT)) 1144 goto wait; 1145 if (!(verb & ~DPAA2_SWP_VALID_BIT)) 1146 goto wait; 1147 swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT; 1148 } else { 1149 ret = bus_read_4(map, offset); 1150 verb = ret & ~DPAA2_SWP_VALID_BIT; /* remove valid bit */ 1151 if (verb == 0u) 1152 goto wait; 1153 swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT; 1154 } 1155 break; 1156 wait: 1157 DELAY(CMD_SPIN_TIMEOUT); 1158 } 1159 /* Return an error on expired timeout. */ 1160 rc = i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0; 1161 1162 /* Read command response. */ 1163 for (i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++) 1164 rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t)); 1165 1166 return (rc); 1167 } 1168