1 /****************************************************************************** 2 3 � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc. 4 All rights reserved. 5 6 This is proprietary source code of Freescale Semiconductor Inc., 7 and its use is subject to the NetComm Device Drivers EULA. 8 The copyright notice above does not evidence any actual or intended 9 publication of such source code. 10 11 ALTERNATIVELY, redistribution and use in source and binary forms, with 12 or without modification, are permitted provided that the following 13 conditions are met: 14 * Redistributions of source code must retain the above copyright 15 notice, this list of conditions and the following disclaimer. 16 * Redistributions in binary form must reproduce the above copyright 17 notice, this list of conditions and the following disclaimer in the 18 documentation and/or other materials provided with the distribution. 19 * Neither the name of Freescale Semiconductor nor the 20 names of its contributors may be used to endorse or promote products 21 derived from this software without specific prior written permission. 22 23 THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY 24 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 25 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 26 DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY 27 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 35 **************************************************************************/ 36 /****************************************************************************** 37 @File bman_low.c 38 39 @Description BM low-level implementation 40 *//***************************************************************************/ 41 #include <sys/cdefs.h> 42 #include <sys/types.h> 43 #include <machine/atomic.h> 44 45 #include "std_ext.h" 46 #include "core_ext.h" 47 #include "xx_ext.h" 48 #include "error_ext.h" 49 50 #include "bman_private.h" 51 52 53 /***************************/ 54 /* Portal register assists */ 55 /***************************/ 56 57 /* Cache-inhibited register offsets */ 58 #define REG_RCR_PI_CINH 0x0000 59 #define REG_RCR_CI_CINH 0x0004 60 #define REG_RCR_ITR 0x0008 61 #define REG_CFG 0x0100 62 #define REG_SCN(n) (0x0200 + ((n) << 2)) 63 #define REG_ISR 0x0e00 64 #define REG_IER 0x0e04 65 #define REG_ISDR 0x0e08 66 #define REG_IIR 0x0e0c 67 68 /* Cache-enabled register offsets */ 69 #define CL_CR 0x0000 70 #define CL_RR0 0x0100 71 #define CL_RR1 0x0140 72 #define CL_RCR 0x1000 73 #define CL_RCR_PI_CENA 0x3000 74 #define CL_RCR_CI_CENA 0x3100 75 76 /* The h/w design requires mappings to be size-aligned so that "add"s can be 77 * reduced to "or"s. The primitives below do the same for s/w. */ 78 79 static __inline__ void *ptr_ADD(void *a, uintptr_t b) 80 { 81 return (void *)((uintptr_t)a + b); 82 } 83 84 /* Bitwise-OR two pointers */ 85 static __inline__ void *ptr_OR(void *a, uintptr_t b) 86 { 87 return (void *)((uintptr_t)a | b); 88 } 89 90 /* Cache-inhibited register access */ 91 static __inline__ uint32_t __bm_in(struct bm_addr *bm, uintptr_t offset) 92 { 93 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset); 94 return GET_UINT32(*tmp); 95 } 96 static __inline__ void __bm_out(struct bm_addr *bm, uintptr_t offset, uint32_t val) 97 { 98 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset); 99 WRITE_UINT32(*tmp, val); 100 } 101 #define bm_in(reg) __bm_in(&portal->addr, REG_##reg) 102 #define bm_out(reg, val) __bm_out(&portal->addr, REG_##reg, val) 103 104 /* Convert 'n' cachelines to a pointer value for bitwise OR */ 105 #define bm_cl(n) (void *)((n) << 6) 106 107 /* Cache-enabled (index) register access */ 108 static __inline__ void __bm_cl_touch_ro(struct bm_addr *bm, uintptr_t offset) 109 { 110 dcbt_ro(ptr_ADD(bm->addr_ce, offset)); 111 } 112 static __inline__ void __bm_cl_touch_rw(struct bm_addr *bm, uintptr_t offset) 113 { 114 dcbt_rw(ptr_ADD(bm->addr_ce, offset)); 115 } 116 static __inline__ uint32_t __bm_cl_in(struct bm_addr *bm, uintptr_t offset) 117 { 118 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset); 119 return GET_UINT32(*tmp); 120 } 121 static __inline__ void __bm_cl_out(struct bm_addr *bm, uintptr_t offset, uint32_t val) 122 { 123 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset); 124 WRITE_UINT32(*tmp, val); 125 dcbf(tmp); 126 } 127 static __inline__ void __bm_cl_invalidate(struct bm_addr *bm, uintptr_t offset) 128 { 129 dcbi(ptr_ADD(bm->addr_ce, offset)); 130 } 131 #define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, CL_##reg##_CENA) 132 #define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, CL_##reg##_CENA) 133 #define bm_cl_in(reg) __bm_cl_in(&portal->addr, CL_##reg##_CENA) 134 #define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, CL_##reg##_CENA, val) 135 #define bm_cl_invalidate(reg) __bm_cl_invalidate(&portal->addr, CL_##reg##_CENA) 136 137 /* Cyclic helper for rings. TODO: once we are able to do fine-grain perf 138 * analysis, look at using the "extra" bit in the ring index registers to avoid 139 * cyclic issues. */ 140 static __inline__ uint8_t cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last) 141 { 142 /* 'first' is included, 'last' is excluded */ 143 if (first <= last) 144 return (uint8_t)(last - first); 145 return (uint8_t)(ringsize + last - first); 146 } 147 148 /* --------------- */ 149 /* --- RCR API --- */ 150 151 /* It's safer to code in terms of the 'rcr' object than the 'portal' object, 152 * because the latter runs the risk of copy-n-paste errors from other code where 153 * we could manipulate some other structure within 'portal'. */ 154 /* #define RCR_API_START() register struct bm_rcr *rcr = &portal->rcr */ 155 156 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ 157 #define RCR_CARRYCLEAR(p) \ 158 (void *)((uintptr_t)(p) & (~(uintptr_t)(BM_RCR_SIZE << 6))) 159 160 /* Bit-wise logic to convert a ring pointer to a ring index */ 161 static __inline__ uint8_t RCR_PTR2IDX(struct bm_rcr_entry *e) 162 { 163 return (uint8_t)(((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1)); 164 } 165 166 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */ 167 static __inline__ void RCR_INC(struct bm_rcr *rcr) 168 { 169 /* NB: this is odd-looking, but experiments show that it generates 170 * fast code with essentially no branching overheads. We increment to 171 * the next RCR pointer and handle overflow and 'vbit'. */ 172 struct bm_rcr_entry *partial = rcr->cursor + 1; 173 rcr->cursor = RCR_CARRYCLEAR(partial); 174 if (partial != rcr->cursor) 175 rcr->vbit ^= BM_RCR_VERB_VBIT; 176 } 177 178 t_Error bm_rcr_init(struct bm_portal *portal, 179 e_BmPortalProduceMode pmode, 180 e_BmPortalRcrConsumeMode cmode) 181 { 182 register struct bm_rcr *rcr = &portal->rcr; 183 uint32_t cfg; 184 uint8_t pi; 185 186 rcr->ring = ptr_ADD(portal->addr.addr_ce, CL_RCR); 187 rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1)); 188 pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1)); 189 rcr->cursor = rcr->ring + pi; 190 rcr->vbit = (uint8_t)((bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0); 191 rcr->available = (uint8_t)(BM_RCR_SIZE - 1 - cyc_diff(BM_RCR_SIZE, rcr->ci, pi)); 192 rcr->ithresh = (uint8_t)bm_in(RCR_ITR); 193 #ifdef BM_CHECKING 194 rcr->busy = 0; 195 rcr->pmode = pmode; 196 rcr->cmode = cmode; 197 #else 198 UNUSED(cmode); 199 #endif /* BM_CHECKING */ 200 cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */ 201 bm_out(CFG, cfg); 202 return 0; 203 } 204 205 void bm_rcr_finish(struct bm_portal *portal) 206 { 207 register struct bm_rcr *rcr = &portal->rcr; 208 uint8_t pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1)); 209 uint8_t ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1)); 210 ASSERT_COND(!rcr->busy); 211 if (pi != RCR_PTR2IDX(rcr->cursor)) 212 REPORT_ERROR(WARNING, E_INVALID_STATE, ("losing uncommitted RCR entries")); 213 if (ci != rcr->ci) 214 REPORT_ERROR(WARNING, E_INVALID_STATE, ("missing existing RCR completions")); 215 if (rcr->ci != RCR_PTR2IDX(rcr->cursor)) 216 REPORT_ERROR(WARNING, E_INVALID_STATE, ("RCR destroyed unquiesced")); 217 } 218 219 struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal) 220 { 221 register struct bm_rcr *rcr = &portal->rcr; 222 ASSERT_COND(!rcr->busy); 223 if (!rcr->available) 224 return NULL; 225 #ifdef BM_CHECKING 226 rcr->busy = 1; 227 #endif /* BM_CHECKING */ 228 dcbz_64(rcr->cursor); 229 return rcr->cursor; 230 } 231 232 void bm_rcr_abort(struct bm_portal *portal) 233 { 234 register struct bm_rcr *rcr = &portal->rcr; 235 ASSERT_COND(rcr->busy); 236 #ifdef BM_CHECKING 237 rcr->busy = 0; 238 #else 239 UNUSED(rcr); 240 #endif /* BM_CHECKING */ 241 } 242 243 struct bm_rcr_entry *bm_rcr_pend_and_next(struct bm_portal *portal, uint8_t myverb) 244 { 245 register struct bm_rcr *rcr = &portal->rcr; 246 ASSERT_COND(rcr->busy); 247 ASSERT_COND(rcr->pmode != e_BmPortalPVB); 248 if (rcr->available == 1) 249 return NULL; 250 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); 251 dcbf_64(rcr->cursor); 252 RCR_INC(rcr); 253 rcr->available--; 254 dcbz_64(rcr->cursor); 255 return rcr->cursor; 256 } 257 258 void bm_rcr_pci_commit(struct bm_portal *portal, uint8_t myverb) 259 { 260 register struct bm_rcr *rcr = &portal->rcr; 261 ASSERT_COND(rcr->busy); 262 ASSERT_COND(rcr->pmode == e_BmPortalPCI); 263 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); 264 RCR_INC(rcr); 265 rcr->available--; 266 mb(); 267 bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor)); 268 #ifdef BM_CHECKING 269 rcr->busy = 0; 270 #endif /* BM_CHECKING */ 271 } 272 273 void bm_rcr_pce_prefetch(struct bm_portal *portal) 274 { 275 ASSERT_COND(((struct bm_rcr *)&portal->rcr)->pmode == e_BmPortalPCE); 276 bm_cl_invalidate(RCR_PI); 277 bm_cl_touch_rw(RCR_PI); 278 } 279 280 void bm_rcr_pce_commit(struct bm_portal *portal, uint8_t myverb) 281 { 282 register struct bm_rcr *rcr = &portal->rcr; 283 ASSERT_COND(rcr->busy); 284 ASSERT_COND(rcr->pmode == e_BmPortalPCE); 285 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); 286 RCR_INC(rcr); 287 rcr->available--; 288 wmb(); 289 bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor)); 290 #ifdef BM_CHECKING 291 rcr->busy = 0; 292 #endif /* BM_CHECKING */ 293 } 294 295 void bm_rcr_pvb_commit(struct bm_portal *portal, uint8_t myverb) 296 { 297 register struct bm_rcr *rcr = &portal->rcr; 298 struct bm_rcr_entry *rcursor; 299 ASSERT_COND(rcr->busy); 300 ASSERT_COND(rcr->pmode == e_BmPortalPVB); 301 rmb(); 302 rcursor = rcr->cursor; 303 rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); 304 dcbf_64(rcursor); 305 RCR_INC(rcr); 306 rcr->available--; 307 #ifdef BM_CHECKING 308 rcr->busy = 0; 309 #endif /* BM_CHECKING */ 310 } 311 312 313 uint8_t bm_rcr_cci_update(struct bm_portal *portal) 314 { 315 register struct bm_rcr *rcr = &portal->rcr; 316 uint8_t diff, old_ci = rcr->ci; 317 ASSERT_COND(rcr->cmode == e_BmPortalRcrCCI); 318 rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1)); 319 diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); 320 rcr->available += diff; 321 return diff; 322 } 323 324 325 void bm_rcr_cce_prefetch(struct bm_portal *portal) 326 { 327 ASSERT_COND(((struct bm_rcr *)&portal->rcr)->cmode == e_BmPortalRcrCCE); 328 bm_cl_touch_ro(RCR_CI); 329 } 330 331 332 uint8_t bm_rcr_cce_update(struct bm_portal *portal) 333 { 334 register struct bm_rcr *rcr = &portal->rcr; 335 uint8_t diff, old_ci = rcr->ci; 336 ASSERT_COND(rcr->cmode == e_BmPortalRcrCCE); 337 rcr->ci = (uint8_t)(bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1)); 338 bm_cl_invalidate(RCR_CI); 339 diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); 340 rcr->available += diff; 341 return diff; 342 } 343 344 345 uint8_t bm_rcr_get_ithresh(struct bm_portal *portal) 346 { 347 register struct bm_rcr *rcr = &portal->rcr; 348 return rcr->ithresh; 349 } 350 351 352 void bm_rcr_set_ithresh(struct bm_portal *portal, uint8_t ithresh) 353 { 354 register struct bm_rcr *rcr = &portal->rcr; 355 rcr->ithresh = ithresh; 356 bm_out(RCR_ITR, ithresh); 357 } 358 359 360 uint8_t bm_rcr_get_avail(struct bm_portal *portal) 361 { 362 register struct bm_rcr *rcr = &portal->rcr; 363 return rcr->available; 364 } 365 366 367 uint8_t bm_rcr_get_fill(struct bm_portal *portal) 368 { 369 register struct bm_rcr *rcr = &portal->rcr; 370 return (uint8_t)(BM_RCR_SIZE - 1 - rcr->available); 371 } 372 373 374 /* ------------------------------ */ 375 /* --- Management command API --- */ 376 377 /* It's safer to code in terms of the 'mc' object than the 'portal' object, 378 * because the latter runs the risk of copy-n-paste errors from other code where 379 * we could manipulate some other structure within 'portal'. */ 380 /* #define MC_API_START() register struct bm_mc *mc = &portal->mc */ 381 382 383 t_Error bm_mc_init(struct bm_portal *portal) 384 { 385 register struct bm_mc *mc = &portal->mc; 386 mc->cr = ptr_ADD(portal->addr.addr_ce, CL_CR); 387 mc->rr = ptr_ADD(portal->addr.addr_ce, CL_RR0); 388 mc->rridx = (uint8_t)((mc->cr->__dont_write_directly__verb & BM_MCC_VERB_VBIT) ? 389 0 : 1); 390 mc->vbit = (uint8_t)(mc->rridx ? BM_MCC_VERB_VBIT : 0); 391 #ifdef BM_CHECKING 392 mc->state = mc_idle; 393 #endif /* BM_CHECKING */ 394 return 0; 395 } 396 397 398 void bm_mc_finish(struct bm_portal *portal) 399 { 400 register struct bm_mc *mc = &portal->mc; 401 ASSERT_COND(mc->state == mc_idle); 402 #ifdef BM_CHECKING 403 if (mc->state != mc_idle) 404 REPORT_ERROR(WARNING, E_INVALID_STATE, ("Losing incomplete MC command")); 405 #else 406 UNUSED(mc); 407 #endif /* BM_CHECKING */ 408 } 409 410 411 struct bm_mc_command *bm_mc_start(struct bm_portal *portal) 412 { 413 register struct bm_mc *mc = &portal->mc; 414 ASSERT_COND(mc->state == mc_idle); 415 #ifdef BM_CHECKING 416 mc->state = mc_user; 417 #endif /* BM_CHECKING */ 418 dcbz_64(mc->cr); 419 return mc->cr; 420 } 421 422 423 void bm_mc_abort(struct bm_portal *portal) 424 { 425 register struct bm_mc *mc = &portal->mc; 426 ASSERT_COND(mc->state == mc_user); 427 #ifdef BM_CHECKING 428 mc->state = mc_idle; 429 #else 430 UNUSED(mc); 431 #endif /* BM_CHECKING */ 432 } 433 434 435 void bm_mc_commit(struct bm_portal *portal, uint8_t myverb) 436 { 437 register struct bm_mc *mc = &portal->mc; 438 ASSERT_COND(mc->state == mc_user); 439 rmb(); 440 mc->cr->__dont_write_directly__verb = (uint8_t)(myverb | mc->vbit); 441 dcbf_64(mc->cr); 442 dcbit_ro(mc->rr + mc->rridx); 443 #ifdef BM_CHECKING 444 mc->state = mc_hw; 445 #endif /* BM_CHECKING */ 446 } 447 448 449 struct bm_mc_result *bm_mc_result(struct bm_portal *portal) 450 { 451 register struct bm_mc *mc = &portal->mc; 452 struct bm_mc_result *rr = mc->rr + mc->rridx; 453 ASSERT_COND(mc->state == mc_hw); 454 /* The inactive response register's verb byte always returns zero until 455 * its command is submitted and completed. This includes the valid-bit, 456 * in case you were wondering... */ 457 if (!rr->verb) { 458 dcbit_ro(rr); 459 return NULL; 460 } 461 mc->rridx ^= 1; 462 mc->vbit ^= BM_MCC_VERB_VBIT; 463 #ifdef BM_CHECKING 464 mc->state = mc_idle; 465 #endif /* BM_CHECKING */ 466 return rr; 467 } 468 469 /* ------------------------------------- */ 470 /* --- Portal interrupt register API --- */ 471 472 #define SCN_REG(bpid) REG_SCN((bpid) / 32) 473 #define SCN_BIT(bpid) (0x80000000 >> (bpid & 31)) 474 void bm_isr_bscn_mask(struct bm_portal *portal, uint8_t bpid, int enable) 475 { 476 uint32_t val; 477 ASSERT_COND(bpid < BM_MAX_NUM_OF_POOLS); 478 /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */ 479 val = __bm_in(&portal->addr, SCN_REG(bpid)); 480 if (enable) 481 val |= SCN_BIT(bpid); 482 else 483 val &= ~SCN_BIT(bpid); 484 __bm_out(&portal->addr, SCN_REG(bpid), val); 485 } 486 487 488 uint32_t __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n) 489 { 490 return __bm_in(&portal->addr, REG_ISR + (n << 2)); 491 } 492 493 494 void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, uint32_t val) 495 { 496 __bm_out(&portal->addr, REG_ISR + (n << 2), val); 497 } 498 499