1 /*- 2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/kernel.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/rmlock.h> 35 #include <sys/malloc.h> 36 #include <sys/module.h> 37 #include <sys/sysctl.h> 38 39 #include "ntb.h" 40 41 devclass_t ntb_hw_devclass; 42 SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 43 "NTB sysctls"); 44 45 struct ntb_child { 46 device_t dev; 47 int function; 48 int enabled; 49 int mwoff; 50 int mwcnt; 51 int spadoff; 52 int spadcnt; 53 int dboff; 54 int dbcnt; 55 uint64_t dbmask; 56 void *ctx; 57 const struct ntb_ctx_ops *ctx_ops; 58 struct rmlock ctx_lock; 59 struct ntb_child *next; 60 }; 61 62 int 63 ntb_register_device(device_t dev) 64 { 65 struct ntb_child **cpp = device_get_softc(dev); 66 struct ntb_child *nc; 67 int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt; 68 char cfg[128] = ""; 69 char buf[32]; 70 char *n, *np, *c, *p, *name; 71 72 mwu = 0; 73 mwt = NTB_MW_COUNT(dev); 74 spadu = 0; 75 spadt = NTB_SPAD_COUNT(dev); 76 dbu = 0; 77 dbt = flsll(NTB_DB_VALID_MASK(dev)); 78 79 device_printf(dev, "%d memory windows, %d scratchpads, " 80 "%d doorbells\n", mwt, spadt, dbt); 81 82 snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), 83 device_get_unit(dev)); 84 TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); 85 n = cfg; 86 i = 0; 87 while ((c = strsep(&n, ",")) != NULL) { 88 np = c; 89 name = strsep(&np, ":"); 90 if (name != NULL && name[0] == 0) 91 name = NULL; 92 p = strsep(&np, ":"); 93 mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu; 94 p = strsep(&np, ":"); 95 spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu; 96 db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu; 97 98 if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) { 99 device_printf(dev, "Not enough resources for config\n"); 100 break; 101 } 102 103 nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); 104 nc->function = i; 105 nc->mwoff = mwu; 106 nc->mwcnt = mw; 107 nc->spadoff = spadu; 108 nc->spadcnt = spad; 109 nc->dboff = dbu; 110 nc->dbcnt = db; 111 nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db)); 112 rm_init(&nc->ctx_lock, "ntb ctx"); 113 nc->dev = device_add_child(dev, name, -1); 114 if (nc->dev == NULL) { 115 ntb_unregister_device(dev); 116 return (ENOMEM); 117 } 118 device_set_ivars(nc->dev, nc); 119 *cpp = nc; 120 cpp = &nc->next; 121 122 if (bootverbose) { 123 device_printf(dev, "%d \"%s\":", i, name); 124 if (mw > 0) { 125 printf(" memory windows %d", mwu); 126 if (mw > 1) 127 printf("-%d", mwu + mw - 1); 128 } 129 if (spad > 0) { 130 printf(" scratchpads %d", spadu); 131 if (spad > 1) 132 printf("-%d", spadu + spad - 1); 133 } 134 if (db > 0) { 135 printf(" doorbells %d", dbu); 136 if (db > 1) 137 printf("-%d", dbu + db - 1); 138 } 139 printf("\n"); 140 } 141 142 mwu += mw; 143 spadu += spad; 144 dbu += db; 145 i++; 146 } 147 148 bus_generic_attach(dev); 149 return (0); 150 } 151 152 int 153 ntb_unregister_device(device_t dev) 154 { 155 struct ntb_child **cpp = device_get_softc(dev); 156 struct ntb_child *nc; 157 int error = 0; 158 159 while ((nc = *cpp) != NULL) { 160 *cpp = (*cpp)->next; 161 error = device_delete_child(dev, nc->dev); 162 if (error) 163 break; 164 rm_destroy(&nc->ctx_lock); 165 free(nc, M_DEVBUF); 166 } 167 return (error); 168 } 169 170 int 171 ntb_child_location_str(device_t dev, device_t child, char *buf, 172 size_t buflen) 173 { 174 struct ntb_child *nc = device_get_ivars(child); 175 176 snprintf(buf, buflen, "function=%d", nc->function); 177 return (0); 178 } 179 180 int 181 ntb_print_child(device_t dev, device_t child) 182 { 183 struct ntb_child *nc = device_get_ivars(child); 184 int retval; 185 186 retval = bus_print_child_header(dev, child); 187 if (nc->mwcnt > 0) { 188 printf(" mw %d", nc->mwoff); 189 if (nc->mwcnt > 1) 190 printf("-%d", nc->mwoff + nc->mwcnt - 1); 191 } 192 if (nc->spadcnt > 0) { 193 printf(" spad %d", nc->spadoff); 194 if (nc->spadcnt > 1) 195 printf("-%d", nc->spadoff + nc->spadcnt - 1); 196 } 197 if (nc->dbcnt > 0) { 198 printf(" db %d", nc->dboff); 199 if (nc->dbcnt > 1) 200 printf("-%d", nc->dboff + nc->dbcnt - 1); 201 } 202 retval += printf(" at function %d", nc->function); 203 retval += bus_print_child_domain(dev, child); 204 retval += bus_print_child_footer(dev, child); 205 206 return (retval); 207 } 208 209 bus_dma_tag_t 210 ntb_get_dma_tag(device_t bus, device_t child) 211 { 212 213 return (bus_get_dma_tag(bus)); 214 } 215 216 void 217 ntb_link_event(device_t dev) 218 { 219 struct ntb_child **cpp = device_get_softc(dev); 220 struct ntb_child *nc; 221 struct rm_priotracker ctx_tracker; 222 enum ntb_speed speed; 223 enum ntb_width width; 224 225 if (NTB_LINK_IS_UP(dev, &speed, &width)) { 226 device_printf(dev, "Link is up (PCIe %d.x / x%d)\n", 227 (int)speed, (int)width); 228 } else { 229 device_printf(dev, "Link is down\n"); 230 } 231 for (nc = *cpp; nc != NULL; nc = nc->next) { 232 rm_rlock(&nc->ctx_lock, &ctx_tracker); 233 if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL) 234 nc->ctx_ops->link_event(nc->ctx); 235 rm_runlock(&nc->ctx_lock, &ctx_tracker); 236 } 237 } 238 239 void 240 ntb_db_event(device_t dev, uint32_t vec) 241 { 242 struct ntb_child **cpp = device_get_softc(dev); 243 struct ntb_child *nc; 244 struct rm_priotracker ctx_tracker; 245 246 for (nc = *cpp; nc != NULL; nc = nc->next) { 247 rm_rlock(&nc->ctx_lock, &ctx_tracker); 248 if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL) 249 nc->ctx_ops->db_event(nc->ctx, vec); 250 rm_runlock(&nc->ctx_lock, &ctx_tracker); 251 } 252 } 253 254 int 255 ntb_port_number(device_t ntb) 256 { 257 return (NTB_PORT_NUMBER(device_get_parent(ntb))); 258 } 259 260 int 261 ntb_peer_port_count(device_t ntb) 262 { 263 return (NTB_PEER_PORT_COUNT(device_get_parent(ntb))); 264 } 265 266 int 267 ntb_peer_port_number(device_t ntb, int pidx) 268 { 269 return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx)); 270 } 271 272 int 273 ntb_peer_port_idx(device_t ntb, int port) 274 { 275 return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port)); 276 } 277 278 bool 279 ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) 280 { 281 282 return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width)); 283 } 284 285 int 286 ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width) 287 { 288 struct ntb_child *nc = device_get_ivars(ntb); 289 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 290 struct ntb_child *nc1; 291 292 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 293 if (nc1->enabled) { 294 nc->enabled = 1; 295 return (0); 296 } 297 } 298 nc->enabled = 1; 299 return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width)); 300 } 301 302 int 303 ntb_link_disable(device_t ntb) 304 { 305 struct ntb_child *nc = device_get_ivars(ntb); 306 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 307 struct ntb_child *nc1; 308 309 if (!nc->enabled) 310 return (0); 311 nc->enabled = 0; 312 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 313 if (nc1->enabled) 314 return (0); 315 } 316 return (NTB_LINK_DISABLE(device_get_parent(ntb))); 317 } 318 319 bool 320 ntb_link_enabled(device_t ntb) 321 { 322 struct ntb_child *nc = device_get_ivars(ntb); 323 324 return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb))); 325 } 326 327 int 328 ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops) 329 { 330 struct ntb_child *nc = device_get_ivars(ntb); 331 332 if (ctx == NULL || ctx_ops == NULL) 333 return (EINVAL); 334 335 rm_wlock(&nc->ctx_lock); 336 if (nc->ctx_ops != NULL) { 337 rm_wunlock(&nc->ctx_lock); 338 return (EINVAL); 339 } 340 nc->ctx = ctx; 341 nc->ctx_ops = ctx_ops; 342 343 /* 344 * If applicaiton driver asks for link events, generate fake one now 345 * to let it update link state without races while we hold the lock. 346 */ 347 if (ctx_ops->link_event != NULL) 348 ctx_ops->link_event(ctx); 349 rm_wunlock(&nc->ctx_lock); 350 351 return (0); 352 } 353 354 void * 355 ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops) 356 { 357 struct ntb_child *nc = device_get_ivars(ntb); 358 359 KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus")); 360 if (ctx_ops != NULL) 361 *ctx_ops = nc->ctx_ops; 362 return (nc->ctx); 363 } 364 365 void 366 ntb_clear_ctx(device_t ntb) 367 { 368 struct ntb_child *nc = device_get_ivars(ntb); 369 370 rm_wlock(&nc->ctx_lock); 371 nc->ctx = NULL; 372 nc->ctx_ops = NULL; 373 rm_wunlock(&nc->ctx_lock); 374 } 375 376 uint8_t 377 ntb_mw_count(device_t ntb) 378 { 379 struct ntb_child *nc = device_get_ivars(ntb); 380 381 return (nc->mwcnt); 382 } 383 384 int 385 ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, 386 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 387 bus_addr_t *plimit) 388 { 389 struct ntb_child *nc = device_get_ivars(ntb); 390 391 return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff, 392 base, vbase, size, align, align_size, plimit)); 393 } 394 395 int 396 ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size) 397 { 398 struct ntb_child *nc = device_get_ivars(ntb); 399 400 return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff, 401 addr, size)); 402 } 403 404 int 405 ntb_mw_clear_trans(device_t ntb, unsigned mw_idx) 406 { 407 struct ntb_child *nc = device_get_ivars(ntb); 408 409 return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff)); 410 } 411 412 int 413 ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode) 414 { 415 struct ntb_child *nc = device_get_ivars(ntb); 416 417 return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 418 } 419 420 int 421 ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode) 422 { 423 struct ntb_child *nc = device_get_ivars(ntb); 424 425 return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 426 } 427 428 uint8_t 429 ntb_spad_count(device_t ntb) 430 { 431 struct ntb_child *nc = device_get_ivars(ntb); 432 433 return (nc->spadcnt); 434 } 435 436 void 437 ntb_spad_clear(device_t ntb) 438 { 439 struct ntb_child *nc = device_get_ivars(ntb); 440 unsigned i; 441 442 for (i = 0; i < nc->spadcnt; i++) 443 NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0); 444 } 445 446 int 447 ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val) 448 { 449 struct ntb_child *nc = device_get_ivars(ntb); 450 451 return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val)); 452 } 453 454 int 455 ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 456 { 457 struct ntb_child *nc = device_get_ivars(ntb); 458 459 return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val)); 460 } 461 462 int 463 ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val) 464 { 465 struct ntb_child *nc = device_get_ivars(ntb); 466 467 return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, 468 val)); 469 } 470 471 int 472 ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 473 { 474 struct ntb_child *nc = device_get_ivars(ntb); 475 476 return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, 477 val)); 478 } 479 480 uint64_t 481 ntb_db_valid_mask(device_t ntb) 482 { 483 struct ntb_child *nc = device_get_ivars(ntb); 484 485 return (nc->dbmask); 486 } 487 488 int 489 ntb_db_vector_count(device_t ntb) 490 { 491 492 return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb))); 493 } 494 495 uint64_t 496 ntb_db_vector_mask(device_t ntb, uint32_t vector) 497 { 498 struct ntb_child *nc = device_get_ivars(ntb); 499 500 return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector) 501 >> nc->dboff) & nc->dbmask); 502 } 503 504 int 505 ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size) 506 { 507 508 return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size)); 509 } 510 511 void 512 ntb_db_clear(device_t ntb, uint64_t bits) 513 { 514 struct ntb_child *nc = device_get_ivars(ntb); 515 516 return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff)); 517 } 518 519 void 520 ntb_db_clear_mask(device_t ntb, uint64_t bits) 521 { 522 struct ntb_child *nc = device_get_ivars(ntb); 523 524 return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff)); 525 } 526 527 uint64_t 528 ntb_db_read(device_t ntb) 529 { 530 struct ntb_child *nc = device_get_ivars(ntb); 531 532 return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff) 533 & nc->dbmask); 534 } 535 536 void 537 ntb_db_set_mask(device_t ntb, uint64_t bits) 538 { 539 struct ntb_child *nc = device_get_ivars(ntb); 540 541 return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff)); 542 } 543 544 void 545 ntb_peer_db_set(device_t ntb, uint64_t bits) 546 { 547 struct ntb_child *nc = device_get_ivars(ntb); 548 549 return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff)); 550 } 551 552 MODULE_VERSION(ntb, 1); 553