1 /* 2 * Copyright (c) 2016 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include "nvme.h" 36 37 static int nvme_pci_attach(device_t); 38 static int nvme_pci_detach(device_t); 39 40 static const nvme_device_t nvme_devices[] = { 41 /* Vendor-specific table goes here (see ahci for example) */ 42 { 0, 0, nvme_pci_attach, nvme_pci_detach, "NVME-PCIe" } 43 }; 44 45 static int nvme_msix_enable = 1; 46 TUNABLE_INT("hw.nvme.msix.enable", &nvme_msix_enable); 47 static int nvme_msi_enable = 0; 48 TUNABLE_INT("hw.nvme.msi.enable", &nvme_msi_enable); 49 50 TAILQ_HEAD(, nvme_softc) nvme_sc_list = TAILQ_HEAD_INITIALIZER(nvme_sc_list); 51 struct lock nvme_master_lock = LOCK_INITIALIZER("nvmstr", 0, 0); 52 53 static int last_global_cpu; 54 55 /* 56 * Match during probe and attach. The device does not yet have a softc. 57 */ 58 const nvme_device_t * 59 nvme_lookup_device(device_t dev) 60 { 61 const nvme_device_t *ad; 62 uint16_t vendor = pci_get_vendor(dev); 63 uint16_t product = pci_get_device(dev); 64 uint8_t class = pci_get_class(dev); 65 uint8_t subclass = pci_get_subclass(dev); 66 uint8_t progif = pci_read_config(dev, PCIR_PROGIF, 1); 67 int is_nvme; 68 69 /* 70 * Generally speaking if the pci device does not identify as 71 * AHCI we skip it. 72 */ 73 if (class == PCIC_STORAGE && subclass == PCIS_STORAGE_NVM && 74 progif == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) { 75 is_nvme = 1; 76 } else { 77 is_nvme = 0; 78 } 79 80 for (ad = &nvme_devices[0]; ad->vendor; ++ad) { 81 if (ad->vendor == vendor && ad->product == product) 82 return (ad); 83 } 84 85 /* 86 * Last ad is the default match if the PCI device matches SATA. 87 */ 88 if (is_nvme == 0) 89 ad = NULL; 90 return (ad); 91 } 92 93 /* 94 * Attach functions. They all eventually fall through to nvme_pci_attach(). 95 */ 96 static int 97 nvme_pci_attach(device_t dev) 98 { 99 nvme_softc_t *sc = device_get_softc(dev); 100 uint32_t reg; 101 int error; 102 int msi_enable; 103 int msix_enable; 104 105 #if 0 106 if (pci_read_config(dev, PCIR_COMMAND, 2) & 0x0400) { 107 device_printf(dev, "BIOS disabled PCI interrupt, " 108 "re-enabling\n"); 109 pci_write_config(dev, PCIR_COMMAND, 110 pci_read_config(dev, PCIR_COMMAND, 2) & ~0x0400, 2); 111 } 112 #endif 113 114 sc->dev = dev; 115 116 /* 117 * Map the register window 118 */ 119 sc->rid_regs = PCIR_BAR(0); 120 sc->regs = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 121 &sc->rid_regs, RF_ACTIVE); 122 if (sc->regs == NULL) { 123 device_printf(dev, "unable to map registers\n"); 124 nvme_pci_detach(dev); 125 return (ENXIO); 126 } 127 sc->iot = rman_get_bustag(sc->regs); 128 sc->ioh = rman_get_bushandle(sc->regs); 129 130 /* 131 * NVMe allows the MSI-X table to be mapped to BAR 4/5. 132 * Always try to map BAR4, but it's ok if it fails. Must 133 * be done prior to allocating our interrupts. 134 */ 135 sc->rid_bar4 = PCIR_BAR(4); 136 sc->bar4 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 137 &sc->rid_bar4, RF_ACTIVE); 138 139 /* 140 * Map the interrupt or initial interrupt which will be used for 141 * the admin queue. NVME chipsets can potentially support a huge 142 * number of MSIX vectors but we really only need enough for 143 * available cpus, plus 1. 144 */ 145 msi_enable = device_getenv_int(dev, "msi.enable", nvme_msi_enable); 146 msix_enable = device_getenv_int(dev, "msix.enable", nvme_msix_enable); 147 148 error = 0; 149 if (msix_enable) { 150 int i; 151 int cpu; 152 153 sc->nirqs = pci_msix_count(dev); 154 sc->irq_type = PCI_INTR_TYPE_MSIX; 155 if (sc->nirqs > ncpus + 1) /* max we need */ 156 sc->nirqs = ncpus + 1; 157 158 error = pci_setup_msix(dev); 159 cpu = (last_global_cpu + 0) % ncpus; /* GCC warn */ 160 for (i = 0; error == 0 && i < sc->nirqs; ++i) { 161 cpu = (last_global_cpu + i) % ncpus; 162 error = pci_alloc_msix_vector(dev, i, 163 &sc->rid_irq[i], cpu); 164 if (error) 165 break; 166 sc->irq[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, 167 &sc->rid_irq[i], 168 RF_ACTIVE); 169 /* 170 * We want this to overwrite queue 0's cpu vector 171 * when the cpu's rotate through later on. 172 */ 173 if (sc->cputovect[cpu] == 0) 174 sc->cputovect[cpu] = i; 175 } 176 177 /* 178 * If we did not iterate enough cpus (that is, there weren't 179 * enough irqs for all available cpus) we still need to 180 * finish or sc->cputovect[] mapping. 181 */ 182 while (error == 0) { 183 cpu = (cpu + 1) % ncpus; 184 i = (i + 1) % sc->nirqs; 185 if (i == 0) 186 i = 1; 187 if (sc->cputovect[cpu] != 0) 188 break; 189 sc->cputovect[cpu] = i; 190 } 191 192 if (error) { 193 while (--i >= 0) { 194 bus_release_resource(dev, SYS_RES_IRQ, 195 sc->rid_irq[i], 196 sc->irq[i]); 197 pci_release_msix_vector(dev, sc->rid_irq[i]); 198 sc->irq[i] = NULL; 199 } 200 /* leave error intact to fall through to normal */ 201 } else { 202 last_global_cpu = (last_global_cpu + sc->nirqs) % ncpus; 203 pci_enable_msix(dev); 204 } 205 } 206 if (msix_enable == 0 || error) { 207 uint32_t irq_flags; 208 209 error = 0; 210 sc->nirqs = 1; 211 sc->irq_type = pci_alloc_1intr(dev, msi_enable, 212 &sc->rid_irq[0], &irq_flags); 213 sc->irq[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, 214 &sc->rid_irq[0], irq_flags); 215 } 216 if (sc->irq[0] == NULL) { 217 device_printf(dev, "unable to map interrupt\n"); 218 nvme_pci_detach(dev); 219 return (ENXIO); 220 } else { 221 const char *type; 222 switch(sc->irq_type) { 223 case PCI_INTR_TYPE_MSI: 224 type = "MSI"; 225 break; 226 case PCI_INTR_TYPE_MSIX: 227 type = "MSIX"; 228 break; 229 default: 230 type = "normal-int"; 231 break; 232 } 233 device_printf(dev, "mapped %d %s IRQs\n", sc->nirqs, type); 234 } 235 236 /* 237 * Make sure the chip is disabled, which will reset all controller 238 * registers except for the admin queue registers. Device should 239 * already be disabled so this is usually instantanious. Use a 240 * fixed 5-second timeout in case it is not. I'd like my other 241 * reads to occur after the device has been disabled. 242 */ 243 sc->entimo = hz * 5; 244 error = nvme_enable(sc, 0); 245 if (error) { 246 nvme_pci_detach(dev); 247 return (ENXIO); 248 } 249 250 /* 251 * Get capabillities and version and report 252 */ 253 sc->vers = nvme_read(sc, NVME_REG_VERS); 254 sc->cap = nvme_read8(sc, NVME_REG_CAP); 255 sc->maxqe = NVME_CAP_MQES_GET(sc->cap); 256 sc->dstrd4 = NVME_CAP_DSTRD_GET(sc->cap); 257 258 device_printf(dev, "NVME Version %u.%u maxqe=%u caps=%016jx\n", 259 NVME_VERS_MAJOR_GET(sc->vers), 260 NVME_VERS_MINOR_GET(sc->vers), 261 sc->maxqe, sc->cap); 262 263 /* 264 * Enable timeout, 500ms increments. Convert to ticks. 265 */ 266 sc->entimo = NVME_CAP_TIMEOUT_GET(sc->cap) * hz / 2; /* in ticks */ 267 ++sc->entimo; /* fudge */ 268 269 /* 270 * Validate maxqe. To cap the amount of memory we reserve for 271 * PRPs we limit maxqe to 256. Also make sure it is a power of 272 * two. 273 */ 274 if (sc->maxqe < 2) { 275 device_printf(dev, 276 "Attach failed, max queue entries (%d) " 277 "below minimum (2)\n", sc->maxqe); 278 nvme_pci_detach(dev); 279 return (ENXIO); 280 } 281 if (sc->maxqe > 256) 282 sc->maxqe = 256; 283 for (reg = 2; reg <= sc->maxqe; reg <<= 1) 284 ; 285 sc->maxqe = reg >> 1; 286 287 /* 288 * DMA tags 289 * 290 * PRP - Worst case PRPs needed per queue is MAXPHYS / PAGE_SIZE 291 * (typically 64), multiplied by maxqe (typ 256). Roughly 292 * ~128KB per queue. Align for cache performance. We actually 293 * need one more PRP per queue entry worst-case to handle 294 * buffer overlap, but we have an extra one in the command 295 * structure so we don't have to calculate that out. 296 * 297 * Remember that we intend to allocate potentially many queues, 298 * so we don't want to bloat this too much. A queue depth of 299 * 256 is plenty. 300 * 301 * CMD - Storage for the submit queue. maxqe * 64 (~16KB) 302 * 303 * RES - Storage for the completion queue. maxqe * 16 (~4KB) 304 * 305 * ADM - Storage for admin command DMA data. Maximum admin command 306 * DMA data is 4KB so reserve maxqe * 4KB (~1MB). There is only 307 * one admin queue. 308 * 309 * NOTE: There are no boundary requirements for NVMe, but I specify a 310 * 4MB boundary anyway because this reduces mass-bit flipping 311 * of address bits inside the controller when incrementing 312 * DMA addresses. Why not? Can't hurt. 313 */ 314 sc->prp_bytes = sizeof(uint64_t) * (MAXPHYS / PAGE_SIZE) * sc->maxqe; 315 sc->cmd_bytes = sizeof(nvme_subq_item_t) * sc->maxqe; 316 sc->res_bytes = sizeof(nvme_comq_item_t) * sc->maxqe; 317 sc->adm_bytes = NVME_MAX_ADMIN_BUFFER * sc->maxqe; 318 319 error = 0; 320 321 error += bus_dma_tag_create( 322 NULL, /* parent tag */ 323 PAGE_SIZE, /* alignment */ 324 4 * 1024 * 1024, /* boundary */ 325 BUS_SPACE_MAXADDR, /* loaddr? */ 326 BUS_SPACE_MAXADDR, /* hiaddr */ 327 NULL, /* filter */ 328 NULL, /* filterarg */ 329 sc->prp_bytes, /* [max]size */ 330 1, /* maxsegs */ 331 sc->prp_bytes, /* maxsegsz */ 332 0, /* flags */ 333 &sc->prps_tag); /* return tag */ 334 335 error += bus_dma_tag_create( 336 NULL, /* parent tag */ 337 PAGE_SIZE, /* alignment */ 338 4 * 1024 * 1024, /* boundary */ 339 BUS_SPACE_MAXADDR, /* loaddr? */ 340 BUS_SPACE_MAXADDR, /* hiaddr */ 341 NULL, /* filter */ 342 NULL, /* filterarg */ 343 sc->cmd_bytes, /* [max]size */ 344 1, /* maxsegs */ 345 sc->cmd_bytes, /* maxsegsz */ 346 0, /* flags */ 347 &sc->sque_tag); /* return tag */ 348 349 error += bus_dma_tag_create( 350 NULL, /* parent tag */ 351 PAGE_SIZE, /* alignment */ 352 4 * 1024 * 1024, /* boundary */ 353 BUS_SPACE_MAXADDR, /* loaddr? */ 354 BUS_SPACE_MAXADDR, /* hiaddr */ 355 NULL, /* filter */ 356 NULL, /* filterarg */ 357 sc->res_bytes, /* [max]size */ 358 1, /* maxsegs */ 359 sc->res_bytes, /* maxsegsz */ 360 0, /* flags */ 361 &sc->cque_tag); /* return tag */ 362 363 error += bus_dma_tag_create( 364 NULL, /* parent tag */ 365 PAGE_SIZE, /* alignment */ 366 4 * 1024 * 1024, /* boundary */ 367 BUS_SPACE_MAXADDR, /* loaddr? */ 368 BUS_SPACE_MAXADDR, /* hiaddr */ 369 NULL, /* filter */ 370 NULL, /* filterarg */ 371 sc->adm_bytes, /* [max]size */ 372 1, /* maxsegs */ 373 sc->adm_bytes, /* maxsegsz */ 374 0, /* flags */ 375 &sc->adm_tag); /* return tag */ 376 377 if (error) { 378 device_printf(dev, "unable to create dma tags\n"); 379 nvme_pci_detach(dev); 380 return (ENXIO); 381 } 382 383 /* 384 * Setup the admin queues (qid 0). 385 */ 386 error = nvme_alloc_subqueue(sc, 0); 387 if (error) { 388 device_printf(dev, "unable to allocate admin subqueue\n"); 389 nvme_pci_detach(dev); 390 return (ENXIO); 391 } 392 error = nvme_alloc_comqueue(sc, 0); 393 if (error) { 394 device_printf(dev, "unable to allocate admin comqueue\n"); 395 nvme_pci_detach(dev); 396 return (ENXIO); 397 } 398 399 /* 400 * Initialize the admin queue registers 401 */ 402 reg = NVME_ATTR_COM_SET(sc->maxqe) | NVME_ATTR_SUB_SET(sc->maxqe); 403 nvme_write(sc, NVME_REG_ADM_ATTR, reg); 404 nvme_write8(sc, NVME_REG_ADM_SUBADR, (uint64_t)sc->subqueues[0].psubq); 405 nvme_write8(sc, NVME_REG_ADM_COMADR, (uint64_t)sc->comqueues[0].pcomq); 406 407 /* 408 * qemu appears to require this, real hardware does not appear 409 * to require this. 410 */ 411 pci_enable_busmaster(dev); 412 413 /* 414 * Other configuration registers 415 */ 416 reg = NVME_CONFIG_IOSUB_ES_SET(6) | /* 64 byte sub entry */ 417 NVME_CONFIG_IOCOM_ES_SET(4) | /* 16 byte com entry */ 418 NVME_CONFIG_MEMPG_SET(PAGE_SHIFT) | /* 4K pages */ 419 NVME_CONFIG_CSS_NVM; /* NVME command set */ 420 nvme_write(sc, NVME_REG_CONFIG, reg); 421 422 reg = nvme_read(sc, NVME_REG_MEMSIZE); 423 424 /* 425 * Enable the chip for operation 426 */ 427 error = nvme_enable(sc, 1); 428 if (error) { 429 nvme_enable(sc, 0); 430 nvme_pci_detach(dev); 431 return (ENXIO); 432 } 433 434 /* 435 * Start the admin thread. This will also setup the admin queue 436 * interrupt. 437 */ 438 error = nvme_start_admin_thread(sc); 439 if (error) { 440 nvme_pci_detach(dev); 441 return (ENXIO); 442 } 443 lockmgr(&nvme_master_lock, LK_EXCLUSIVE); 444 sc->flags |= NVME_SC_ATTACHED; 445 TAILQ_INSERT_TAIL(&nvme_sc_list, sc, entry); 446 lockmgr(&nvme_master_lock, LK_RELEASE); 447 448 return(0); 449 } 450 451 /* 452 * Device unload / detachment 453 */ 454 static int 455 nvme_pci_detach(device_t dev) 456 { 457 nvme_softc_t *sc = device_get_softc(dev); 458 int i; 459 460 /* 461 * Stop the admin thread 462 */ 463 nvme_stop_admin_thread(sc); 464 465 /* 466 * Issue a normal shutdown and wait for completion 467 */ 468 nvme_issue_shutdown(sc); 469 470 /* 471 * Disable the chip 472 */ 473 nvme_enable(sc, 0); 474 475 /* 476 * Free admin memory 477 */ 478 nvme_free_subqueue(sc, 0); 479 nvme_free_comqueue(sc, 0); 480 481 /* 482 * Release related resources. 483 */ 484 for (i = 0; i < sc->nirqs; ++i) { 485 if (sc->irq[i]) { 486 bus_release_resource(dev, SYS_RES_IRQ, 487 sc->rid_irq[i], sc->irq[i]); 488 sc->irq[i] = NULL; 489 if (sc->irq_type == PCI_INTR_TYPE_MSIX) 490 pci_release_msix_vector(dev, sc->rid_irq[i]); 491 } 492 } 493 switch(sc->irq_type) { 494 case PCI_INTR_TYPE_MSI: 495 pci_release_msi(dev); 496 break; 497 case PCI_INTR_TYPE_MSIX: 498 pci_teardown_msix(dev); 499 break; 500 default: 501 break; 502 } 503 504 /* 505 * Release remaining chipset resources 506 */ 507 if (sc->regs) { 508 bus_release_resource(dev, SYS_RES_MEMORY, 509 sc->rid_regs, sc->regs); 510 sc->regs = NULL; 511 } 512 if (sc->bar4) { 513 bus_release_resource(dev, SYS_RES_MEMORY, 514 sc->rid_bar4, sc->bar4); 515 sc->bar4 = NULL; 516 } 517 518 /* 519 * Cleanup the DMA tags 520 */ 521 if (sc->prps_tag) { 522 bus_dma_tag_destroy(sc->prps_tag); 523 sc->prps_tag = NULL; 524 } 525 if (sc->sque_tag) { 526 bus_dma_tag_destroy(sc->sque_tag); 527 sc->sque_tag = NULL; 528 } 529 if (sc->cque_tag) { 530 bus_dma_tag_destroy(sc->cque_tag); 531 sc->cque_tag = NULL; 532 } 533 if (sc->adm_tag) { 534 bus_dma_tag_destroy(sc->adm_tag); 535 sc->adm_tag = NULL; 536 } 537 538 if (sc->flags & NVME_SC_ATTACHED) { 539 lockmgr(&nvme_master_lock, LK_EXCLUSIVE); 540 sc->flags &= ~NVME_SC_ATTACHED; 541 TAILQ_REMOVE(&nvme_sc_list, sc, entry); 542 lockmgr(&nvme_master_lock, LK_RELEASE); 543 } 544 545 return (0); 546 } 547