1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2017 Shunsuke Mie 5 * Copyright (c) 2018 Leon Dang 6 * Copyright (c) 2020 Chuck Tuffli 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * bhyve PCIe-NVMe device emulation. 32 * 33 * options: 34 * -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm=<opt> 35 * 36 * accepted devpath: 37 * /dev/blockdev 38 * /path/to/image 39 * ram=size_in_MiB 40 * 41 * maxq = max number of queues 42 * qsz = max elements in each queue 43 * ioslots = max number of concurrent io requests 44 * sectsz = sector size (defaults to blockif sector size) 45 * ser = serial number (20-chars max) 46 * eui64 = IEEE Extended Unique Identifier (8 byte value) 47 * dsm = DataSet Management support. Option is one of auto, enable,disable 48 * 49 */ 50 51 /* TODO: 52 - create async event for smart and log 53 - intr coalesce 54 */ 55 56 #include <sys/cdefs.h> 57 __FBSDID("$FreeBSD$"); 58 59 #include <sys/errno.h> 60 #include <sys/types.h> 61 #include <sys/crc16.h> 62 #include <net/ieee_oui.h> 63 64 #include <assert.h> 65 #include <pthread.h> 66 #include <pthread_np.h> 67 #include <semaphore.h> 68 #include <stdbool.h> 69 #include <stddef.h> 70 #include <stdint.h> 71 #include <stdio.h> 72 #include <stdlib.h> 73 #include <string.h> 74 75 #include <machine/atomic.h> 76 #include <machine/vmm.h> 77 #include <vmmapi.h> 78 79 #include <dev/nvme/nvme.h> 80 81 #include "bhyverun.h" 82 #include "block_if.h" 83 #include "config.h" 84 #include "debug.h" 85 #include "pci_emul.h" 86 87 88 static int nvme_debug = 0; 89 #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args) 90 #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args) 91 92 /* defaults; can be overridden */ 93 #define NVME_MSIX_BAR 4 94 95 #define NVME_IOSLOTS 8 96 97 /* The NVMe spec defines bits 13:4 in BAR0 as reserved */ 98 #define NVME_MMIO_SPACE_MIN (1 << 14) 99 100 #define NVME_QUEUES 16 101 #define NVME_MAX_QENTRIES 2048 102 /* Memory Page size Minimum reported in CAP register */ 103 #define NVME_MPSMIN 0 104 /* MPSMIN converted to bytes */ 105 #define NVME_MPSMIN_BYTES (1 << (12 + NVME_MPSMIN)) 106 107 #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t)) 108 #define NVME_MDTS 9 109 /* Note the + 1 allows for the initial descriptor to not be page aligned */ 110 #define NVME_MAX_IOVEC ((1 << NVME_MDTS) + 1) 111 #define NVME_MAX_DATA_SIZE ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES) 112 113 /* This is a synthetic status code to indicate there is no status */ 114 #define NVME_NO_STATUS 0xffff 115 #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS) 116 117 /* Reported temperature in Kelvin (i.e. room temperature) */ 118 #define NVME_TEMPERATURE 296 119 120 /* helpers */ 121 122 /* Convert a zero-based value into a one-based value */ 123 #define ONE_BASED(zero) ((zero) + 1) 124 /* Convert a one-based value into a zero-based value */ 125 #define ZERO_BASED(one) ((one) - 1) 126 127 /* Encode number of SQ's and CQ's for Set/Get Features */ 128 #define NVME_FEATURE_NUM_QUEUES(sc) \ 129 (ZERO_BASED((sc)->num_squeues) & 0xffff) | \ 130 (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16 131 132 #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell) 133 134 enum nvme_controller_register_offsets { 135 NVME_CR_CAP_LOW = 0x00, 136 NVME_CR_CAP_HI = 0x04, 137 NVME_CR_VS = 0x08, 138 NVME_CR_INTMS = 0x0c, 139 NVME_CR_INTMC = 0x10, 140 NVME_CR_CC = 0x14, 141 NVME_CR_CSTS = 0x1c, 142 NVME_CR_NSSR = 0x20, 143 NVME_CR_AQA = 0x24, 144 NVME_CR_ASQ_LOW = 0x28, 145 NVME_CR_ASQ_HI = 0x2c, 146 NVME_CR_ACQ_LOW = 0x30, 147 NVME_CR_ACQ_HI = 0x34, 148 }; 149 150 enum nvme_cmd_cdw11 { 151 NVME_CMD_CDW11_PC = 0x0001, 152 NVME_CMD_CDW11_IEN = 0x0002, 153 NVME_CMD_CDW11_IV = 0xFFFF0000, 154 }; 155 156 enum nvme_copy_dir { 157 NVME_COPY_TO_PRP, 158 NVME_COPY_FROM_PRP, 159 }; 160 161 #define NVME_CQ_INTEN 0x01 162 #define NVME_CQ_INTCOAL 0x02 163 164 struct nvme_completion_queue { 165 struct nvme_completion *qbase; 166 pthread_mutex_t mtx; 167 uint32_t size; 168 uint16_t tail; /* nvme progress */ 169 uint16_t head; /* guest progress */ 170 uint16_t intr_vec; 171 uint32_t intr_en; 172 }; 173 174 struct nvme_submission_queue { 175 struct nvme_command *qbase; 176 pthread_mutex_t mtx; 177 uint32_t size; 178 uint16_t head; /* nvme progress */ 179 uint16_t tail; /* guest progress */ 180 uint16_t cqid; /* completion queue id */ 181 int qpriority; 182 }; 183 184 enum nvme_storage_type { 185 NVME_STOR_BLOCKIF = 0, 186 NVME_STOR_RAM = 1, 187 }; 188 189 struct pci_nvme_blockstore { 190 enum nvme_storage_type type; 191 void *ctx; 192 uint64_t size; 193 uint32_t sectsz; 194 uint32_t sectsz_bits; 195 uint64_t eui64; 196 uint32_t deallocate:1; 197 }; 198 199 /* 200 * Calculate the number of additional page descriptors for guest IO requests 201 * based on the advertised Max Data Transfer (MDTS) and given the number of 202 * default iovec's in a struct blockif_req. 203 */ 204 #define MDTS_PAD_SIZE \ 205 ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \ 206 NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \ 207 0 ) 208 209 struct pci_nvme_ioreq { 210 struct pci_nvme_softc *sc; 211 STAILQ_ENTRY(pci_nvme_ioreq) link; 212 struct nvme_submission_queue *nvme_sq; 213 uint16_t sqid; 214 215 /* command information */ 216 uint16_t opc; 217 uint16_t cid; 218 uint32_t nsid; 219 220 uint64_t prev_gpaddr; 221 size_t prev_size; 222 size_t bytes; 223 224 struct blockif_req io_req; 225 226 struct iovec iovpadding[MDTS_PAD_SIZE]; 227 }; 228 229 enum nvme_dsm_type { 230 /* Dataset Management bit in ONCS reflects backing storage capability */ 231 NVME_DATASET_MANAGEMENT_AUTO, 232 /* Unconditionally set Dataset Management bit in ONCS */ 233 NVME_DATASET_MANAGEMENT_ENABLE, 234 /* Unconditionally clear Dataset Management bit in ONCS */ 235 NVME_DATASET_MANAGEMENT_DISABLE, 236 }; 237 238 struct pci_nvme_softc; 239 struct nvme_feature_obj; 240 241 typedef void (*nvme_feature_cb)(struct pci_nvme_softc *, 242 struct nvme_feature_obj *, 243 struct nvme_command *, 244 struct nvme_completion *); 245 246 struct nvme_feature_obj { 247 uint32_t cdw11; 248 nvme_feature_cb set; 249 nvme_feature_cb get; 250 bool namespace_specific; 251 }; 252 253 #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) 254 255 typedef enum { 256 PCI_NVME_AE_TYPE_ERROR = 0, 257 PCI_NVME_AE_TYPE_SMART, 258 PCI_NVME_AE_TYPE_NOTICE, 259 PCI_NVME_AE_TYPE_IO_CMD = 6, 260 PCI_NVME_AE_TYPE_VENDOR = 7, 261 PCI_NVME_AE_TYPE_MAX /* Must be last */ 262 } pci_nvme_async_type; 263 264 /* Asynchronous Event Requests */ 265 struct pci_nvme_aer { 266 STAILQ_ENTRY(pci_nvme_aer) link; 267 uint16_t cid; /* Command ID of the submitted AER */ 268 }; 269 270 /** Asynchronous Event Information - Notice */ 271 typedef enum { 272 PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0, 273 PCI_NVME_AEI_NOTICE_FW_ACTIVATION, 274 PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE, 275 PCI_NVME_AEI_NOTICE_ANA_CHANGE, 276 PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE, 277 PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT, 278 PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE, 279 PCI_NVME_AEI_NOTICE_MAX, 280 } pci_nvme_async_event_info_notice; 281 282 #define PCI_NVME_AEI_NOTICE_SHIFT 8 283 #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT)) 284 285 /* Asynchronous Event Notifications */ 286 struct pci_nvme_aen { 287 pci_nvme_async_type atype; 288 uint32_t event_data; 289 bool posted; 290 }; 291 292 /* 293 * By default, enable all Asynchrnous Event Notifications: 294 * SMART / Health Critical Warnings 295 * Namespace Attribute Notices 296 */ 297 #define PCI_NVME_AEN_DEFAULT_MASK 0x11f 298 299 typedef enum { 300 NVME_CNTRLTYPE_IO = 1, 301 NVME_CNTRLTYPE_DISCOVERY = 2, 302 NVME_CNTRLTYPE_ADMIN = 3, 303 } pci_nvme_cntrl_type; 304 305 struct pci_nvme_softc { 306 struct pci_devinst *nsc_pi; 307 308 pthread_mutex_t mtx; 309 310 struct nvme_registers regs; 311 312 struct nvme_namespace_data nsdata; 313 struct nvme_controller_data ctrldata; 314 struct nvme_error_information_entry err_log; 315 struct nvme_health_information_page health_log; 316 struct nvme_firmware_page fw_log; 317 struct nvme_ns_list ns_log; 318 319 struct pci_nvme_blockstore nvstore; 320 321 uint16_t max_qentries; /* max entries per queue */ 322 uint32_t max_queues; /* max number of IO SQ's or CQ's */ 323 uint32_t num_cqueues; 324 uint32_t num_squeues; 325 bool num_q_is_set; /* Has host set Number of Queues */ 326 327 struct pci_nvme_ioreq *ioreqs; 328 STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */ 329 uint32_t pending_ios; 330 uint32_t ioslots; 331 sem_t iosemlock; 332 333 /* 334 * Memory mapped Submission and Completion queues 335 * Each array includes both Admin and IO queues 336 */ 337 struct nvme_completion_queue *compl_queues; 338 struct nvme_submission_queue *submit_queues; 339 340 struct nvme_feature_obj feat[NVME_FID_MAX]; 341 342 enum nvme_dsm_type dataset_management; 343 344 /* Accounting for SMART data */ 345 __uint128_t read_data_units; 346 __uint128_t write_data_units; 347 __uint128_t read_commands; 348 __uint128_t write_commands; 349 uint32_t read_dunits_remainder; 350 uint32_t write_dunits_remainder; 351 352 STAILQ_HEAD(, pci_nvme_aer) aer_list; 353 pthread_mutex_t aer_mtx; 354 uint32_t aer_count; 355 struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX]; 356 pthread_t aen_tid; 357 pthread_mutex_t aen_mtx; 358 pthread_cond_t aen_cond; 359 }; 360 361 362 static void pci_nvme_cq_update(struct pci_nvme_softc *sc, 363 struct nvme_completion_queue *cq, 364 uint32_t cdw0, 365 uint16_t cid, 366 uint16_t sqid, 367 uint16_t status); 368 static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); 369 static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); 370 static void pci_nvme_io_done(struct blockif_req *, int); 371 372 /* Controller Configuration utils */ 373 #define NVME_CC_GET_EN(cc) \ 374 ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK) 375 #define NVME_CC_GET_CSS(cc) \ 376 ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK) 377 #define NVME_CC_GET_SHN(cc) \ 378 ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK) 379 #define NVME_CC_GET_IOSQES(cc) \ 380 ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK) 381 #define NVME_CC_GET_IOCQES(cc) \ 382 ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK) 383 384 #define NVME_CC_WRITE_MASK \ 385 ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \ 386 (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \ 387 (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT)) 388 389 #define NVME_CC_NEN_WRITE_MASK \ 390 ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \ 391 (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \ 392 (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT)) 393 394 /* Controller Status utils */ 395 #define NVME_CSTS_GET_RDY(sts) \ 396 ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK) 397 398 #define NVME_CSTS_RDY (1 << NVME_CSTS_REG_RDY_SHIFT) 399 #define NVME_CSTS_CFS (1 << NVME_CSTS_REG_CFS_SHIFT) 400 401 /* Completion Queue status word utils */ 402 #define NVME_STATUS_P (1 << NVME_STATUS_P_SHIFT) 403 #define NVME_STATUS_MASK \ 404 ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\ 405 (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT)) 406 407 #define NVME_ONCS_DSM (NVME_CTRLR_DATA_ONCS_DSM_MASK << \ 408 NVME_CTRLR_DATA_ONCS_DSM_SHIFT) 409 410 static void nvme_feature_invalid_cb(struct pci_nvme_softc *, 411 struct nvme_feature_obj *, 412 struct nvme_command *, 413 struct nvme_completion *); 414 static void nvme_feature_temperature(struct pci_nvme_softc *, 415 struct nvme_feature_obj *, 416 struct nvme_command *, 417 struct nvme_completion *); 418 static void nvme_feature_num_queues(struct pci_nvme_softc *, 419 struct nvme_feature_obj *, 420 struct nvme_command *, 421 struct nvme_completion *); 422 static void nvme_feature_iv_config(struct pci_nvme_softc *, 423 struct nvme_feature_obj *, 424 struct nvme_command *, 425 struct nvme_completion *); 426 static void nvme_feature_async_event(struct pci_nvme_softc *, 427 struct nvme_feature_obj *, 428 struct nvme_command *, 429 struct nvme_completion *); 430 431 static void *aen_thr(void *arg); 432 433 static __inline void 434 cpywithpad(char *dst, size_t dst_size, const char *src, char pad) 435 { 436 size_t len; 437 438 len = strnlen(src, dst_size); 439 memset(dst, pad, dst_size); 440 memcpy(dst, src, len); 441 } 442 443 static __inline void 444 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code) 445 { 446 447 *status &= ~NVME_STATUS_MASK; 448 *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT | 449 (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT; 450 } 451 452 static __inline void 453 pci_nvme_status_genc(uint16_t *status, uint16_t code) 454 { 455 456 pci_nvme_status_tc(status, NVME_SCT_GENERIC, code); 457 } 458 459 /* 460 * Initialize the requested number or IO Submission and Completion Queues. 461 * Admin queues are allocated implicitly. 462 */ 463 static void 464 pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq) 465 { 466 uint32_t i; 467 468 /* 469 * Allocate and initialize the Submission Queues 470 */ 471 if (nsq > NVME_QUEUES) { 472 WPRINTF("%s: clamping number of SQ from %u to %u", 473 __func__, nsq, NVME_QUEUES); 474 nsq = NVME_QUEUES; 475 } 476 477 sc->num_squeues = nsq; 478 479 sc->submit_queues = calloc(sc->num_squeues + 1, 480 sizeof(struct nvme_submission_queue)); 481 if (sc->submit_queues == NULL) { 482 WPRINTF("%s: SQ allocation failed", __func__); 483 sc->num_squeues = 0; 484 } else { 485 struct nvme_submission_queue *sq = sc->submit_queues; 486 487 for (i = 0; i < sc->num_squeues + 1; i++) 488 pthread_mutex_init(&sq[i].mtx, NULL); 489 } 490 491 /* 492 * Allocate and initialize the Completion Queues 493 */ 494 if (ncq > NVME_QUEUES) { 495 WPRINTF("%s: clamping number of CQ from %u to %u", 496 __func__, ncq, NVME_QUEUES); 497 ncq = NVME_QUEUES; 498 } 499 500 sc->num_cqueues = ncq; 501 502 sc->compl_queues = calloc(sc->num_cqueues + 1, 503 sizeof(struct nvme_completion_queue)); 504 if (sc->compl_queues == NULL) { 505 WPRINTF("%s: CQ allocation failed", __func__); 506 sc->num_cqueues = 0; 507 } else { 508 struct nvme_completion_queue *cq = sc->compl_queues; 509 510 for (i = 0; i < sc->num_cqueues + 1; i++) 511 pthread_mutex_init(&cq[i].mtx, NULL); 512 } 513 } 514 515 static void 516 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc) 517 { 518 struct nvme_controller_data *cd = &sc->ctrldata; 519 520 cd->vid = 0xFB5D; 521 cd->ssvid = 0x0000; 522 523 cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' '); 524 cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' '); 525 526 /* Num of submission commands that we can handle at a time (2^rab) */ 527 cd->rab = 4; 528 529 /* FreeBSD OUI */ 530 cd->ieee[0] = 0x58; 531 cd->ieee[1] = 0x9c; 532 cd->ieee[2] = 0xfc; 533 534 cd->mic = 0; 535 536 cd->mdts = NVME_MDTS; /* max data transfer size (2^mdts * CAP.MPSMIN) */ 537 538 cd->ver = NVME_REV(1,4); 539 540 cd->cntrltype = NVME_CNTRLTYPE_IO; 541 cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT; 542 cd->oaes = NVMEB(NVME_CTRLR_DATA_OAES_NS_ATTR); 543 cd->acl = 2; 544 cd->aerl = 4; 545 546 /* Advertise 1, Read-only firmware slot */ 547 cd->frmw = NVMEB(NVME_CTRLR_DATA_FRMW_SLOT1_RO) | 548 (1 << NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT); 549 cd->lpa = 0; /* TODO: support some simple things like SMART */ 550 cd->elpe = 0; /* max error log page entries */ 551 /* 552 * Report a single power state (zero-based value) 553 * power_state[] values are left as zero to indicate "Not reported" 554 */ 555 cd->npss = 0; 556 557 /* Warning Composite Temperature Threshold */ 558 cd->wctemp = 0x0157; 559 cd->cctemp = 0x0157; 560 561 /* SANICAP must not be 0 for Revision 1.4 and later NVMe Controllers */ 562 cd->sanicap = (NVME_CTRLR_DATA_SANICAP_NODMMAS_NO << 563 NVME_CTRLR_DATA_SANICAP_NODMMAS_SHIFT); 564 565 cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) | 566 (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT); 567 cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) | 568 (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT); 569 cd->nn = 1; /* number of namespaces */ 570 571 cd->oncs = 0; 572 switch (sc->dataset_management) { 573 case NVME_DATASET_MANAGEMENT_AUTO: 574 if (sc->nvstore.deallocate) 575 cd->oncs |= NVME_ONCS_DSM; 576 break; 577 case NVME_DATASET_MANAGEMENT_ENABLE: 578 cd->oncs |= NVME_ONCS_DSM; 579 break; 580 default: 581 break; 582 } 583 584 cd->fna = NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK << 585 NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT; 586 587 cd->vwc = NVME_CTRLR_DATA_VWC_ALL_NO << NVME_CTRLR_DATA_VWC_ALL_SHIFT; 588 } 589 590 static void 591 pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore, 592 struct nvme_namespace_data *nd) 593 { 594 595 /* Get capacity and block size information from backing store */ 596 nd->nsze = nvstore->size / nvstore->sectsz; 597 nd->ncap = nd->nsze; 598 nd->nuse = nd->nsze; 599 } 600 601 static void 602 pci_nvme_init_nsdata(struct pci_nvme_softc *sc, 603 struct nvme_namespace_data *nd, uint32_t nsid, 604 struct pci_nvme_blockstore *nvstore) 605 { 606 607 pci_nvme_init_nsdata_size(nvstore, nd); 608 609 if (nvstore->type == NVME_STOR_BLOCKIF) 610 nvstore->deallocate = blockif_candelete(nvstore->ctx); 611 612 nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */ 613 nd->flbas = 0; 614 615 /* Create an EUI-64 if user did not provide one */ 616 if (nvstore->eui64 == 0) { 617 char *data = NULL; 618 uint64_t eui64 = nvstore->eui64; 619 620 asprintf(&data, "%s%u%u%u", get_config_value("name"), 621 sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, 622 sc->nsc_pi->pi_func); 623 624 if (data != NULL) { 625 eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data)); 626 free(data); 627 } 628 nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff); 629 } 630 be64enc(nd->eui64, nvstore->eui64); 631 632 /* LBA data-sz = 2^lbads */ 633 nd->lbaf[0] = nvstore->sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT; 634 } 635 636 static void 637 pci_nvme_init_logpages(struct pci_nvme_softc *sc) 638 { 639 __uint128_t power_cycles = 1; 640 641 memset(&sc->err_log, 0, sizeof(sc->err_log)); 642 memset(&sc->health_log, 0, sizeof(sc->health_log)); 643 memset(&sc->fw_log, 0, sizeof(sc->fw_log)); 644 memset(&sc->ns_log, 0, sizeof(sc->ns_log)); 645 646 /* Set read/write remainder to round up according to spec */ 647 sc->read_dunits_remainder = 999; 648 sc->write_dunits_remainder = 999; 649 650 /* Set nominal Health values checked by implementations */ 651 sc->health_log.temperature = NVME_TEMPERATURE; 652 sc->health_log.available_spare = 100; 653 sc->health_log.available_spare_threshold = 10; 654 655 /* Set Active Firmware Info to slot 1 */ 656 sc->fw_log.afi = (1 << NVME_FIRMWARE_PAGE_AFI_SLOT_SHIFT); 657 memcpy(&sc->fw_log.revision[0], sc->ctrldata.fr, 658 sizeof(sc->fw_log.revision[0])); 659 660 memcpy(&sc->health_log.power_cycles, &power_cycles, 661 sizeof(sc->health_log.power_cycles)); 662 } 663 664 static void 665 pci_nvme_init_features(struct pci_nvme_softc *sc) 666 { 667 enum nvme_feature fid; 668 669 for (fid = 0; fid < NVME_FID_MAX; fid++) { 670 switch (fid) { 671 case NVME_FEAT_ARBITRATION: 672 case NVME_FEAT_POWER_MANAGEMENT: 673 case NVME_FEAT_INTERRUPT_COALESCING: //XXX 674 case NVME_FEAT_WRITE_ATOMICITY: 675 /* Mandatory but no special handling required */ 676 //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: 677 //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 678 // this returns a data buffer 679 break; 680 case NVME_FEAT_TEMPERATURE_THRESHOLD: 681 sc->feat[fid].set = nvme_feature_temperature; 682 break; 683 case NVME_FEAT_ERROR_RECOVERY: 684 sc->feat[fid].namespace_specific = true; 685 break; 686 case NVME_FEAT_NUMBER_OF_QUEUES: 687 sc->feat[fid].set = nvme_feature_num_queues; 688 break; 689 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 690 sc->feat[fid].set = nvme_feature_iv_config; 691 break; 692 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 693 sc->feat[fid].set = nvme_feature_async_event; 694 /* Enable all AENs by default */ 695 sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK; 696 break; 697 default: 698 sc->feat[fid].set = nvme_feature_invalid_cb; 699 sc->feat[fid].get = nvme_feature_invalid_cb; 700 } 701 } 702 } 703 704 static void 705 pci_nvme_aer_reset(struct pci_nvme_softc *sc) 706 { 707 708 STAILQ_INIT(&sc->aer_list); 709 sc->aer_count = 0; 710 } 711 712 static void 713 pci_nvme_aer_init(struct pci_nvme_softc *sc) 714 { 715 716 pthread_mutex_init(&sc->aer_mtx, NULL); 717 pci_nvme_aer_reset(sc); 718 } 719 720 static void 721 pci_nvme_aer_destroy(struct pci_nvme_softc *sc) 722 { 723 struct pci_nvme_aer *aer = NULL; 724 725 pthread_mutex_lock(&sc->aer_mtx); 726 while (!STAILQ_EMPTY(&sc->aer_list)) { 727 aer = STAILQ_FIRST(&sc->aer_list); 728 STAILQ_REMOVE_HEAD(&sc->aer_list, link); 729 free(aer); 730 } 731 pthread_mutex_unlock(&sc->aer_mtx); 732 733 pci_nvme_aer_reset(sc); 734 } 735 736 static bool 737 pci_nvme_aer_available(struct pci_nvme_softc *sc) 738 { 739 740 return (sc->aer_count != 0); 741 } 742 743 static bool 744 pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc) 745 { 746 struct nvme_controller_data *cd = &sc->ctrldata; 747 748 /* AERL is a zero based value while aer_count is one's based */ 749 return (sc->aer_count == (cd->aerl + 1U)); 750 } 751 752 /* 753 * Add an Async Event Request 754 * 755 * Stores an AER to be returned later if the Controller needs to notify the 756 * host of an event. 757 * Note that while the NVMe spec doesn't require Controllers to return AER's 758 * in order, this implementation does preserve the order. 759 */ 760 static int 761 pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid) 762 { 763 struct pci_nvme_aer *aer = NULL; 764 765 aer = calloc(1, sizeof(struct pci_nvme_aer)); 766 if (aer == NULL) 767 return (-1); 768 769 /* Save the Command ID for use in the completion message */ 770 aer->cid = cid; 771 772 pthread_mutex_lock(&sc->aer_mtx); 773 sc->aer_count++; 774 STAILQ_INSERT_TAIL(&sc->aer_list, aer, link); 775 pthread_mutex_unlock(&sc->aer_mtx); 776 777 return (0); 778 } 779 780 /* 781 * Get an Async Event Request structure 782 * 783 * Returns a pointer to an AER previously submitted by the host or NULL if 784 * no AER's exist. Caller is responsible for freeing the returned struct. 785 */ 786 static struct pci_nvme_aer * 787 pci_nvme_aer_get(struct pci_nvme_softc *sc) 788 { 789 struct pci_nvme_aer *aer = NULL; 790 791 pthread_mutex_lock(&sc->aer_mtx); 792 aer = STAILQ_FIRST(&sc->aer_list); 793 if (aer != NULL) { 794 STAILQ_REMOVE_HEAD(&sc->aer_list, link); 795 sc->aer_count--; 796 } 797 pthread_mutex_unlock(&sc->aer_mtx); 798 799 return (aer); 800 } 801 802 static void 803 pci_nvme_aen_reset(struct pci_nvme_softc *sc) 804 { 805 uint32_t atype; 806 807 memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen)); 808 809 for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { 810 sc->aen[atype].atype = atype; 811 } 812 } 813 814 static void 815 pci_nvme_aen_init(struct pci_nvme_softc *sc) 816 { 817 char nstr[80]; 818 819 pci_nvme_aen_reset(sc); 820 821 pthread_mutex_init(&sc->aen_mtx, NULL); 822 pthread_create(&sc->aen_tid, NULL, aen_thr, sc); 823 snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot, 824 sc->nsc_pi->pi_func); 825 pthread_set_name_np(sc->aen_tid, nstr); 826 } 827 828 static void 829 pci_nvme_aen_destroy(struct pci_nvme_softc *sc) 830 { 831 832 pci_nvme_aen_reset(sc); 833 } 834 835 /* Notify the AEN thread of pending work */ 836 static void 837 pci_nvme_aen_notify(struct pci_nvme_softc *sc) 838 { 839 840 pthread_cond_signal(&sc->aen_cond); 841 } 842 843 /* 844 * Post an Asynchronous Event Notification 845 */ 846 static int32_t 847 pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype, 848 uint32_t event_data) 849 { 850 struct pci_nvme_aen *aen; 851 852 if (atype >= PCI_NVME_AE_TYPE_MAX) { 853 return(EINVAL); 854 } 855 856 pthread_mutex_lock(&sc->aen_mtx); 857 aen = &sc->aen[atype]; 858 859 /* Has the controller already posted an event of this type? */ 860 if (aen->posted) { 861 pthread_mutex_unlock(&sc->aen_mtx); 862 return(EALREADY); 863 } 864 865 aen->event_data = event_data; 866 aen->posted = true; 867 pthread_mutex_unlock(&sc->aen_mtx); 868 869 pci_nvme_aen_notify(sc); 870 871 return(0); 872 } 873 874 static void 875 pci_nvme_aen_process(struct pci_nvme_softc *sc) 876 { 877 struct pci_nvme_aer *aer; 878 struct pci_nvme_aen *aen; 879 pci_nvme_async_type atype; 880 uint32_t mask; 881 uint16_t status; 882 uint8_t lid; 883 884 assert(pthread_mutex_isowned_np(&sc->aen_mtx)); 885 for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { 886 aen = &sc->aen[atype]; 887 /* Previous iterations may have depleted the available AER's */ 888 if (!pci_nvme_aer_available(sc)) { 889 DPRINTF("%s: no AER", __func__); 890 break; 891 } 892 893 if (!aen->posted) { 894 DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype); 895 continue; 896 } 897 898 status = NVME_SC_SUCCESS; 899 900 /* Is the event masked? */ 901 mask = 902 sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11; 903 904 DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data); 905 switch (atype) { 906 case PCI_NVME_AE_TYPE_ERROR: 907 lid = NVME_LOG_ERROR; 908 break; 909 case PCI_NVME_AE_TYPE_SMART: 910 mask &= 0xff; 911 if ((mask & aen->event_data) == 0) 912 continue; 913 lid = NVME_LOG_HEALTH_INFORMATION; 914 break; 915 case PCI_NVME_AE_TYPE_NOTICE: 916 if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) { 917 EPRINTLN("%s unknown AEN notice type %u", 918 __func__, aen->event_data); 919 status = NVME_SC_INTERNAL_DEVICE_ERROR; 920 lid = 0; 921 break; 922 } 923 if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0) 924 continue; 925 switch (aen->event_data) { 926 case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED: 927 lid = NVME_LOG_CHANGED_NAMESPACE; 928 break; 929 case PCI_NVME_AEI_NOTICE_FW_ACTIVATION: 930 lid = NVME_LOG_FIRMWARE_SLOT; 931 break; 932 case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE: 933 lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED; 934 break; 935 case PCI_NVME_AEI_NOTICE_ANA_CHANGE: 936 lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; 937 break; 938 case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE: 939 lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE; 940 break; 941 case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT: 942 lid = NVME_LOG_LBA_STATUS_INFORMATION; 943 break; 944 case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE: 945 lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE; 946 break; 947 default: 948 lid = 0; 949 } 950 break; 951 default: 952 /* bad type?!? */ 953 EPRINTLN("%s unknown AEN type %u", __func__, atype); 954 status = NVME_SC_INTERNAL_DEVICE_ERROR; 955 lid = 0; 956 break; 957 } 958 959 aer = pci_nvme_aer_get(sc); 960 assert(aer != NULL); 961 962 DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype); 963 pci_nvme_cq_update(sc, &sc->compl_queues[0], 964 (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */ 965 aer->cid, 966 0, /* SQID */ 967 status); 968 969 aen->event_data = 0; 970 aen->posted = false; 971 972 pci_generate_msix(sc->nsc_pi, 0); 973 } 974 } 975 976 static void * 977 aen_thr(void *arg) 978 { 979 struct pci_nvme_softc *sc; 980 981 sc = arg; 982 983 pthread_mutex_lock(&sc->aen_mtx); 984 for (;;) { 985 pci_nvme_aen_process(sc); 986 pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx); 987 } 988 pthread_mutex_unlock(&sc->aen_mtx); 989 990 pthread_exit(NULL); 991 return (NULL); 992 } 993 994 static void 995 pci_nvme_reset_locked(struct pci_nvme_softc *sc) 996 { 997 uint32_t i; 998 999 DPRINTF("%s", __func__); 1000 1001 sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | 1002 (1 << NVME_CAP_LO_REG_CQR_SHIFT) | 1003 (60 << NVME_CAP_LO_REG_TO_SHIFT); 1004 1005 sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT; 1006 1007 sc->regs.vs = NVME_REV(1,4); /* NVMe v1.4 */ 1008 1009 sc->regs.cc = 0; 1010 1011 assert(sc->submit_queues != NULL); 1012 1013 for (i = 0; i < sc->num_squeues + 1; i++) { 1014 sc->submit_queues[i].qbase = NULL; 1015 sc->submit_queues[i].size = 0; 1016 sc->submit_queues[i].cqid = 0; 1017 sc->submit_queues[i].tail = 0; 1018 sc->submit_queues[i].head = 0; 1019 } 1020 1021 assert(sc->compl_queues != NULL); 1022 1023 for (i = 0; i < sc->num_cqueues + 1; i++) { 1024 sc->compl_queues[i].qbase = NULL; 1025 sc->compl_queues[i].size = 0; 1026 sc->compl_queues[i].tail = 0; 1027 sc->compl_queues[i].head = 0; 1028 } 1029 1030 sc->num_q_is_set = false; 1031 1032 pci_nvme_aer_destroy(sc); 1033 pci_nvme_aen_destroy(sc); 1034 1035 /* 1036 * Clear CSTS.RDY last to prevent the host from enabling Controller 1037 * before cleanup completes 1038 */ 1039 sc->regs.csts = 0; 1040 } 1041 1042 static void 1043 pci_nvme_reset(struct pci_nvme_softc *sc) 1044 { 1045 pthread_mutex_lock(&sc->mtx); 1046 pci_nvme_reset_locked(sc); 1047 pthread_mutex_unlock(&sc->mtx); 1048 } 1049 1050 static int 1051 pci_nvme_init_controller(struct pci_nvme_softc *sc) 1052 { 1053 uint16_t acqs, asqs; 1054 1055 DPRINTF("%s", __func__); 1056 1057 /* 1058 * NVMe 2.0 states that "enabling a controller while this field is 1059 * cleared to 0h produces undefined results" for both ACQS and 1060 * ASQS. If zero, set CFS and do not become ready. 1061 */ 1062 asqs = ONE_BASED(sc->regs.aqa & NVME_AQA_REG_ASQS_MASK); 1063 if (asqs < 2) { 1064 EPRINTLN("%s: illegal ASQS value %#x (aqa=%#x)", __func__, 1065 asqs - 1, sc->regs.aqa); 1066 sc->regs.csts |= NVME_CSTS_CFS; 1067 return (-1); 1068 } 1069 sc->submit_queues[0].size = asqs; 1070 sc->submit_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, 1071 sc->regs.asq, sizeof(struct nvme_command) * asqs); 1072 if (sc->submit_queues[0].qbase == NULL) { 1073 EPRINTLN("%s: ASQ vm_map_gpa(%lx) failed", __func__, 1074 sc->regs.asq); 1075 sc->regs.csts |= NVME_CSTS_CFS; 1076 return (-1); 1077 } 1078 1079 DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p", 1080 __func__, sc->regs.asq, sc->submit_queues[0].qbase); 1081 1082 acqs = ONE_BASED((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) & 1083 NVME_AQA_REG_ACQS_MASK); 1084 if (acqs < 2) { 1085 EPRINTLN("%s: illegal ACQS value %#x (aqa=%#x)", __func__, 1086 acqs - 1, sc->regs.aqa); 1087 sc->regs.csts |= NVME_CSTS_CFS; 1088 return (-1); 1089 } 1090 sc->compl_queues[0].size = acqs; 1091 sc->compl_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, 1092 sc->regs.acq, sizeof(struct nvme_completion) * acqs); 1093 if (sc->compl_queues[0].qbase == NULL) { 1094 EPRINTLN("%s: ACQ vm_map_gpa(%lx) failed", __func__, 1095 sc->regs.acq); 1096 sc->regs.csts |= NVME_CSTS_CFS; 1097 return (-1); 1098 } 1099 sc->compl_queues[0].intr_en = NVME_CQ_INTEN; 1100 1101 DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p", 1102 __func__, sc->regs.acq, sc->compl_queues[0].qbase); 1103 1104 return (0); 1105 } 1106 1107 static int 1108 nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b, 1109 size_t len, enum nvme_copy_dir dir) 1110 { 1111 uint8_t *p; 1112 size_t bytes; 1113 1114 if (len > (8 * 1024)) { 1115 return (-1); 1116 } 1117 1118 /* Copy from the start of prp1 to the end of the physical page */ 1119 bytes = PAGE_SIZE - (prp1 & PAGE_MASK); 1120 bytes = MIN(bytes, len); 1121 1122 p = vm_map_gpa(ctx, prp1, bytes); 1123 if (p == NULL) { 1124 return (-1); 1125 } 1126 1127 if (dir == NVME_COPY_TO_PRP) 1128 memcpy(p, b, bytes); 1129 else 1130 memcpy(b, p, bytes); 1131 1132 b += bytes; 1133 1134 len -= bytes; 1135 if (len == 0) { 1136 return (0); 1137 } 1138 1139 len = MIN(len, PAGE_SIZE); 1140 1141 p = vm_map_gpa(ctx, prp2, len); 1142 if (p == NULL) { 1143 return (-1); 1144 } 1145 1146 if (dir == NVME_COPY_TO_PRP) 1147 memcpy(p, b, len); 1148 else 1149 memcpy(b, p, len); 1150 1151 return (0); 1152 } 1153 1154 /* 1155 * Write a Completion Queue Entry update 1156 * 1157 * Write the completion and update the doorbell value 1158 */ 1159 static void 1160 pci_nvme_cq_update(struct pci_nvme_softc *sc, 1161 struct nvme_completion_queue *cq, 1162 uint32_t cdw0, 1163 uint16_t cid, 1164 uint16_t sqid, 1165 uint16_t status) 1166 { 1167 struct nvme_submission_queue *sq = &sc->submit_queues[sqid]; 1168 struct nvme_completion *cqe; 1169 1170 assert(cq->qbase != NULL); 1171 1172 pthread_mutex_lock(&cq->mtx); 1173 1174 cqe = &cq->qbase[cq->tail]; 1175 1176 /* Flip the phase bit */ 1177 status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK; 1178 1179 cqe->cdw0 = cdw0; 1180 cqe->sqhd = sq->head; 1181 cqe->sqid = sqid; 1182 cqe->cid = cid; 1183 cqe->status = status; 1184 1185 cq->tail++; 1186 if (cq->tail >= cq->size) { 1187 cq->tail = 0; 1188 } 1189 1190 pthread_mutex_unlock(&cq->mtx); 1191 } 1192 1193 static int 1194 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 1195 struct nvme_completion* compl) 1196 { 1197 uint16_t qid = command->cdw10 & 0xffff; 1198 1199 DPRINTF("%s DELETE_IO_SQ %u", __func__, qid); 1200 if (qid == 0 || qid > sc->num_squeues || 1201 (sc->submit_queues[qid].qbase == NULL)) { 1202 WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u", 1203 __func__, qid, sc->num_squeues); 1204 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1205 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1206 return (1); 1207 } 1208 1209 sc->submit_queues[qid].qbase = NULL; 1210 sc->submit_queues[qid].cqid = 0; 1211 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1212 return (1); 1213 } 1214 1215 static int 1216 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 1217 struct nvme_completion* compl) 1218 { 1219 if (command->cdw11 & NVME_CMD_CDW11_PC) { 1220 uint16_t qid = command->cdw10 & 0xffff; 1221 struct nvme_submission_queue *nsq; 1222 1223 if ((qid == 0) || (qid > sc->num_squeues) || 1224 (sc->submit_queues[qid].qbase != NULL)) { 1225 WPRINTF("%s queue index %u > num_squeues %u", 1226 __func__, qid, sc->num_squeues); 1227 pci_nvme_status_tc(&compl->status, 1228 NVME_SCT_COMMAND_SPECIFIC, 1229 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1230 return (1); 1231 } 1232 1233 nsq = &sc->submit_queues[qid]; 1234 nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 1235 DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries); 1236 if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) { 1237 /* 1238 * Queues must specify at least two entries 1239 * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to 1240 * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec 1241 */ 1242 pci_nvme_status_tc(&compl->status, 1243 NVME_SCT_COMMAND_SPECIFIC, 1244 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); 1245 return (1); 1246 } 1247 nsq->head = nsq->tail = 0; 1248 1249 nsq->cqid = (command->cdw11 >> 16) & 0xffff; 1250 if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) { 1251 pci_nvme_status_tc(&compl->status, 1252 NVME_SCT_COMMAND_SPECIFIC, 1253 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1254 return (1); 1255 } 1256 1257 if (sc->compl_queues[nsq->cqid].qbase == NULL) { 1258 pci_nvme_status_tc(&compl->status, 1259 NVME_SCT_COMMAND_SPECIFIC, 1260 NVME_SC_COMPLETION_QUEUE_INVALID); 1261 return (1); 1262 } 1263 1264 nsq->qpriority = (command->cdw11 >> 1) & 0x03; 1265 1266 nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1267 sizeof(struct nvme_command) * (size_t)nsq->size); 1268 1269 DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__, 1270 qid, nsq->size, nsq->qbase, nsq->cqid); 1271 1272 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1273 1274 DPRINTF("%s completed creating IOSQ qid %u", 1275 __func__, qid); 1276 } else { 1277 /* 1278 * Guest sent non-cont submission queue request. 1279 * This setting is unsupported by this emulation. 1280 */ 1281 WPRINTF("%s unsupported non-contig (list-based) " 1282 "create i/o submission queue", __func__); 1283 1284 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1285 } 1286 return (1); 1287 } 1288 1289 static int 1290 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 1291 struct nvme_completion* compl) 1292 { 1293 uint16_t qid = command->cdw10 & 0xffff; 1294 uint16_t sqid; 1295 1296 DPRINTF("%s DELETE_IO_CQ %u", __func__, qid); 1297 if (qid == 0 || qid > sc->num_cqueues || 1298 (sc->compl_queues[qid].qbase == NULL)) { 1299 WPRINTF("%s queue index %u / num_cqueues %u", 1300 __func__, qid, sc->num_cqueues); 1301 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1302 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1303 return (1); 1304 } 1305 1306 /* Deleting an Active CQ is an error */ 1307 for (sqid = 1; sqid < sc->num_squeues + 1; sqid++) 1308 if (sc->submit_queues[sqid].cqid == qid) { 1309 pci_nvme_status_tc(&compl->status, 1310 NVME_SCT_COMMAND_SPECIFIC, 1311 NVME_SC_INVALID_QUEUE_DELETION); 1312 return (1); 1313 } 1314 1315 sc->compl_queues[qid].qbase = NULL; 1316 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1317 return (1); 1318 } 1319 1320 static int 1321 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 1322 struct nvme_completion* compl) 1323 { 1324 struct nvme_completion_queue *ncq; 1325 uint16_t qid = command->cdw10 & 0xffff; 1326 1327 /* Only support Physically Contiguous queues */ 1328 if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) { 1329 WPRINTF("%s unsupported non-contig (list-based) " 1330 "create i/o completion queue", 1331 __func__); 1332 1333 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1334 return (1); 1335 } 1336 1337 if ((qid == 0) || (qid > sc->num_cqueues) || 1338 (sc->compl_queues[qid].qbase != NULL)) { 1339 WPRINTF("%s queue index %u > num_cqueues %u", 1340 __func__, qid, sc->num_cqueues); 1341 pci_nvme_status_tc(&compl->status, 1342 NVME_SCT_COMMAND_SPECIFIC, 1343 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1344 return (1); 1345 } 1346 1347 ncq = &sc->compl_queues[qid]; 1348 ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1; 1349 ncq->intr_vec = (command->cdw11 >> 16) & 0xffff; 1350 if (ncq->intr_vec > (sc->max_queues + 1)) { 1351 pci_nvme_status_tc(&compl->status, 1352 NVME_SCT_COMMAND_SPECIFIC, 1353 NVME_SC_INVALID_INTERRUPT_VECTOR); 1354 return (1); 1355 } 1356 1357 ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 1358 if ((ncq->size < 2) || (ncq->size > sc->max_qentries)) { 1359 /* 1360 * Queues must specify at least two entries 1361 * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to 1362 * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec 1363 */ 1364 pci_nvme_status_tc(&compl->status, 1365 NVME_SCT_COMMAND_SPECIFIC, 1366 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); 1367 return (1); 1368 } 1369 ncq->head = ncq->tail = 0; 1370 ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, 1371 command->prp1, 1372 sizeof(struct nvme_command) * (size_t)ncq->size); 1373 1374 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1375 1376 1377 return (1); 1378 } 1379 1380 static int 1381 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command, 1382 struct nvme_completion* compl) 1383 { 1384 uint64_t logoff; 1385 uint32_t logsize; 1386 uint8_t logpage; 1387 1388 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1389 1390 /* 1391 * Command specifies the number of dwords to return in fields NUMDU 1392 * and NUMDL. This is a zero-based value. 1393 */ 1394 logpage = command->cdw10 & 0xFF; 1395 logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1; 1396 logsize *= sizeof(uint32_t); 1397 logoff = ((uint64_t)(command->cdw13) << 32) | command->cdw12; 1398 1399 DPRINTF("%s log page %u len %u", __func__, logpage, logsize); 1400 1401 switch (logpage) { 1402 case NVME_LOG_ERROR: 1403 if (logoff >= sizeof(sc->err_log)) { 1404 pci_nvme_status_genc(&compl->status, 1405 NVME_SC_INVALID_FIELD); 1406 break; 1407 } 1408 1409 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1410 command->prp2, (uint8_t *)&sc->err_log + logoff, 1411 MIN(logsize - logoff, sizeof(sc->err_log)), 1412 NVME_COPY_TO_PRP); 1413 break; 1414 case NVME_LOG_HEALTH_INFORMATION: 1415 if (logoff >= sizeof(sc->health_log)) { 1416 pci_nvme_status_genc(&compl->status, 1417 NVME_SC_INVALID_FIELD); 1418 break; 1419 } 1420 1421 pthread_mutex_lock(&sc->mtx); 1422 memcpy(&sc->health_log.data_units_read, &sc->read_data_units, 1423 sizeof(sc->health_log.data_units_read)); 1424 memcpy(&sc->health_log.data_units_written, &sc->write_data_units, 1425 sizeof(sc->health_log.data_units_written)); 1426 memcpy(&sc->health_log.host_read_commands, &sc->read_commands, 1427 sizeof(sc->health_log.host_read_commands)); 1428 memcpy(&sc->health_log.host_write_commands, &sc->write_commands, 1429 sizeof(sc->health_log.host_write_commands)); 1430 pthread_mutex_unlock(&sc->mtx); 1431 1432 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1433 command->prp2, (uint8_t *)&sc->health_log + logoff, 1434 MIN(logsize - logoff, sizeof(sc->health_log)), 1435 NVME_COPY_TO_PRP); 1436 break; 1437 case NVME_LOG_FIRMWARE_SLOT: 1438 if (logoff >= sizeof(sc->fw_log)) { 1439 pci_nvme_status_genc(&compl->status, 1440 NVME_SC_INVALID_FIELD); 1441 break; 1442 } 1443 1444 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1445 command->prp2, (uint8_t *)&sc->fw_log + logoff, 1446 MIN(logsize - logoff, sizeof(sc->fw_log)), 1447 NVME_COPY_TO_PRP); 1448 break; 1449 case NVME_LOG_CHANGED_NAMESPACE: 1450 if (logoff >= sizeof(sc->ns_log)) { 1451 pci_nvme_status_genc(&compl->status, 1452 NVME_SC_INVALID_FIELD); 1453 break; 1454 } 1455 1456 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1457 command->prp2, (uint8_t *)&sc->ns_log + logoff, 1458 MIN(logsize - logoff, sizeof(sc->ns_log)), 1459 NVME_COPY_TO_PRP); 1460 memset(&sc->ns_log, 0, sizeof(sc->ns_log)); 1461 break; 1462 default: 1463 DPRINTF("%s get log page %x command not supported", 1464 __func__, logpage); 1465 1466 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1467 NVME_SC_INVALID_LOG_PAGE); 1468 } 1469 1470 return (1); 1471 } 1472 1473 static int 1474 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command, 1475 struct nvme_completion* compl) 1476 { 1477 void *dest; 1478 uint16_t status; 1479 1480 DPRINTF("%s identify 0x%x nsid 0x%x", __func__, 1481 command->cdw10 & 0xFF, command->nsid); 1482 1483 status = 0; 1484 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 1485 1486 switch (command->cdw10 & 0xFF) { 1487 case 0x00: /* return Identify Namespace data structure */ 1488 /* Global NS only valid with NS Management */ 1489 if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) { 1490 pci_nvme_status_genc(&status, 1491 NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 1492 break; 1493 } 1494 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1495 command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata), 1496 NVME_COPY_TO_PRP); 1497 break; 1498 case 0x01: /* return Identify Controller data structure */ 1499 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1500 command->prp2, (uint8_t *)&sc->ctrldata, 1501 sizeof(sc->ctrldata), 1502 NVME_COPY_TO_PRP); 1503 break; 1504 case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */ 1505 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1506 sizeof(uint32_t) * 1024); 1507 /* All unused entries shall be zero */ 1508 memset(dest, 0, sizeof(uint32_t) * 1024); 1509 ((uint32_t *)dest)[0] = 1; 1510 break; 1511 case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */ 1512 if (command->nsid != 1) { 1513 pci_nvme_status_genc(&status, 1514 NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 1515 break; 1516 } 1517 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1518 sizeof(uint32_t) * 1024); 1519 /* All bytes after the descriptor shall be zero */ 1520 memset(dest, 0, sizeof(uint32_t) * 1024); 1521 1522 /* Return NIDT=1 (i.e. EUI64) descriptor */ 1523 ((uint8_t *)dest)[0] = 1; 1524 ((uint8_t *)dest)[1] = sizeof(uint64_t); 1525 memcpy(((uint8_t *)dest) + 4, sc->nsdata.eui64, sizeof(uint64_t)); 1526 break; 1527 case 0x13: 1528 /* 1529 * Controller list is optional but used by UNH tests. Return 1530 * a valid but empty list. 1531 */ 1532 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1533 sizeof(uint16_t) * 2048); 1534 memset(dest, 0, sizeof(uint16_t) * 2048); 1535 break; 1536 default: 1537 DPRINTF("%s unsupported identify command requested 0x%x", 1538 __func__, command->cdw10 & 0xFF); 1539 pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD); 1540 break; 1541 } 1542 1543 compl->status = status; 1544 return (1); 1545 } 1546 1547 static const char * 1548 nvme_fid_to_name(uint8_t fid) 1549 { 1550 const char *name; 1551 1552 switch (fid) { 1553 case NVME_FEAT_ARBITRATION: 1554 name = "Arbitration"; 1555 break; 1556 case NVME_FEAT_POWER_MANAGEMENT: 1557 name = "Power Management"; 1558 break; 1559 case NVME_FEAT_LBA_RANGE_TYPE: 1560 name = "LBA Range Type"; 1561 break; 1562 case NVME_FEAT_TEMPERATURE_THRESHOLD: 1563 name = "Temperature Threshold"; 1564 break; 1565 case NVME_FEAT_ERROR_RECOVERY: 1566 name = "Error Recovery"; 1567 break; 1568 case NVME_FEAT_VOLATILE_WRITE_CACHE: 1569 name = "Volatile Write Cache"; 1570 break; 1571 case NVME_FEAT_NUMBER_OF_QUEUES: 1572 name = "Number of Queues"; 1573 break; 1574 case NVME_FEAT_INTERRUPT_COALESCING: 1575 name = "Interrupt Coalescing"; 1576 break; 1577 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 1578 name = "Interrupt Vector Configuration"; 1579 break; 1580 case NVME_FEAT_WRITE_ATOMICITY: 1581 name = "Write Atomicity Normal"; 1582 break; 1583 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 1584 name = "Asynchronous Event Configuration"; 1585 break; 1586 case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION: 1587 name = "Autonomous Power State Transition"; 1588 break; 1589 case NVME_FEAT_HOST_MEMORY_BUFFER: 1590 name = "Host Memory Buffer"; 1591 break; 1592 case NVME_FEAT_TIMESTAMP: 1593 name = "Timestamp"; 1594 break; 1595 case NVME_FEAT_KEEP_ALIVE_TIMER: 1596 name = "Keep Alive Timer"; 1597 break; 1598 case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT: 1599 name = "Host Controlled Thermal Management"; 1600 break; 1601 case NVME_FEAT_NON_OP_POWER_STATE_CONFIG: 1602 name = "Non-Operation Power State Config"; 1603 break; 1604 case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG: 1605 name = "Read Recovery Level Config"; 1606 break; 1607 case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: 1608 name = "Predictable Latency Mode Config"; 1609 break; 1610 case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW: 1611 name = "Predictable Latency Mode Window"; 1612 break; 1613 case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES: 1614 name = "LBA Status Information Report Interval"; 1615 break; 1616 case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 1617 name = "Host Behavior Support"; 1618 break; 1619 case NVME_FEAT_SANITIZE_CONFIG: 1620 name = "Sanitize Config"; 1621 break; 1622 case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION: 1623 name = "Endurance Group Event Configuration"; 1624 break; 1625 case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: 1626 name = "Software Progress Marker"; 1627 break; 1628 case NVME_FEAT_HOST_IDENTIFIER: 1629 name = "Host Identifier"; 1630 break; 1631 case NVME_FEAT_RESERVATION_NOTIFICATION_MASK: 1632 name = "Reservation Notification Mask"; 1633 break; 1634 case NVME_FEAT_RESERVATION_PERSISTENCE: 1635 name = "Reservation Persistence"; 1636 break; 1637 case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG: 1638 name = "Namespace Write Protection Config"; 1639 break; 1640 default: 1641 name = "Unknown"; 1642 break; 1643 } 1644 1645 return (name); 1646 } 1647 1648 static void 1649 nvme_feature_invalid_cb(struct pci_nvme_softc *sc __unused, 1650 struct nvme_feature_obj *feat __unused, 1651 struct nvme_command *command __unused, 1652 struct nvme_completion *compl) 1653 { 1654 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1655 } 1656 1657 static void 1658 nvme_feature_iv_config(struct pci_nvme_softc *sc, 1659 struct nvme_feature_obj *feat __unused, 1660 struct nvme_command *command, 1661 struct nvme_completion *compl) 1662 { 1663 uint32_t i; 1664 uint32_t cdw11 = command->cdw11; 1665 uint16_t iv; 1666 bool cd; 1667 1668 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1669 1670 iv = cdw11 & 0xffff; 1671 cd = cdw11 & (1 << 16); 1672 1673 if (iv > (sc->max_queues + 1)) { 1674 return; 1675 } 1676 1677 /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */ 1678 if ((iv == 0) && !cd) 1679 return; 1680 1681 /* Requested Interrupt Vector must be used by a CQ */ 1682 for (i = 0; i < sc->num_cqueues + 1; i++) { 1683 if (sc->compl_queues[i].intr_vec == iv) { 1684 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1685 } 1686 } 1687 } 1688 1689 #define NVME_ASYNC_EVENT_ENDURANCE_GROUP (0x4000) 1690 static void 1691 nvme_feature_async_event(struct pci_nvme_softc *sc __unused, 1692 struct nvme_feature_obj *feat __unused, 1693 struct nvme_command *command, 1694 struct nvme_completion *compl) 1695 { 1696 if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP) 1697 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1698 } 1699 1700 #define NVME_TEMP_THRESH_OVER 0 1701 #define NVME_TEMP_THRESH_UNDER 1 1702 static void 1703 nvme_feature_temperature(struct pci_nvme_softc *sc, 1704 struct nvme_feature_obj *feat __unused, 1705 struct nvme_command *command, 1706 struct nvme_completion *compl) 1707 { 1708 uint16_t tmpth; /* Temperature Threshold */ 1709 uint8_t tmpsel; /* Threshold Temperature Select */ 1710 uint8_t thsel; /* Threshold Type Select */ 1711 bool set_crit = false; 1712 bool report_crit; 1713 1714 tmpth = command->cdw11 & 0xffff; 1715 tmpsel = (command->cdw11 >> 16) & 0xf; 1716 thsel = (command->cdw11 >> 20) & 0x3; 1717 1718 DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel); 1719 1720 /* Check for unsupported values */ 1721 if (((tmpsel != 0) && (tmpsel != 0xf)) || 1722 (thsel > NVME_TEMP_THRESH_UNDER)) { 1723 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1724 return; 1725 } 1726 1727 if (((thsel == NVME_TEMP_THRESH_OVER) && (NVME_TEMPERATURE >= tmpth)) || 1728 ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth))) 1729 set_crit = true; 1730 1731 pthread_mutex_lock(&sc->mtx); 1732 if (set_crit) 1733 sc->health_log.critical_warning |= 1734 NVME_CRIT_WARN_ST_TEMPERATURE; 1735 else 1736 sc->health_log.critical_warning &= 1737 ~NVME_CRIT_WARN_ST_TEMPERATURE; 1738 pthread_mutex_unlock(&sc->mtx); 1739 1740 report_crit = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 & 1741 NVME_CRIT_WARN_ST_TEMPERATURE; 1742 1743 if (set_crit && report_crit) 1744 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART, 1745 sc->health_log.critical_warning); 1746 1747 DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status); 1748 } 1749 1750 static void 1751 nvme_feature_num_queues(struct pci_nvme_softc *sc, 1752 struct nvme_feature_obj *feat __unused, 1753 struct nvme_command *command, 1754 struct nvme_completion *compl) 1755 { 1756 uint16_t nqr; /* Number of Queues Requested */ 1757 1758 if (sc->num_q_is_set) { 1759 WPRINTF("%s: Number of Queues already set", __func__); 1760 pci_nvme_status_genc(&compl->status, 1761 NVME_SC_COMMAND_SEQUENCE_ERROR); 1762 return; 1763 } 1764 1765 nqr = command->cdw11 & 0xFFFF; 1766 if (nqr == 0xffff) { 1767 WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr); 1768 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1769 return; 1770 } 1771 1772 sc->num_squeues = ONE_BASED(nqr); 1773 if (sc->num_squeues > sc->max_queues) { 1774 DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues, 1775 sc->max_queues); 1776 sc->num_squeues = sc->max_queues; 1777 } 1778 1779 nqr = (command->cdw11 >> 16) & 0xFFFF; 1780 if (nqr == 0xffff) { 1781 WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr); 1782 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1783 return; 1784 } 1785 1786 sc->num_cqueues = ONE_BASED(nqr); 1787 if (sc->num_cqueues > sc->max_queues) { 1788 DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues, 1789 sc->max_queues); 1790 sc->num_cqueues = sc->max_queues; 1791 } 1792 1793 /* Patch the command value which will be saved on callback's return */ 1794 command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc); 1795 compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); 1796 1797 sc->num_q_is_set = true; 1798 } 1799 1800 static int 1801 nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command, 1802 struct nvme_completion *compl) 1803 { 1804 struct nvme_feature_obj *feat; 1805 uint32_t nsid = command->nsid; 1806 uint8_t fid = NVMEV(NVME_FEAT_SET_FID, command->cdw10); 1807 bool sv = NVMEV(NVME_FEAT_SET_SV, command->cdw10); 1808 1809 DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); 1810 1811 if (fid >= NVME_FID_MAX) { 1812 DPRINTF("%s invalid feature 0x%x", __func__, fid); 1813 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1814 return (1); 1815 } 1816 1817 if (sv) { 1818 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1819 NVME_SC_FEATURE_NOT_SAVEABLE); 1820 return (1); 1821 } 1822 1823 feat = &sc->feat[fid]; 1824 1825 if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) { 1826 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1827 return (1); 1828 } 1829 1830 if (!feat->namespace_specific && 1831 !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) { 1832 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1833 NVME_SC_FEATURE_NOT_NS_SPECIFIC); 1834 return (1); 1835 } 1836 1837 compl->cdw0 = 0; 1838 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1839 1840 if (feat->set) 1841 feat->set(sc, feat, command, compl); 1842 else { 1843 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1844 NVME_SC_FEATURE_NOT_CHANGEABLE); 1845 return (1); 1846 } 1847 1848 DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11); 1849 if (compl->status == NVME_SC_SUCCESS) { 1850 feat->cdw11 = command->cdw11; 1851 if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) && 1852 (command->cdw11 != 0)) 1853 pci_nvme_aen_notify(sc); 1854 } 1855 1856 return (0); 1857 } 1858 1859 #define NVME_FEATURES_SEL_SUPPORTED 0x3 1860 #define NVME_FEATURES_NS_SPECIFIC (1 << 1) 1861 1862 static int 1863 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command, 1864 struct nvme_completion* compl) 1865 { 1866 struct nvme_feature_obj *feat; 1867 uint8_t fid = command->cdw10 & 0xFF; 1868 uint8_t sel = (command->cdw10 >> 8) & 0x7; 1869 1870 DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); 1871 1872 if (fid >= NVME_FID_MAX) { 1873 DPRINTF("%s invalid feature 0x%x", __func__, fid); 1874 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1875 return (1); 1876 } 1877 1878 compl->cdw0 = 0; 1879 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1880 1881 feat = &sc->feat[fid]; 1882 if (feat->get) { 1883 feat->get(sc, feat, command, compl); 1884 } 1885 1886 if (compl->status == NVME_SC_SUCCESS) { 1887 if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific) 1888 compl->cdw0 = NVME_FEATURES_NS_SPECIFIC; 1889 else 1890 compl->cdw0 = feat->cdw11; 1891 } 1892 1893 return (0); 1894 } 1895 1896 static int 1897 nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command, 1898 struct nvme_completion* compl) 1899 { 1900 uint8_t ses, lbaf, pi; 1901 1902 /* Only supports Secure Erase Setting - User Data Erase */ 1903 ses = (command->cdw10 >> 9) & 0x7; 1904 if (ses > 0x1) { 1905 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1906 return (1); 1907 } 1908 1909 /* Only supports a single LBA Format */ 1910 lbaf = command->cdw10 & 0xf; 1911 if (lbaf != 0) { 1912 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1913 NVME_SC_INVALID_FORMAT); 1914 return (1); 1915 } 1916 1917 /* Doesn't support Protection Information */ 1918 pi = (command->cdw10 >> 5) & 0x7; 1919 if (pi != 0) { 1920 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1921 return (1); 1922 } 1923 1924 if (sc->nvstore.type == NVME_STOR_RAM) { 1925 if (sc->nvstore.ctx) 1926 free(sc->nvstore.ctx); 1927 sc->nvstore.ctx = calloc(1, sc->nvstore.size); 1928 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1929 } else { 1930 struct pci_nvme_ioreq *req; 1931 int err; 1932 1933 req = pci_nvme_get_ioreq(sc); 1934 if (req == NULL) { 1935 pci_nvme_status_genc(&compl->status, 1936 NVME_SC_INTERNAL_DEVICE_ERROR); 1937 WPRINTF("%s: unable to allocate IO req", __func__); 1938 return (1); 1939 } 1940 req->nvme_sq = &sc->submit_queues[0]; 1941 req->sqid = 0; 1942 req->opc = command->opc; 1943 req->cid = command->cid; 1944 req->nsid = command->nsid; 1945 1946 req->io_req.br_offset = 0; 1947 req->io_req.br_resid = sc->nvstore.size; 1948 req->io_req.br_callback = pci_nvme_io_done; 1949 1950 err = blockif_delete(sc->nvstore.ctx, &req->io_req); 1951 if (err) { 1952 pci_nvme_status_genc(&compl->status, 1953 NVME_SC_INTERNAL_DEVICE_ERROR); 1954 pci_nvme_release_ioreq(sc, req); 1955 } else 1956 compl->status = NVME_NO_STATUS; 1957 } 1958 1959 return (1); 1960 } 1961 1962 static int 1963 nvme_opc_abort(struct pci_nvme_softc *sc __unused, struct nvme_command *command, 1964 struct nvme_completion *compl) 1965 { 1966 DPRINTF("%s submission queue %u, command ID 0x%x", __func__, 1967 command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF); 1968 1969 /* TODO: search for the command ID and abort it */ 1970 1971 compl->cdw0 = 1; 1972 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1973 return (1); 1974 } 1975 1976 static int 1977 nvme_opc_async_event_req(struct pci_nvme_softc* sc, 1978 struct nvme_command* command, struct nvme_completion* compl) 1979 { 1980 DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__, 1981 sc->aer_count, sc->ctrldata.aerl, command->cid); 1982 1983 /* Don't exceed the Async Event Request Limit (AERL). */ 1984 if (pci_nvme_aer_limit_reached(sc)) { 1985 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1986 NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); 1987 return (1); 1988 } 1989 1990 if (pci_nvme_aer_add(sc, command->cid)) { 1991 pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC, 1992 NVME_SC_INTERNAL_DEVICE_ERROR); 1993 return (1); 1994 } 1995 1996 /* 1997 * Raise events when they happen based on the Set Features cmd. 1998 * These events happen async, so only set completion successful if 1999 * there is an event reflective of the request to get event. 2000 */ 2001 compl->status = NVME_NO_STATUS; 2002 pci_nvme_aen_notify(sc); 2003 2004 return (0); 2005 } 2006 2007 static void 2008 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) 2009 { 2010 struct nvme_completion compl; 2011 struct nvme_command *cmd; 2012 struct nvme_submission_queue *sq; 2013 struct nvme_completion_queue *cq; 2014 uint16_t sqhead; 2015 2016 DPRINTF("%s index %u", __func__, (uint32_t)value); 2017 2018 sq = &sc->submit_queues[0]; 2019 cq = &sc->compl_queues[0]; 2020 2021 pthread_mutex_lock(&sq->mtx); 2022 2023 sqhead = sq->head; 2024 DPRINTF("sqhead %u, tail %u", sqhead, sq->tail); 2025 2026 while (sqhead != atomic_load_acq_short(&sq->tail)) { 2027 cmd = &(sq->qbase)[sqhead]; 2028 compl.cdw0 = 0; 2029 compl.status = 0; 2030 2031 switch (cmd->opc) { 2032 case NVME_OPC_DELETE_IO_SQ: 2033 DPRINTF("%s command DELETE_IO_SQ", __func__); 2034 nvme_opc_delete_io_sq(sc, cmd, &compl); 2035 break; 2036 case NVME_OPC_CREATE_IO_SQ: 2037 DPRINTF("%s command CREATE_IO_SQ", __func__); 2038 nvme_opc_create_io_sq(sc, cmd, &compl); 2039 break; 2040 case NVME_OPC_DELETE_IO_CQ: 2041 DPRINTF("%s command DELETE_IO_CQ", __func__); 2042 nvme_opc_delete_io_cq(sc, cmd, &compl); 2043 break; 2044 case NVME_OPC_CREATE_IO_CQ: 2045 DPRINTF("%s command CREATE_IO_CQ", __func__); 2046 nvme_opc_create_io_cq(sc, cmd, &compl); 2047 break; 2048 case NVME_OPC_GET_LOG_PAGE: 2049 DPRINTF("%s command GET_LOG_PAGE", __func__); 2050 nvme_opc_get_log_page(sc, cmd, &compl); 2051 break; 2052 case NVME_OPC_IDENTIFY: 2053 DPRINTF("%s command IDENTIFY", __func__); 2054 nvme_opc_identify(sc, cmd, &compl); 2055 break; 2056 case NVME_OPC_ABORT: 2057 DPRINTF("%s command ABORT", __func__); 2058 nvme_opc_abort(sc, cmd, &compl); 2059 break; 2060 case NVME_OPC_SET_FEATURES: 2061 DPRINTF("%s command SET_FEATURES", __func__); 2062 nvme_opc_set_features(sc, cmd, &compl); 2063 break; 2064 case NVME_OPC_GET_FEATURES: 2065 DPRINTF("%s command GET_FEATURES", __func__); 2066 nvme_opc_get_features(sc, cmd, &compl); 2067 break; 2068 case NVME_OPC_FIRMWARE_ACTIVATE: 2069 DPRINTF("%s command FIRMWARE_ACTIVATE", __func__); 2070 pci_nvme_status_tc(&compl.status, 2071 NVME_SCT_COMMAND_SPECIFIC, 2072 NVME_SC_INVALID_FIRMWARE_SLOT); 2073 break; 2074 case NVME_OPC_ASYNC_EVENT_REQUEST: 2075 DPRINTF("%s command ASYNC_EVENT_REQ", __func__); 2076 nvme_opc_async_event_req(sc, cmd, &compl); 2077 break; 2078 case NVME_OPC_FORMAT_NVM: 2079 DPRINTF("%s command FORMAT_NVM", __func__); 2080 if ((sc->ctrldata.oacs & 2081 (1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) { 2082 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); 2083 break; 2084 } 2085 nvme_opc_format_nvm(sc, cmd, &compl); 2086 break; 2087 case NVME_OPC_SECURITY_SEND: 2088 case NVME_OPC_SECURITY_RECEIVE: 2089 case NVME_OPC_SANITIZE: 2090 case NVME_OPC_GET_LBA_STATUS: 2091 DPRINTF("%s command OPC=%#x (unsupported)", __func__, 2092 cmd->opc); 2093 /* Valid but unsupported opcodes */ 2094 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD); 2095 break; 2096 default: 2097 DPRINTF("%s command OPC=%#X (not implemented)", 2098 __func__, 2099 cmd->opc); 2100 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); 2101 } 2102 sqhead = (sqhead + 1) % sq->size; 2103 2104 if (NVME_COMPLETION_VALID(compl)) { 2105 pci_nvme_cq_update(sc, &sc->compl_queues[0], 2106 compl.cdw0, 2107 cmd->cid, 2108 0, /* SQID */ 2109 compl.status); 2110 } 2111 } 2112 2113 DPRINTF("setting sqhead %u", sqhead); 2114 sq->head = sqhead; 2115 2116 if (cq->head != cq->tail) 2117 pci_generate_msix(sc->nsc_pi, 0); 2118 2119 pthread_mutex_unlock(&sq->mtx); 2120 } 2121 2122 /* 2123 * Update the Write and Read statistics reported in SMART data 2124 * 2125 * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up. 2126 * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000 2127 * 512 byte blocks. Rounding up is achieved by initializing the remainder to 999. 2128 */ 2129 static void 2130 pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc, 2131 size_t bytes, uint16_t status) 2132 { 2133 2134 pthread_mutex_lock(&sc->mtx); 2135 switch (opc) { 2136 case NVME_OPC_WRITE: 2137 sc->write_commands++; 2138 if (status != NVME_SC_SUCCESS) 2139 break; 2140 sc->write_dunits_remainder += (bytes / 512); 2141 while (sc->write_dunits_remainder >= 1000) { 2142 sc->write_data_units++; 2143 sc->write_dunits_remainder -= 1000; 2144 } 2145 break; 2146 case NVME_OPC_READ: 2147 sc->read_commands++; 2148 if (status != NVME_SC_SUCCESS) 2149 break; 2150 sc->read_dunits_remainder += (bytes / 512); 2151 while (sc->read_dunits_remainder >= 1000) { 2152 sc->read_data_units++; 2153 sc->read_dunits_remainder -= 1000; 2154 } 2155 break; 2156 default: 2157 DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc); 2158 break; 2159 } 2160 pthread_mutex_unlock(&sc->mtx); 2161 } 2162 2163 /* 2164 * Check if the combination of Starting LBA (slba) and number of blocks 2165 * exceeds the range of the underlying storage. 2166 * 2167 * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores 2168 * the capacity in bytes as a uint64_t, care must be taken to avoid integer 2169 * overflow. 2170 */ 2171 static bool 2172 pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba, 2173 uint32_t nblocks) 2174 { 2175 size_t offset, bytes; 2176 2177 /* Overflow check of multiplying Starting LBA by the sector size */ 2178 if (slba >> (64 - nvstore->sectsz_bits)) 2179 return (true); 2180 2181 offset = slba << nvstore->sectsz_bits; 2182 bytes = nblocks << nvstore->sectsz_bits; 2183 2184 /* Overflow check of Number of Logical Blocks */ 2185 if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes)) 2186 return (true); 2187 2188 return (false); 2189 } 2190 2191 static int 2192 pci_nvme_append_iov_req(struct pci_nvme_softc *sc __unused, 2193 struct pci_nvme_ioreq *req, uint64_t gpaddr, size_t size, uint64_t offset) 2194 { 2195 int iovidx; 2196 bool range_is_contiguous; 2197 2198 if (req == NULL) 2199 return (-1); 2200 2201 if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) { 2202 return (-1); 2203 } 2204 2205 /* 2206 * Minimize the number of IOVs by concatenating contiguous address 2207 * ranges. If the IOV count is zero, there is no previous range to 2208 * concatenate. 2209 */ 2210 if (req->io_req.br_iovcnt == 0) 2211 range_is_contiguous = false; 2212 else 2213 range_is_contiguous = (req->prev_gpaddr + req->prev_size) == gpaddr; 2214 2215 if (range_is_contiguous) { 2216 iovidx = req->io_req.br_iovcnt - 1; 2217 2218 req->io_req.br_iov[iovidx].iov_base = 2219 paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 2220 req->prev_gpaddr, size); 2221 if (req->io_req.br_iov[iovidx].iov_base == NULL) 2222 return (-1); 2223 2224 req->prev_size += size; 2225 req->io_req.br_resid += size; 2226 2227 req->io_req.br_iov[iovidx].iov_len = req->prev_size; 2228 } else { 2229 iovidx = req->io_req.br_iovcnt; 2230 if (iovidx == 0) { 2231 req->io_req.br_offset = offset; 2232 req->io_req.br_resid = 0; 2233 req->io_req.br_param = req; 2234 } 2235 2236 req->io_req.br_iov[iovidx].iov_base = 2237 paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 2238 gpaddr, size); 2239 if (req->io_req.br_iov[iovidx].iov_base == NULL) 2240 return (-1); 2241 2242 req->io_req.br_iov[iovidx].iov_len = size; 2243 2244 req->prev_gpaddr = gpaddr; 2245 req->prev_size = size; 2246 req->io_req.br_resid += size; 2247 2248 req->io_req.br_iovcnt++; 2249 } 2250 2251 return (0); 2252 } 2253 2254 static void 2255 pci_nvme_set_completion(struct pci_nvme_softc *sc, 2256 struct nvme_submission_queue *sq, int sqid, uint16_t cid, uint16_t status) 2257 { 2258 struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid]; 2259 2260 DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x", 2261 __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status), 2262 NVME_STATUS_GET_SC(status)); 2263 2264 pci_nvme_cq_update(sc, cq, 0, cid, sqid, status); 2265 2266 if (cq->head != cq->tail) { 2267 if (cq->intr_en & NVME_CQ_INTEN) { 2268 pci_generate_msix(sc->nsc_pi, cq->intr_vec); 2269 } else { 2270 DPRINTF("%s: CQ%u interrupt disabled", 2271 __func__, sq->cqid); 2272 } 2273 } 2274 } 2275 2276 static void 2277 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req) 2278 { 2279 req->sc = NULL; 2280 req->nvme_sq = NULL; 2281 req->sqid = 0; 2282 2283 pthread_mutex_lock(&sc->mtx); 2284 2285 STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link); 2286 sc->pending_ios--; 2287 2288 /* when no more IO pending, can set to ready if device reset/enabled */ 2289 if (sc->pending_ios == 0 && 2290 NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts))) 2291 sc->regs.csts |= NVME_CSTS_RDY; 2292 2293 pthread_mutex_unlock(&sc->mtx); 2294 2295 sem_post(&sc->iosemlock); 2296 } 2297 2298 static struct pci_nvme_ioreq * 2299 pci_nvme_get_ioreq(struct pci_nvme_softc *sc) 2300 { 2301 struct pci_nvme_ioreq *req = NULL; 2302 2303 sem_wait(&sc->iosemlock); 2304 pthread_mutex_lock(&sc->mtx); 2305 2306 req = STAILQ_FIRST(&sc->ioreqs_free); 2307 assert(req != NULL); 2308 STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link); 2309 2310 req->sc = sc; 2311 2312 sc->pending_ios++; 2313 2314 pthread_mutex_unlock(&sc->mtx); 2315 2316 req->io_req.br_iovcnt = 0; 2317 req->io_req.br_offset = 0; 2318 req->io_req.br_resid = 0; 2319 req->io_req.br_param = req; 2320 req->prev_gpaddr = 0; 2321 req->prev_size = 0; 2322 2323 return req; 2324 } 2325 2326 static void 2327 pci_nvme_io_done(struct blockif_req *br, int err) 2328 { 2329 struct pci_nvme_ioreq *req = br->br_param; 2330 struct nvme_submission_queue *sq = req->nvme_sq; 2331 uint16_t code, status; 2332 2333 DPRINTF("%s error %d %s", __func__, err, strerror(err)); 2334 2335 /* TODO return correct error */ 2336 code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS; 2337 status = 0; 2338 pci_nvme_status_genc(&status, code); 2339 2340 pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, status); 2341 pci_nvme_stats_write_read_update(req->sc, req->opc, 2342 req->bytes, status); 2343 pci_nvme_release_ioreq(req->sc, req); 2344 } 2345 2346 /* 2347 * Implements the Flush command. The specification states: 2348 * If a volatile write cache is not present, Flush commands complete 2349 * successfully and have no effect 2350 * in the description of the Volatile Write Cache (VWC) field of the Identify 2351 * Controller data. Therefore, set status to Success if the command is 2352 * not supported (i.e. RAM or as indicated by the blockif). 2353 */ 2354 static bool 2355 nvme_opc_flush(struct pci_nvme_softc *sc __unused, 2356 struct nvme_command *cmd __unused, 2357 struct pci_nvme_blockstore *nvstore, 2358 struct pci_nvme_ioreq *req, 2359 uint16_t *status) 2360 { 2361 bool pending = false; 2362 2363 if (nvstore->type == NVME_STOR_RAM) { 2364 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2365 } else { 2366 int err; 2367 2368 req->io_req.br_callback = pci_nvme_io_done; 2369 2370 err = blockif_flush(nvstore->ctx, &req->io_req); 2371 switch (err) { 2372 case 0: 2373 pending = true; 2374 break; 2375 case EOPNOTSUPP: 2376 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2377 break; 2378 default: 2379 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2380 } 2381 } 2382 2383 return (pending); 2384 } 2385 2386 static uint16_t 2387 nvme_write_read_ram(struct pci_nvme_softc *sc, 2388 struct pci_nvme_blockstore *nvstore, 2389 uint64_t prp1, uint64_t prp2, 2390 size_t offset, uint64_t bytes, 2391 bool is_write) 2392 { 2393 uint8_t *buf = nvstore->ctx; 2394 enum nvme_copy_dir dir; 2395 uint16_t status; 2396 2397 if (is_write) 2398 dir = NVME_COPY_TO_PRP; 2399 else 2400 dir = NVME_COPY_FROM_PRP; 2401 2402 status = 0; 2403 if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2, 2404 buf + offset, bytes, dir)) 2405 pci_nvme_status_genc(&status, 2406 NVME_SC_DATA_TRANSFER_ERROR); 2407 else 2408 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 2409 2410 return (status); 2411 } 2412 2413 static uint16_t 2414 nvme_write_read_blockif(struct pci_nvme_softc *sc, 2415 struct pci_nvme_blockstore *nvstore, 2416 struct pci_nvme_ioreq *req, 2417 uint64_t prp1, uint64_t prp2, 2418 size_t offset, uint64_t bytes, 2419 bool is_write) 2420 { 2421 uint64_t size; 2422 int err; 2423 uint16_t status = NVME_NO_STATUS; 2424 2425 size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes); 2426 if (pci_nvme_append_iov_req(sc, req, prp1, size, offset)) { 2427 err = -1; 2428 goto out; 2429 } 2430 2431 offset += size; 2432 bytes -= size; 2433 2434 if (bytes == 0) { 2435 ; 2436 } else if (bytes <= PAGE_SIZE) { 2437 size = bytes; 2438 if (pci_nvme_append_iov_req(sc, req, prp2, size, offset)) { 2439 err = -1; 2440 goto out; 2441 } 2442 } else { 2443 void *vmctx = sc->nsc_pi->pi_vmctx; 2444 uint64_t *prp_list = &prp2; 2445 uint64_t *last = prp_list; 2446 2447 /* PRP2 is pointer to a physical region page list */ 2448 while (bytes) { 2449 /* Last entry in list points to the next list */ 2450 if ((prp_list == last) && (bytes > PAGE_SIZE)) { 2451 uint64_t prp = *prp_list; 2452 2453 prp_list = paddr_guest2host(vmctx, prp, 2454 PAGE_SIZE - (prp % PAGE_SIZE)); 2455 if (prp_list == NULL) { 2456 err = -1; 2457 goto out; 2458 } 2459 last = prp_list + (NVME_PRP2_ITEMS - 1); 2460 } 2461 2462 size = MIN(bytes, PAGE_SIZE); 2463 2464 if (pci_nvme_append_iov_req(sc, req, *prp_list, size, 2465 offset)) { 2466 err = -1; 2467 goto out; 2468 } 2469 2470 offset += size; 2471 bytes -= size; 2472 2473 prp_list++; 2474 } 2475 } 2476 req->io_req.br_callback = pci_nvme_io_done; 2477 if (is_write) 2478 err = blockif_write(nvstore->ctx, &req->io_req); 2479 else 2480 err = blockif_read(nvstore->ctx, &req->io_req); 2481 out: 2482 if (err) 2483 pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR); 2484 2485 return (status); 2486 } 2487 2488 static bool 2489 nvme_opc_write_read(struct pci_nvme_softc *sc, 2490 struct nvme_command *cmd, 2491 struct pci_nvme_blockstore *nvstore, 2492 struct pci_nvme_ioreq *req, 2493 uint16_t *status) 2494 { 2495 uint64_t lba, nblocks, bytes; 2496 size_t offset; 2497 bool is_write = cmd->opc == NVME_OPC_WRITE; 2498 bool pending = false; 2499 2500 lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; 2501 nblocks = (cmd->cdw12 & 0xFFFF) + 1; 2502 bytes = nblocks << nvstore->sectsz_bits; 2503 if (bytes > NVME_MAX_DATA_SIZE) { 2504 WPRINTF("%s command would exceed MDTS", __func__); 2505 pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD); 2506 goto out; 2507 } 2508 2509 if (pci_nvme_out_of_range(nvstore, lba, nblocks)) { 2510 WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)", 2511 __func__, lba, nblocks); 2512 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); 2513 goto out; 2514 } 2515 2516 offset = lba << nvstore->sectsz_bits; 2517 2518 req->bytes = bytes; 2519 req->io_req.br_offset = lba; 2520 2521 /* PRP bits 1:0 must be zero */ 2522 cmd->prp1 &= ~0x3UL; 2523 cmd->prp2 &= ~0x3UL; 2524 2525 if (nvstore->type == NVME_STOR_RAM) { 2526 *status = nvme_write_read_ram(sc, nvstore, cmd->prp1, 2527 cmd->prp2, offset, bytes, is_write); 2528 } else { 2529 *status = nvme_write_read_blockif(sc, nvstore, req, 2530 cmd->prp1, cmd->prp2, offset, bytes, is_write); 2531 2532 if (*status == NVME_NO_STATUS) 2533 pending = true; 2534 } 2535 out: 2536 if (!pending) 2537 pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status); 2538 2539 return (pending); 2540 } 2541 2542 static void 2543 pci_nvme_dealloc_sm(struct blockif_req *br, int err) 2544 { 2545 struct pci_nvme_ioreq *req = br->br_param; 2546 struct pci_nvme_softc *sc = req->sc; 2547 bool done = true; 2548 uint16_t status; 2549 2550 status = 0; 2551 if (err) { 2552 pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); 2553 } else if ((req->prev_gpaddr + 1) == (req->prev_size)) { 2554 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 2555 } else { 2556 struct iovec *iov = req->io_req.br_iov; 2557 2558 req->prev_gpaddr++; 2559 iov += req->prev_gpaddr; 2560 2561 /* The iov_* values already include the sector size */ 2562 req->io_req.br_offset = (off_t)iov->iov_base; 2563 req->io_req.br_resid = iov->iov_len; 2564 if (blockif_delete(sc->nvstore.ctx, &req->io_req)) { 2565 pci_nvme_status_genc(&status, 2566 NVME_SC_INTERNAL_DEVICE_ERROR); 2567 } else 2568 done = false; 2569 } 2570 2571 if (done) { 2572 pci_nvme_set_completion(sc, req->nvme_sq, req->sqid, req->cid, 2573 status); 2574 pci_nvme_release_ioreq(sc, req); 2575 } 2576 } 2577 2578 static bool 2579 nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc, 2580 struct nvme_command *cmd, 2581 struct pci_nvme_blockstore *nvstore, 2582 struct pci_nvme_ioreq *req, 2583 uint16_t *status) 2584 { 2585 struct nvme_dsm_range *range = NULL; 2586 uint32_t nr, r, non_zero, dr; 2587 int err; 2588 bool pending = false; 2589 2590 if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) { 2591 pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE); 2592 goto out; 2593 } 2594 2595 nr = cmd->cdw10 & 0xff; 2596 2597 /* copy locally because a range entry could straddle PRPs */ 2598 range = calloc(1, NVME_MAX_DSM_TRIM); 2599 if (range == NULL) { 2600 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2601 goto out; 2602 } 2603 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2, 2604 (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP); 2605 2606 /* Check for invalid ranges and the number of non-zero lengths */ 2607 non_zero = 0; 2608 for (r = 0; r <= nr; r++) { 2609 if (pci_nvme_out_of_range(nvstore, 2610 range[r].starting_lba, range[r].length)) { 2611 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); 2612 goto out; 2613 } 2614 if (range[r].length != 0) 2615 non_zero++; 2616 } 2617 2618 if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) { 2619 size_t offset, bytes; 2620 int sectsz_bits = sc->nvstore.sectsz_bits; 2621 2622 /* 2623 * DSM calls are advisory only, and compliant controllers 2624 * may choose to take no actions (i.e. return Success). 2625 */ 2626 if (!nvstore->deallocate) { 2627 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2628 goto out; 2629 } 2630 2631 /* If all ranges have a zero length, return Success */ 2632 if (non_zero == 0) { 2633 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2634 goto out; 2635 } 2636 2637 if (req == NULL) { 2638 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2639 goto out; 2640 } 2641 2642 offset = range[0].starting_lba << sectsz_bits; 2643 bytes = range[0].length << sectsz_bits; 2644 2645 /* 2646 * If the request is for more than a single range, store 2647 * the ranges in the br_iov. Optimize for the common case 2648 * of a single range. 2649 * 2650 * Note that NVMe Number of Ranges is a zero based value 2651 */ 2652 req->io_req.br_iovcnt = 0; 2653 req->io_req.br_offset = offset; 2654 req->io_req.br_resid = bytes; 2655 2656 if (nr == 0) { 2657 req->io_req.br_callback = pci_nvme_io_done; 2658 } else { 2659 struct iovec *iov = req->io_req.br_iov; 2660 2661 for (r = 0, dr = 0; r <= nr; r++) { 2662 offset = range[r].starting_lba << sectsz_bits; 2663 bytes = range[r].length << sectsz_bits; 2664 if (bytes == 0) 2665 continue; 2666 2667 if ((nvstore->size - offset) < bytes) { 2668 pci_nvme_status_genc(status, 2669 NVME_SC_LBA_OUT_OF_RANGE); 2670 goto out; 2671 } 2672 iov[dr].iov_base = (void *)offset; 2673 iov[dr].iov_len = bytes; 2674 dr++; 2675 } 2676 req->io_req.br_callback = pci_nvme_dealloc_sm; 2677 2678 /* 2679 * Use prev_gpaddr to track the current entry and 2680 * prev_size to track the number of entries 2681 */ 2682 req->prev_gpaddr = 0; 2683 req->prev_size = dr; 2684 } 2685 2686 err = blockif_delete(nvstore->ctx, &req->io_req); 2687 if (err) 2688 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2689 else 2690 pending = true; 2691 } 2692 out: 2693 free(range); 2694 return (pending); 2695 } 2696 2697 static void 2698 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx) 2699 { 2700 struct nvme_submission_queue *sq; 2701 uint16_t status; 2702 uint16_t sqhead; 2703 2704 /* handle all submissions up to sq->tail index */ 2705 sq = &sc->submit_queues[idx]; 2706 2707 pthread_mutex_lock(&sq->mtx); 2708 2709 sqhead = sq->head; 2710 DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p", 2711 idx, sqhead, sq->tail, sq->qbase); 2712 2713 while (sqhead != atomic_load_acq_short(&sq->tail)) { 2714 struct nvme_command *cmd; 2715 struct pci_nvme_ioreq *req; 2716 uint32_t nsid; 2717 bool pending; 2718 2719 pending = false; 2720 req = NULL; 2721 status = 0; 2722 2723 cmd = &sq->qbase[sqhead]; 2724 sqhead = (sqhead + 1) % sq->size; 2725 2726 nsid = le32toh(cmd->nsid); 2727 if ((nsid == 0) || (nsid > sc->ctrldata.nn)) { 2728 pci_nvme_status_genc(&status, 2729 NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 2730 status |= 2731 NVME_STATUS_DNR_MASK << NVME_STATUS_DNR_SHIFT; 2732 goto complete; 2733 } 2734 2735 req = pci_nvme_get_ioreq(sc); 2736 if (req == NULL) { 2737 pci_nvme_status_genc(&status, 2738 NVME_SC_INTERNAL_DEVICE_ERROR); 2739 WPRINTF("%s: unable to allocate IO req", __func__); 2740 goto complete; 2741 } 2742 req->nvme_sq = sq; 2743 req->sqid = idx; 2744 req->opc = cmd->opc; 2745 req->cid = cmd->cid; 2746 req->nsid = cmd->nsid; 2747 2748 switch (cmd->opc) { 2749 case NVME_OPC_FLUSH: 2750 pending = nvme_opc_flush(sc, cmd, &sc->nvstore, 2751 req, &status); 2752 break; 2753 case NVME_OPC_WRITE: 2754 case NVME_OPC_READ: 2755 pending = nvme_opc_write_read(sc, cmd, &sc->nvstore, 2756 req, &status); 2757 break; 2758 case NVME_OPC_WRITE_ZEROES: 2759 /* TODO: write zeroes 2760 WPRINTF("%s write zeroes lba 0x%lx blocks %u", 2761 __func__, lba, cmd->cdw12 & 0xFFFF); */ 2762 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 2763 break; 2764 case NVME_OPC_DATASET_MANAGEMENT: 2765 pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore, 2766 req, &status); 2767 break; 2768 default: 2769 WPRINTF("%s unhandled io command 0x%x", 2770 __func__, cmd->opc); 2771 pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE); 2772 } 2773 complete: 2774 if (!pending) { 2775 pci_nvme_set_completion(sc, sq, idx, cmd->cid, status); 2776 if (req != NULL) 2777 pci_nvme_release_ioreq(sc, req); 2778 } 2779 } 2780 2781 sq->head = sqhead; 2782 2783 pthread_mutex_unlock(&sq->mtx); 2784 } 2785 2786 static void 2787 pci_nvme_handle_doorbell(struct pci_nvme_softc* sc, 2788 uint64_t idx, int is_sq, uint64_t value) 2789 { 2790 DPRINTF("nvme doorbell %lu, %s, val 0x%lx", 2791 idx, is_sq ? "SQ" : "CQ", value & 0xFFFF); 2792 2793 if (is_sq) { 2794 if (idx > sc->num_squeues) { 2795 WPRINTF("%s queue index %lu overflow from " 2796 "guest (max %u)", 2797 __func__, idx, sc->num_squeues); 2798 return; 2799 } 2800 2801 atomic_store_short(&sc->submit_queues[idx].tail, 2802 (uint16_t)value); 2803 2804 if (idx == 0) { 2805 pci_nvme_handle_admin_cmd(sc, value); 2806 } else { 2807 /* submission queue; handle new entries in SQ */ 2808 if (idx > sc->num_squeues) { 2809 WPRINTF("%s SQ index %lu overflow from " 2810 "guest (max %u)", 2811 __func__, idx, sc->num_squeues); 2812 return; 2813 } 2814 pci_nvme_handle_io_cmd(sc, (uint16_t)idx); 2815 } 2816 } else { 2817 if (idx > sc->num_cqueues) { 2818 WPRINTF("%s queue index %lu overflow from " 2819 "guest (max %u)", 2820 __func__, idx, sc->num_cqueues); 2821 return; 2822 } 2823 2824 atomic_store_short(&sc->compl_queues[idx].head, 2825 (uint16_t)value); 2826 } 2827 } 2828 2829 static void 2830 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite) 2831 { 2832 const char *s = iswrite ? "WRITE" : "READ"; 2833 2834 switch (offset) { 2835 case NVME_CR_CAP_LOW: 2836 DPRINTF("%s %s NVME_CR_CAP_LOW", func, s); 2837 break; 2838 case NVME_CR_CAP_HI: 2839 DPRINTF("%s %s NVME_CR_CAP_HI", func, s); 2840 break; 2841 case NVME_CR_VS: 2842 DPRINTF("%s %s NVME_CR_VS", func, s); 2843 break; 2844 case NVME_CR_INTMS: 2845 DPRINTF("%s %s NVME_CR_INTMS", func, s); 2846 break; 2847 case NVME_CR_INTMC: 2848 DPRINTF("%s %s NVME_CR_INTMC", func, s); 2849 break; 2850 case NVME_CR_CC: 2851 DPRINTF("%s %s NVME_CR_CC", func, s); 2852 break; 2853 case NVME_CR_CSTS: 2854 DPRINTF("%s %s NVME_CR_CSTS", func, s); 2855 break; 2856 case NVME_CR_NSSR: 2857 DPRINTF("%s %s NVME_CR_NSSR", func, s); 2858 break; 2859 case NVME_CR_AQA: 2860 DPRINTF("%s %s NVME_CR_AQA", func, s); 2861 break; 2862 case NVME_CR_ASQ_LOW: 2863 DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s); 2864 break; 2865 case NVME_CR_ASQ_HI: 2866 DPRINTF("%s %s NVME_CR_ASQ_HI", func, s); 2867 break; 2868 case NVME_CR_ACQ_LOW: 2869 DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s); 2870 break; 2871 case NVME_CR_ACQ_HI: 2872 DPRINTF("%s %s NVME_CR_ACQ_HI", func, s); 2873 break; 2874 default: 2875 DPRINTF("unknown nvme bar-0 offset 0x%lx", offset); 2876 } 2877 2878 } 2879 2880 static void 2881 pci_nvme_write_bar_0(struct pci_nvme_softc *sc, uint64_t offset, int size, 2882 uint64_t value) 2883 { 2884 uint32_t ccreg; 2885 2886 if (offset >= NVME_DOORBELL_OFFSET) { 2887 uint64_t belloffset = offset - NVME_DOORBELL_OFFSET; 2888 uint64_t idx = belloffset / 8; /* door bell size = 2*int */ 2889 int is_sq = (belloffset % 8) < 4; 2890 2891 if ((sc->regs.csts & NVME_CSTS_RDY) == 0) { 2892 WPRINTF("doorbell write prior to RDY (offset=%#lx)\n", 2893 offset); 2894 return; 2895 } 2896 2897 if (belloffset > ((sc->max_queues+1) * 8 - 4)) { 2898 WPRINTF("guest attempted an overflow write offset " 2899 "0x%lx, val 0x%lx in %s", 2900 offset, value, __func__); 2901 return; 2902 } 2903 2904 if (is_sq) { 2905 if (sc->submit_queues[idx].qbase == NULL) 2906 return; 2907 } else if (sc->compl_queues[idx].qbase == NULL) 2908 return; 2909 2910 pci_nvme_handle_doorbell(sc, idx, is_sq, value); 2911 return; 2912 } 2913 2914 DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx", 2915 offset, size, value); 2916 2917 if (size != 4) { 2918 WPRINTF("guest wrote invalid size %d (offset 0x%lx, " 2919 "val 0x%lx) to bar0 in %s", 2920 size, offset, value, __func__); 2921 /* TODO: shutdown device */ 2922 return; 2923 } 2924 2925 pci_nvme_bar0_reg_dumps(__func__, offset, 1); 2926 2927 pthread_mutex_lock(&sc->mtx); 2928 2929 switch (offset) { 2930 case NVME_CR_CAP_LOW: 2931 case NVME_CR_CAP_HI: 2932 /* readonly */ 2933 break; 2934 case NVME_CR_VS: 2935 /* readonly */ 2936 break; 2937 case NVME_CR_INTMS: 2938 /* MSI-X, so ignore */ 2939 break; 2940 case NVME_CR_INTMC: 2941 /* MSI-X, so ignore */ 2942 break; 2943 case NVME_CR_CC: 2944 ccreg = (uint32_t)value; 2945 2946 DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u " 2947 "iocqes %u", 2948 __func__, 2949 NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg), 2950 NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg), 2951 NVME_CC_GET_IOCQES(ccreg)); 2952 2953 if (NVME_CC_GET_SHN(ccreg)) { 2954 /* perform shutdown - flush out data to backend */ 2955 sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK << 2956 NVME_CSTS_REG_SHST_SHIFT); 2957 sc->regs.csts |= NVME_SHST_COMPLETE << 2958 NVME_CSTS_REG_SHST_SHIFT; 2959 } 2960 if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) { 2961 if (NVME_CC_GET_EN(ccreg) == 0) 2962 /* transition 1-> causes controller reset */ 2963 pci_nvme_reset_locked(sc); 2964 else 2965 pci_nvme_init_controller(sc); 2966 } 2967 2968 /* Insert the iocqes, iosqes and en bits from the write */ 2969 sc->regs.cc &= ~NVME_CC_WRITE_MASK; 2970 sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK; 2971 if (NVME_CC_GET_EN(ccreg) == 0) { 2972 /* Insert the ams, mps and css bit fields */ 2973 sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK; 2974 sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK; 2975 sc->regs.csts &= ~NVME_CSTS_RDY; 2976 } else if ((sc->pending_ios == 0) && 2977 !(sc->regs.csts & NVME_CSTS_CFS)) { 2978 sc->regs.csts |= NVME_CSTS_RDY; 2979 } 2980 break; 2981 case NVME_CR_CSTS: 2982 break; 2983 case NVME_CR_NSSR: 2984 /* ignore writes; don't support subsystem reset */ 2985 break; 2986 case NVME_CR_AQA: 2987 sc->regs.aqa = (uint32_t)value; 2988 break; 2989 case NVME_CR_ASQ_LOW: 2990 sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) | 2991 (0xFFFFF000 & value); 2992 break; 2993 case NVME_CR_ASQ_HI: 2994 sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) | 2995 (value << 32); 2996 break; 2997 case NVME_CR_ACQ_LOW: 2998 sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) | 2999 (0xFFFFF000 & value); 3000 break; 3001 case NVME_CR_ACQ_HI: 3002 sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) | 3003 (value << 32); 3004 break; 3005 default: 3006 DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d", 3007 __func__, offset, value, size); 3008 } 3009 pthread_mutex_unlock(&sc->mtx); 3010 } 3011 3012 static void 3013 pci_nvme_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, 3014 uint64_t value) 3015 { 3016 struct pci_nvme_softc* sc = pi->pi_arg; 3017 3018 if (baridx == pci_msix_table_bar(pi) || 3019 baridx == pci_msix_pba_bar(pi)) { 3020 DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, " 3021 " value 0x%lx", baridx, offset, size, value); 3022 3023 pci_emul_msix_twrite(pi, offset, size, value); 3024 return; 3025 } 3026 3027 switch (baridx) { 3028 case 0: 3029 pci_nvme_write_bar_0(sc, offset, size, value); 3030 break; 3031 3032 default: 3033 DPRINTF("%s unknown baridx %d, val 0x%lx", 3034 __func__, baridx, value); 3035 } 3036 } 3037 3038 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc, 3039 uint64_t offset, int size) 3040 { 3041 uint64_t value; 3042 3043 pci_nvme_bar0_reg_dumps(__func__, offset, 0); 3044 3045 if (offset < NVME_DOORBELL_OFFSET) { 3046 void *p = &(sc->regs); 3047 pthread_mutex_lock(&sc->mtx); 3048 memcpy(&value, (void *)((uintptr_t)p + offset), size); 3049 pthread_mutex_unlock(&sc->mtx); 3050 } else { 3051 value = 0; 3052 WPRINTF("pci_nvme: read invalid offset %ld", offset); 3053 } 3054 3055 switch (size) { 3056 case 1: 3057 value &= 0xFF; 3058 break; 3059 case 2: 3060 value &= 0xFFFF; 3061 break; 3062 case 4: 3063 value &= 0xFFFFFFFF; 3064 break; 3065 } 3066 3067 DPRINTF(" nvme-read offset 0x%lx, size %d -> value 0x%x", 3068 offset, size, (uint32_t)value); 3069 3070 return (value); 3071 } 3072 3073 3074 3075 static uint64_t 3076 pci_nvme_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) 3077 { 3078 struct pci_nvme_softc* sc = pi->pi_arg; 3079 3080 if (baridx == pci_msix_table_bar(pi) || 3081 baridx == pci_msix_pba_bar(pi)) { 3082 DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d", 3083 baridx, offset, size); 3084 3085 return pci_emul_msix_tread(pi, offset, size); 3086 } 3087 3088 switch (baridx) { 3089 case 0: 3090 return pci_nvme_read_bar_0(sc, offset, size); 3091 3092 default: 3093 DPRINTF("unknown bar %d, 0x%lx", baridx, offset); 3094 } 3095 3096 return (0); 3097 } 3098 3099 static int 3100 pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl) 3101 { 3102 char bident[sizeof("XXX:XXX")]; 3103 const char *value; 3104 uint32_t sectsz; 3105 3106 sc->max_queues = NVME_QUEUES; 3107 sc->max_qentries = NVME_MAX_QENTRIES; 3108 sc->ioslots = NVME_IOSLOTS; 3109 sc->num_squeues = sc->max_queues; 3110 sc->num_cqueues = sc->max_queues; 3111 sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; 3112 sectsz = 0; 3113 snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn), 3114 "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 3115 3116 value = get_config_value_node(nvl, "maxq"); 3117 if (value != NULL) 3118 sc->max_queues = atoi(value); 3119 value = get_config_value_node(nvl, "qsz"); 3120 if (value != NULL) { 3121 sc->max_qentries = atoi(value); 3122 if (sc->max_qentries <= 0) { 3123 EPRINTLN("nvme: Invalid qsz option %d", 3124 sc->max_qentries); 3125 return (-1); 3126 } 3127 } 3128 value = get_config_value_node(nvl, "ioslots"); 3129 if (value != NULL) { 3130 sc->ioslots = atoi(value); 3131 if (sc->ioslots <= 0) { 3132 EPRINTLN("Invalid ioslots option %d", sc->ioslots); 3133 return (-1); 3134 } 3135 } 3136 value = get_config_value_node(nvl, "sectsz"); 3137 if (value != NULL) 3138 sectsz = atoi(value); 3139 value = get_config_value_node(nvl, "ser"); 3140 if (value != NULL) { 3141 /* 3142 * This field indicates the Product Serial Number in 3143 * 7-bit ASCII, unused bytes should be space characters. 3144 * Ref: NVMe v1.3c. 3145 */ 3146 cpywithpad((char *)sc->ctrldata.sn, 3147 sizeof(sc->ctrldata.sn), value, ' '); 3148 } 3149 value = get_config_value_node(nvl, "eui64"); 3150 if (value != NULL) 3151 sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0)); 3152 value = get_config_value_node(nvl, "dsm"); 3153 if (value != NULL) { 3154 if (strcmp(value, "auto") == 0) 3155 sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; 3156 else if (strcmp(value, "enable") == 0) 3157 sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE; 3158 else if (strcmp(value, "disable") == 0) 3159 sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE; 3160 } 3161 3162 value = get_config_value_node(nvl, "bootindex"); 3163 if (value != NULL) { 3164 if (pci_emul_add_boot_device(sc->nsc_pi, atoi(value))) { 3165 EPRINTLN("Invalid bootindex %d", atoi(value)); 3166 return (-1); 3167 } 3168 } 3169 3170 value = get_config_value_node(nvl, "ram"); 3171 if (value != NULL) { 3172 uint64_t sz = strtoull(value, NULL, 10); 3173 3174 sc->nvstore.type = NVME_STOR_RAM; 3175 sc->nvstore.size = sz * 1024 * 1024; 3176 sc->nvstore.ctx = calloc(1, sc->nvstore.size); 3177 sc->nvstore.sectsz = 4096; 3178 sc->nvstore.sectsz_bits = 12; 3179 if (sc->nvstore.ctx == NULL) { 3180 EPRINTLN("nvme: Unable to allocate RAM"); 3181 return (-1); 3182 } 3183 } else { 3184 snprintf(bident, sizeof(bident), "%u:%u", 3185 sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 3186 sc->nvstore.ctx = blockif_open(nvl, bident); 3187 if (sc->nvstore.ctx == NULL) { 3188 EPRINTLN("nvme: Could not open backing file: %s", 3189 strerror(errno)); 3190 return (-1); 3191 } 3192 sc->nvstore.type = NVME_STOR_BLOCKIF; 3193 sc->nvstore.size = blockif_size(sc->nvstore.ctx); 3194 } 3195 3196 if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) 3197 sc->nvstore.sectsz = sectsz; 3198 else if (sc->nvstore.type != NVME_STOR_RAM) 3199 sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); 3200 for (sc->nvstore.sectsz_bits = 9; 3201 (1U << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; 3202 sc->nvstore.sectsz_bits++); 3203 3204 if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) 3205 sc->max_queues = NVME_QUEUES; 3206 3207 return (0); 3208 } 3209 3210 static void 3211 pci_nvme_resized(struct blockif_ctxt *bctxt __unused, void *arg, 3212 size_t new_size) 3213 { 3214 struct pci_nvme_softc *sc; 3215 struct pci_nvme_blockstore *nvstore; 3216 struct nvme_namespace_data *nd; 3217 3218 sc = arg; 3219 nvstore = &sc->nvstore; 3220 nd = &sc->nsdata; 3221 3222 nvstore->size = new_size; 3223 pci_nvme_init_nsdata_size(nvstore, nd); 3224 3225 /* Add changed NSID to list */ 3226 sc->ns_log.ns[0] = 1; 3227 sc->ns_log.ns[1] = 0; 3228 3229 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE, 3230 PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED); 3231 } 3232 3233 static int 3234 pci_nvme_init(struct pci_devinst *pi, nvlist_t *nvl) 3235 { 3236 struct pci_nvme_softc *sc; 3237 uint32_t pci_membar_sz; 3238 int error; 3239 3240 error = 0; 3241 3242 sc = calloc(1, sizeof(struct pci_nvme_softc)); 3243 pi->pi_arg = sc; 3244 sc->nsc_pi = pi; 3245 3246 error = pci_nvme_parse_config(sc, nvl); 3247 if (error < 0) 3248 goto done; 3249 else 3250 error = 0; 3251 3252 STAILQ_INIT(&sc->ioreqs_free); 3253 sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq)); 3254 for (uint32_t i = 0; i < sc->ioslots; i++) { 3255 STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link); 3256 } 3257 3258 pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A); 3259 pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); 3260 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); 3261 pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM); 3262 pci_set_cfgdata8(pi, PCIR_PROGIF, 3263 PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); 3264 3265 /* 3266 * Allocate size of NVMe registers + doorbell space for all queues. 3267 * 3268 * The specification requires a minimum memory I/O window size of 16K. 3269 * The Windows driver will refuse to start a device with a smaller 3270 * window. 3271 */ 3272 pci_membar_sz = sizeof(struct nvme_registers) + 3273 2 * sizeof(uint32_t) * (sc->max_queues + 1); 3274 pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); 3275 3276 DPRINTF("nvme membar size: %u", pci_membar_sz); 3277 3278 error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz); 3279 if (error) { 3280 WPRINTF("%s pci alloc mem bar failed", __func__); 3281 goto done; 3282 } 3283 3284 error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR); 3285 if (error) { 3286 WPRINTF("%s pci add msixcap failed", __func__); 3287 goto done; 3288 } 3289 3290 error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP); 3291 if (error) { 3292 WPRINTF("%s pci add Express capability failed", __func__); 3293 goto done; 3294 } 3295 3296 pthread_mutex_init(&sc->mtx, NULL); 3297 sem_init(&sc->iosemlock, 0, sc->ioslots); 3298 blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc); 3299 3300 pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues); 3301 /* 3302 * Controller data depends on Namespace data so initialize Namespace 3303 * data first. 3304 */ 3305 pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore); 3306 pci_nvme_init_ctrldata(sc); 3307 pci_nvme_init_logpages(sc); 3308 pci_nvme_init_features(sc); 3309 3310 pci_nvme_aer_init(sc); 3311 pci_nvme_aen_init(sc); 3312 3313 pci_nvme_reset(sc); 3314 3315 pci_lintr_request(pi); 3316 3317 done: 3318 return (error); 3319 } 3320 3321 static int 3322 pci_nvme_legacy_config(nvlist_t *nvl, const char *opts) 3323 { 3324 char *cp, *ram; 3325 3326 if (opts == NULL) 3327 return (0); 3328 3329 if (strncmp(opts, "ram=", 4) == 0) { 3330 cp = strchr(opts, ','); 3331 if (cp == NULL) { 3332 set_config_value_node(nvl, "ram", opts + 4); 3333 return (0); 3334 } 3335 ram = strndup(opts + 4, cp - opts - 4); 3336 set_config_value_node(nvl, "ram", ram); 3337 free(ram); 3338 return (pci_parse_legacy_config(nvl, cp + 1)); 3339 } else 3340 return (blockif_legacy_config(nvl, opts)); 3341 } 3342 3343 static const struct pci_devemu pci_de_nvme = { 3344 .pe_emu = "nvme", 3345 .pe_init = pci_nvme_init, 3346 .pe_legacy_config = pci_nvme_legacy_config, 3347 .pe_barwrite = pci_nvme_write, 3348 .pe_barread = pci_nvme_read 3349 }; 3350 PCI_EMUL_SET(pci_de_nvme); 3351