1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017 Shunsuke Mie 5 * Copyright (c) 2018 Leon Dang 6 * Copyright (c) 2020 Chuck Tuffli 7 * 8 * Function crc16 Copyright (c) 2017, Fedor Uporov 9 * Obtained from function ext2_crc16() in sys/fs/ext2fs/ext2_csum.c 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * bhyve PCIe-NVMe device emulation. 35 * 36 * options: 37 * -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm=<opt> 38 * 39 * accepted devpath: 40 * /dev/blockdev 41 * /path/to/image 42 * ram=size_in_MiB 43 * 44 * maxq = max number of queues 45 * qsz = max elements in each queue 46 * ioslots = max number of concurrent io requests 47 * sectsz = sector size (defaults to blockif sector size) 48 * ser = serial number (20-chars max) 49 * eui64 = IEEE Extended Unique Identifier (8 byte value) 50 * dsm = DataSet Management support. Option is one of auto, enable,disable 51 * 52 */ 53 54 /* TODO: 55 - create async event for smart and log 56 - intr coalesce 57 */ 58 59 #include <sys/cdefs.h> 60 __FBSDID("$FreeBSD$"); 61 62 #include <sys/errno.h> 63 #include <sys/types.h> 64 #include <net/ieee_oui.h> 65 66 #include <assert.h> 67 #include <pthread.h> 68 #include <pthread_np.h> 69 #include <semaphore.h> 70 #include <stdbool.h> 71 #include <stddef.h> 72 #include <stdint.h> 73 #include <stdio.h> 74 #include <stdlib.h> 75 #include <string.h> 76 77 #include <machine/atomic.h> 78 #include <machine/vmm.h> 79 #include <vmmapi.h> 80 81 #include <dev/nvme/nvme.h> 82 83 #include "bhyverun.h" 84 #include "block_if.h" 85 #include "config.h" 86 #include "debug.h" 87 #include "pci_emul.h" 88 89 90 static int nvme_debug = 0; 91 #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args) 92 #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args) 93 94 /* defaults; can be overridden */ 95 #define NVME_MSIX_BAR 4 96 97 #define NVME_IOSLOTS 8 98 99 /* The NVMe spec defines bits 13:4 in BAR0 as reserved */ 100 #define NVME_MMIO_SPACE_MIN (1 << 14) 101 102 #define NVME_QUEUES 16 103 #define NVME_MAX_QENTRIES 2048 104 /* Memory Page size Minimum reported in CAP register */ 105 #define NVME_MPSMIN 0 106 /* MPSMIN converted to bytes */ 107 #define NVME_MPSMIN_BYTES (1 << (12 + NVME_MPSMIN)) 108 109 #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t)) 110 #define NVME_MDTS 9 111 /* Note the + 1 allows for the initial descriptor to not be page aligned */ 112 #define NVME_MAX_IOVEC ((1 << NVME_MDTS) + 1) 113 #define NVME_MAX_DATA_SIZE ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES) 114 115 /* This is a synthetic status code to indicate there is no status */ 116 #define NVME_NO_STATUS 0xffff 117 #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS) 118 119 /* Reported temperature in Kelvin (i.e. room temperature) */ 120 #define NVME_TEMPERATURE 296 121 122 /* helpers */ 123 124 /* Convert a zero-based value into a one-based value */ 125 #define ONE_BASED(zero) ((zero) + 1) 126 /* Convert a one-based value into a zero-based value */ 127 #define ZERO_BASED(one) ((one) - 1) 128 129 /* Encode number of SQ's and CQ's for Set/Get Features */ 130 #define NVME_FEATURE_NUM_QUEUES(sc) \ 131 (ZERO_BASED((sc)->num_squeues) & 0xffff) | \ 132 (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16; 133 134 #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell) 135 136 enum nvme_controller_register_offsets { 137 NVME_CR_CAP_LOW = 0x00, 138 NVME_CR_CAP_HI = 0x04, 139 NVME_CR_VS = 0x08, 140 NVME_CR_INTMS = 0x0c, 141 NVME_CR_INTMC = 0x10, 142 NVME_CR_CC = 0x14, 143 NVME_CR_CSTS = 0x1c, 144 NVME_CR_NSSR = 0x20, 145 NVME_CR_AQA = 0x24, 146 NVME_CR_ASQ_LOW = 0x28, 147 NVME_CR_ASQ_HI = 0x2c, 148 NVME_CR_ACQ_LOW = 0x30, 149 NVME_CR_ACQ_HI = 0x34, 150 }; 151 152 enum nvme_cmd_cdw11 { 153 NVME_CMD_CDW11_PC = 0x0001, 154 NVME_CMD_CDW11_IEN = 0x0002, 155 NVME_CMD_CDW11_IV = 0xFFFF0000, 156 }; 157 158 enum nvme_copy_dir { 159 NVME_COPY_TO_PRP, 160 NVME_COPY_FROM_PRP, 161 }; 162 163 #define NVME_CQ_INTEN 0x01 164 #define NVME_CQ_INTCOAL 0x02 165 166 struct nvme_completion_queue { 167 struct nvme_completion *qbase; 168 pthread_mutex_t mtx; 169 uint32_t size; 170 uint16_t tail; /* nvme progress */ 171 uint16_t head; /* guest progress */ 172 uint16_t intr_vec; 173 uint32_t intr_en; 174 }; 175 176 struct nvme_submission_queue { 177 struct nvme_command *qbase; 178 pthread_mutex_t mtx; 179 uint32_t size; 180 uint16_t head; /* nvme progress */ 181 uint16_t tail; /* guest progress */ 182 uint16_t cqid; /* completion queue id */ 183 int qpriority; 184 }; 185 186 enum nvme_storage_type { 187 NVME_STOR_BLOCKIF = 0, 188 NVME_STOR_RAM = 1, 189 }; 190 191 struct pci_nvme_blockstore { 192 enum nvme_storage_type type; 193 void *ctx; 194 uint64_t size; 195 uint32_t sectsz; 196 uint32_t sectsz_bits; 197 uint64_t eui64; 198 uint32_t deallocate:1; 199 }; 200 201 /* 202 * Calculate the number of additional page descriptors for guest IO requests 203 * based on the advertised Max Data Transfer (MDTS) and given the number of 204 * default iovec's in a struct blockif_req. 205 */ 206 #define MDTS_PAD_SIZE \ 207 ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \ 208 NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \ 209 0 ) 210 211 struct pci_nvme_ioreq { 212 struct pci_nvme_softc *sc; 213 STAILQ_ENTRY(pci_nvme_ioreq) link; 214 struct nvme_submission_queue *nvme_sq; 215 uint16_t sqid; 216 217 /* command information */ 218 uint16_t opc; 219 uint16_t cid; 220 uint32_t nsid; 221 222 uint64_t prev_gpaddr; 223 size_t prev_size; 224 size_t bytes; 225 226 struct blockif_req io_req; 227 228 struct iovec iovpadding[MDTS_PAD_SIZE]; 229 }; 230 231 enum nvme_dsm_type { 232 /* Dataset Management bit in ONCS reflects backing storage capability */ 233 NVME_DATASET_MANAGEMENT_AUTO, 234 /* Unconditionally set Dataset Management bit in ONCS */ 235 NVME_DATASET_MANAGEMENT_ENABLE, 236 /* Unconditionally clear Dataset Management bit in ONCS */ 237 NVME_DATASET_MANAGEMENT_DISABLE, 238 }; 239 240 struct pci_nvme_softc; 241 struct nvme_feature_obj; 242 243 typedef void (*nvme_feature_cb)(struct pci_nvme_softc *, 244 struct nvme_feature_obj *, 245 struct nvme_command *, 246 struct nvme_completion *); 247 248 struct nvme_feature_obj { 249 uint32_t cdw11; 250 nvme_feature_cb set; 251 nvme_feature_cb get; 252 bool namespace_specific; 253 }; 254 255 #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) 256 257 typedef enum { 258 PCI_NVME_AE_TYPE_ERROR = 0, 259 PCI_NVME_AE_TYPE_SMART, 260 PCI_NVME_AE_TYPE_NOTICE, 261 PCI_NVME_AE_TYPE_IO_CMD = 6, 262 PCI_NVME_AE_TYPE_VENDOR = 7, 263 PCI_NVME_AE_TYPE_MAX /* Must be last */ 264 } pci_nvme_async_type; 265 266 /* Asynchronous Event Requests */ 267 struct pci_nvme_aer { 268 STAILQ_ENTRY(pci_nvme_aer) link; 269 uint16_t cid; /* Command ID of the submitted AER */ 270 }; 271 272 /** Asynchronous Event Information - Notice */ 273 typedef enum { 274 PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0, 275 PCI_NVME_AEI_NOTICE_FW_ACTIVATION, 276 PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE, 277 PCI_NVME_AEI_NOTICE_ANA_CHANGE, 278 PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE, 279 PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT, 280 PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE, 281 PCI_NVME_AEI_NOTICE_MAX, 282 } pci_nvme_async_event_info_notice; 283 284 #define PCI_NVME_AEI_NOTICE_SHIFT 8 285 #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT)) 286 287 /* Asynchronous Event Notifications */ 288 struct pci_nvme_aen { 289 pci_nvme_async_type atype; 290 uint32_t event_data; 291 bool posted; 292 }; 293 294 /* 295 * By default, enable all Asynchrnous Event Notifications: 296 * SMART / Health Critical Warnings 297 * Namespace Attribute Notices 298 */ 299 #define PCI_NVME_AEN_DEFAULT_MASK 0x11f 300 301 typedef enum { 302 NVME_CNTRLTYPE_IO = 1, 303 NVME_CNTRLTYPE_DISCOVERY = 2, 304 NVME_CNTRLTYPE_ADMIN = 3, 305 } pci_nvme_cntrl_type; 306 307 struct pci_nvme_softc { 308 struct pci_devinst *nsc_pi; 309 310 pthread_mutex_t mtx; 311 312 struct nvme_registers regs; 313 314 struct nvme_namespace_data nsdata; 315 struct nvme_controller_data ctrldata; 316 struct nvme_error_information_entry err_log; 317 struct nvme_health_information_page health_log; 318 struct nvme_firmware_page fw_log; 319 struct nvme_ns_list ns_log; 320 321 struct pci_nvme_blockstore nvstore; 322 323 uint16_t max_qentries; /* max entries per queue */ 324 uint32_t max_queues; /* max number of IO SQ's or CQ's */ 325 uint32_t num_cqueues; 326 uint32_t num_squeues; 327 bool num_q_is_set; /* Has host set Number of Queues */ 328 329 struct pci_nvme_ioreq *ioreqs; 330 STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */ 331 uint32_t pending_ios; 332 uint32_t ioslots; 333 sem_t iosemlock; 334 335 /* 336 * Memory mapped Submission and Completion queues 337 * Each array includes both Admin and IO queues 338 */ 339 struct nvme_completion_queue *compl_queues; 340 struct nvme_submission_queue *submit_queues; 341 342 struct nvme_feature_obj feat[NVME_FID_MAX]; 343 344 enum nvme_dsm_type dataset_management; 345 346 /* Accounting for SMART data */ 347 __uint128_t read_data_units; 348 __uint128_t write_data_units; 349 __uint128_t read_commands; 350 __uint128_t write_commands; 351 uint32_t read_dunits_remainder; 352 uint32_t write_dunits_remainder; 353 354 STAILQ_HEAD(, pci_nvme_aer) aer_list; 355 pthread_mutex_t aer_mtx; 356 uint32_t aer_count; 357 struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX]; 358 pthread_t aen_tid; 359 pthread_mutex_t aen_mtx; 360 pthread_cond_t aen_cond; 361 }; 362 363 364 static void pci_nvme_cq_update(struct pci_nvme_softc *sc, 365 struct nvme_completion_queue *cq, 366 uint32_t cdw0, 367 uint16_t cid, 368 uint16_t sqid, 369 uint16_t status); 370 static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); 371 static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); 372 static void pci_nvme_io_done(struct blockif_req *, int); 373 374 /* Controller Configuration utils */ 375 #define NVME_CC_GET_EN(cc) \ 376 ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK) 377 #define NVME_CC_GET_CSS(cc) \ 378 ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK) 379 #define NVME_CC_GET_SHN(cc) \ 380 ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK) 381 #define NVME_CC_GET_IOSQES(cc) \ 382 ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK) 383 #define NVME_CC_GET_IOCQES(cc) \ 384 ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK) 385 386 #define NVME_CC_WRITE_MASK \ 387 ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \ 388 (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \ 389 (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT)) 390 391 #define NVME_CC_NEN_WRITE_MASK \ 392 ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \ 393 (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \ 394 (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT)) 395 396 /* Controller Status utils */ 397 #define NVME_CSTS_GET_RDY(sts) \ 398 ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK) 399 400 #define NVME_CSTS_RDY (1 << NVME_CSTS_REG_RDY_SHIFT) 401 402 /* Completion Queue status word utils */ 403 #define NVME_STATUS_P (1 << NVME_STATUS_P_SHIFT) 404 #define NVME_STATUS_MASK \ 405 ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\ 406 (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT)) 407 408 #define NVME_ONCS_DSM (NVME_CTRLR_DATA_ONCS_DSM_MASK << \ 409 NVME_CTRLR_DATA_ONCS_DSM_SHIFT) 410 411 static void nvme_feature_invalid_cb(struct pci_nvme_softc *, 412 struct nvme_feature_obj *, 413 struct nvme_command *, 414 struct nvme_completion *); 415 static void nvme_feature_temperature(struct pci_nvme_softc *, 416 struct nvme_feature_obj *, 417 struct nvme_command *, 418 struct nvme_completion *); 419 static void nvme_feature_num_queues(struct pci_nvme_softc *, 420 struct nvme_feature_obj *, 421 struct nvme_command *, 422 struct nvme_completion *); 423 static void nvme_feature_iv_config(struct pci_nvme_softc *, 424 struct nvme_feature_obj *, 425 struct nvme_command *, 426 struct nvme_completion *); 427 static void nvme_feature_async_event(struct pci_nvme_softc *, 428 struct nvme_feature_obj *, 429 struct nvme_command *, 430 struct nvme_completion *); 431 432 static void *aen_thr(void *arg); 433 434 static __inline void 435 cpywithpad(char *dst, size_t dst_size, const char *src, char pad) 436 { 437 size_t len; 438 439 len = strnlen(src, dst_size); 440 memset(dst, pad, dst_size); 441 memcpy(dst, src, len); 442 } 443 444 static __inline void 445 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code) 446 { 447 448 *status &= ~NVME_STATUS_MASK; 449 *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT | 450 (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT; 451 } 452 453 static __inline void 454 pci_nvme_status_genc(uint16_t *status, uint16_t code) 455 { 456 457 pci_nvme_status_tc(status, NVME_SCT_GENERIC, code); 458 } 459 460 /* 461 * Initialize the requested number or IO Submission and Completion Queues. 462 * Admin queues are allocated implicitly. 463 */ 464 static void 465 pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq) 466 { 467 uint32_t i; 468 469 /* 470 * Allocate and initialize the Submission Queues 471 */ 472 if (nsq > NVME_QUEUES) { 473 WPRINTF("%s: clamping number of SQ from %u to %u", 474 __func__, nsq, NVME_QUEUES); 475 nsq = NVME_QUEUES; 476 } 477 478 sc->num_squeues = nsq; 479 480 sc->submit_queues = calloc(sc->num_squeues + 1, 481 sizeof(struct nvme_submission_queue)); 482 if (sc->submit_queues == NULL) { 483 WPRINTF("%s: SQ allocation failed", __func__); 484 sc->num_squeues = 0; 485 } else { 486 struct nvme_submission_queue *sq = sc->submit_queues; 487 488 for (i = 0; i < sc->num_squeues + 1; i++) 489 pthread_mutex_init(&sq[i].mtx, NULL); 490 } 491 492 /* 493 * Allocate and initialize the Completion Queues 494 */ 495 if (ncq > NVME_QUEUES) { 496 WPRINTF("%s: clamping number of CQ from %u to %u", 497 __func__, ncq, NVME_QUEUES); 498 ncq = NVME_QUEUES; 499 } 500 501 sc->num_cqueues = ncq; 502 503 sc->compl_queues = calloc(sc->num_cqueues + 1, 504 sizeof(struct nvme_completion_queue)); 505 if (sc->compl_queues == NULL) { 506 WPRINTF("%s: CQ allocation failed", __func__); 507 sc->num_cqueues = 0; 508 } else { 509 struct nvme_completion_queue *cq = sc->compl_queues; 510 511 for (i = 0; i < sc->num_cqueues + 1; i++) 512 pthread_mutex_init(&cq[i].mtx, NULL); 513 } 514 } 515 516 static void 517 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc) 518 { 519 struct nvme_controller_data *cd = &sc->ctrldata; 520 521 cd->vid = 0xFB5D; 522 cd->ssvid = 0x0000; 523 524 cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' '); 525 cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' '); 526 527 /* Num of submission commands that we can handle at a time (2^rab) */ 528 cd->rab = 4; 529 530 /* FreeBSD OUI */ 531 cd->ieee[0] = 0x58; 532 cd->ieee[1] = 0x9c; 533 cd->ieee[2] = 0xfc; 534 535 cd->mic = 0; 536 537 cd->mdts = NVME_MDTS; /* max data transfer size (2^mdts * CAP.MPSMIN) */ 538 539 cd->ver = NVME_REV(1,4); 540 541 cd->cntrltype = NVME_CNTRLTYPE_IO; 542 cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT; 543 cd->oaes = NVMEB(NVME_CTRLR_DATA_OAES_NS_ATTR); 544 cd->acl = 2; 545 cd->aerl = 4; 546 547 /* Advertise 1, Read-only firmware slot */ 548 cd->frmw = NVME_CTRLR_DATA_FRMW_SLOT1_RO_MASK | 549 (1 << NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT); 550 cd->lpa = 0; /* TODO: support some simple things like SMART */ 551 cd->elpe = 0; /* max error log page entries */ 552 cd->npss = 1; /* number of power states support */ 553 554 /* Warning Composite Temperature Threshold */ 555 cd->wctemp = 0x0157; 556 cd->cctemp = 0x0157; 557 558 cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) | 559 (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT); 560 cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) | 561 (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT); 562 cd->nn = 1; /* number of namespaces */ 563 564 cd->oncs = 0; 565 switch (sc->dataset_management) { 566 case NVME_DATASET_MANAGEMENT_AUTO: 567 if (sc->nvstore.deallocate) 568 cd->oncs |= NVME_ONCS_DSM; 569 break; 570 case NVME_DATASET_MANAGEMENT_ENABLE: 571 cd->oncs |= NVME_ONCS_DSM; 572 break; 573 default: 574 break; 575 } 576 577 cd->fna = NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK << 578 NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT; 579 580 cd->vwc = NVME_CTRLR_DATA_VWC_ALL_NO << NVME_CTRLR_DATA_VWC_ALL_SHIFT; 581 582 cd->power_state[0].mp = 10; 583 } 584 585 /* 586 * Calculate the CRC-16 of the given buffer 587 * See copyright attribution at top of file 588 */ 589 static uint16_t 590 crc16(uint16_t crc, const void *buffer, unsigned int len) 591 { 592 const unsigned char *cp = buffer; 593 /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ 594 static uint16_t const crc16_table[256] = { 595 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 596 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 597 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 598 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 599 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 600 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 601 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 602 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 603 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 604 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 605 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 606 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 607 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 608 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 609 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 610 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 611 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 612 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 613 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 614 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 615 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 616 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 617 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 618 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 619 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 620 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 621 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 622 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 623 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 624 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 625 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 626 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 627 }; 628 629 while (len--) 630 crc = (((crc >> 8) & 0xffU) ^ 631 crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; 632 return crc; 633 } 634 635 static void 636 pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore, 637 struct nvme_namespace_data *nd) 638 { 639 640 /* Get capacity and block size information from backing store */ 641 nd->nsze = nvstore->size / nvstore->sectsz; 642 nd->ncap = nd->nsze; 643 nd->nuse = nd->nsze; 644 } 645 646 static void 647 pci_nvme_init_nsdata(struct pci_nvme_softc *sc, 648 struct nvme_namespace_data *nd, uint32_t nsid, 649 struct pci_nvme_blockstore *nvstore) 650 { 651 652 pci_nvme_init_nsdata_size(nvstore, nd); 653 654 if (nvstore->type == NVME_STOR_BLOCKIF) 655 nvstore->deallocate = blockif_candelete(nvstore->ctx); 656 657 nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */ 658 nd->flbas = 0; 659 660 /* Create an EUI-64 if user did not provide one */ 661 if (nvstore->eui64 == 0) { 662 char *data = NULL; 663 uint64_t eui64 = nvstore->eui64; 664 665 asprintf(&data, "%s%u%u%u", get_config_value("name"), 666 sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, 667 sc->nsc_pi->pi_func); 668 669 if (data != NULL) { 670 eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data)); 671 free(data); 672 } 673 nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff); 674 } 675 be64enc(nd->eui64, nvstore->eui64); 676 677 /* LBA data-sz = 2^lbads */ 678 nd->lbaf[0] = nvstore->sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT; 679 } 680 681 static void 682 pci_nvme_init_logpages(struct pci_nvme_softc *sc) 683 { 684 685 memset(&sc->err_log, 0, sizeof(sc->err_log)); 686 memset(&sc->health_log, 0, sizeof(sc->health_log)); 687 memset(&sc->fw_log, 0, sizeof(sc->fw_log)); 688 memset(&sc->ns_log, 0, sizeof(sc->ns_log)); 689 690 /* Set read/write remainder to round up according to spec */ 691 sc->read_dunits_remainder = 999; 692 sc->write_dunits_remainder = 999; 693 694 /* Set nominal Health values checked by implementations */ 695 sc->health_log.temperature = NVME_TEMPERATURE; 696 sc->health_log.available_spare = 100; 697 sc->health_log.available_spare_threshold = 10; 698 } 699 700 static void 701 pci_nvme_init_features(struct pci_nvme_softc *sc) 702 { 703 enum nvme_feature fid; 704 705 for (fid = 0; fid < NVME_FID_MAX; fid++) { 706 switch (fid) { 707 case NVME_FEAT_ARBITRATION: 708 case NVME_FEAT_POWER_MANAGEMENT: 709 case NVME_FEAT_INTERRUPT_COALESCING: //XXX 710 case NVME_FEAT_WRITE_ATOMICITY: 711 /* Mandatory but no special handling required */ 712 //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: 713 //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 714 // this returns a data buffer 715 break; 716 case NVME_FEAT_TEMPERATURE_THRESHOLD: 717 sc->feat[fid].set = nvme_feature_temperature; 718 break; 719 case NVME_FEAT_ERROR_RECOVERY: 720 sc->feat[fid].namespace_specific = true; 721 break; 722 case NVME_FEAT_NUMBER_OF_QUEUES: 723 sc->feat[fid].set = nvme_feature_num_queues; 724 break; 725 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 726 sc->feat[fid].set = nvme_feature_iv_config; 727 break; 728 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 729 sc->feat[fid].set = nvme_feature_async_event; 730 /* Enable all AENs by default */ 731 sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK; 732 break; 733 default: 734 sc->feat[fid].set = nvme_feature_invalid_cb; 735 sc->feat[fid].get = nvme_feature_invalid_cb; 736 } 737 } 738 } 739 740 static void 741 pci_nvme_aer_reset(struct pci_nvme_softc *sc) 742 { 743 744 STAILQ_INIT(&sc->aer_list); 745 sc->aer_count = 0; 746 } 747 748 static void 749 pci_nvme_aer_init(struct pci_nvme_softc *sc) 750 { 751 752 pthread_mutex_init(&sc->aer_mtx, NULL); 753 pci_nvme_aer_reset(sc); 754 } 755 756 static void 757 pci_nvme_aer_destroy(struct pci_nvme_softc *sc) 758 { 759 struct pci_nvme_aer *aer = NULL; 760 761 pthread_mutex_lock(&sc->aer_mtx); 762 while (!STAILQ_EMPTY(&sc->aer_list)) { 763 aer = STAILQ_FIRST(&sc->aer_list); 764 STAILQ_REMOVE_HEAD(&sc->aer_list, link); 765 free(aer); 766 } 767 pthread_mutex_unlock(&sc->aer_mtx); 768 769 pci_nvme_aer_reset(sc); 770 } 771 772 static bool 773 pci_nvme_aer_available(struct pci_nvme_softc *sc) 774 { 775 776 return (sc->aer_count != 0); 777 } 778 779 static bool 780 pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc) 781 { 782 struct nvme_controller_data *cd = &sc->ctrldata; 783 784 /* AERL is a zero based value while aer_count is one's based */ 785 return (sc->aer_count == (cd->aerl + 1)); 786 } 787 788 /* 789 * Add an Async Event Request 790 * 791 * Stores an AER to be returned later if the Controller needs to notify the 792 * host of an event. 793 * Note that while the NVMe spec doesn't require Controllers to return AER's 794 * in order, this implementation does preserve the order. 795 */ 796 static int 797 pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid) 798 { 799 struct pci_nvme_aer *aer = NULL; 800 801 aer = calloc(1, sizeof(struct pci_nvme_aer)); 802 if (aer == NULL) 803 return (-1); 804 805 /* Save the Command ID for use in the completion message */ 806 aer->cid = cid; 807 808 pthread_mutex_lock(&sc->aer_mtx); 809 sc->aer_count++; 810 STAILQ_INSERT_TAIL(&sc->aer_list, aer, link); 811 pthread_mutex_unlock(&sc->aer_mtx); 812 813 return (0); 814 } 815 816 /* 817 * Get an Async Event Request structure 818 * 819 * Returns a pointer to an AER previously submitted by the host or NULL if 820 * no AER's exist. Caller is responsible for freeing the returned struct. 821 */ 822 static struct pci_nvme_aer * 823 pci_nvme_aer_get(struct pci_nvme_softc *sc) 824 { 825 struct pci_nvme_aer *aer = NULL; 826 827 pthread_mutex_lock(&sc->aer_mtx); 828 aer = STAILQ_FIRST(&sc->aer_list); 829 if (aer != NULL) { 830 STAILQ_REMOVE_HEAD(&sc->aer_list, link); 831 sc->aer_count--; 832 } 833 pthread_mutex_unlock(&sc->aer_mtx); 834 835 return (aer); 836 } 837 838 static void 839 pci_nvme_aen_reset(struct pci_nvme_softc *sc) 840 { 841 uint32_t atype; 842 843 memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen)); 844 845 for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { 846 sc->aen[atype].atype = atype; 847 } 848 } 849 850 static void 851 pci_nvme_aen_init(struct pci_nvme_softc *sc) 852 { 853 char nstr[80]; 854 855 pci_nvme_aen_reset(sc); 856 857 pthread_mutex_init(&sc->aen_mtx, NULL); 858 pthread_create(&sc->aen_tid, NULL, aen_thr, sc); 859 snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot, 860 sc->nsc_pi->pi_func); 861 pthread_set_name_np(sc->aen_tid, nstr); 862 } 863 864 static void 865 pci_nvme_aen_destroy(struct pci_nvme_softc *sc) 866 { 867 868 pci_nvme_aen_reset(sc); 869 } 870 871 /* Notify the AEN thread of pending work */ 872 static void 873 pci_nvme_aen_notify(struct pci_nvme_softc *sc) 874 { 875 876 pthread_cond_signal(&sc->aen_cond); 877 } 878 879 /* 880 * Post an Asynchronous Event Notification 881 */ 882 static int32_t 883 pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype, 884 uint32_t event_data) 885 { 886 struct pci_nvme_aen *aen; 887 888 if (atype >= PCI_NVME_AE_TYPE_MAX) { 889 return(EINVAL); 890 } 891 892 pthread_mutex_lock(&sc->aen_mtx); 893 aen = &sc->aen[atype]; 894 895 /* Has the controller already posted an event of this type? */ 896 if (aen->posted) { 897 pthread_mutex_unlock(&sc->aen_mtx); 898 return(EALREADY); 899 } 900 901 aen->event_data = event_data; 902 aen->posted = true; 903 pthread_mutex_unlock(&sc->aen_mtx); 904 905 pci_nvme_aen_notify(sc); 906 907 return(0); 908 } 909 910 static void 911 pci_nvme_aen_process(struct pci_nvme_softc *sc) 912 { 913 struct pci_nvme_aer *aer; 914 struct pci_nvme_aen *aen; 915 pci_nvme_async_type atype; 916 uint32_t mask; 917 uint16_t status; 918 uint8_t lid; 919 920 assert(pthread_mutex_isowned_np(&sc->aen_mtx)); 921 for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { 922 aen = &sc->aen[atype]; 923 /* Previous iterations may have depleted the available AER's */ 924 if (!pci_nvme_aer_available(sc)) { 925 DPRINTF("%s: no AER", __func__); 926 break; 927 } 928 929 if (!aen->posted) { 930 DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype); 931 continue; 932 } 933 934 status = NVME_SC_SUCCESS; 935 936 /* Is the event masked? */ 937 mask = 938 sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11; 939 940 DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data); 941 switch (atype) { 942 case PCI_NVME_AE_TYPE_ERROR: 943 lid = NVME_LOG_ERROR; 944 break; 945 case PCI_NVME_AE_TYPE_SMART: 946 mask &= 0xff; 947 if ((mask & aen->event_data) == 0) 948 continue; 949 lid = NVME_LOG_HEALTH_INFORMATION; 950 break; 951 case PCI_NVME_AE_TYPE_NOTICE: 952 if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) { 953 EPRINTLN("%s unknown AEN notice type %u", 954 __func__, aen->event_data); 955 status = NVME_SC_INTERNAL_DEVICE_ERROR; 956 break; 957 } 958 if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0) 959 continue; 960 switch (aen->event_data) { 961 case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED: 962 lid = NVME_LOG_CHANGED_NAMESPACE; 963 break; 964 case PCI_NVME_AEI_NOTICE_FW_ACTIVATION: 965 lid = NVME_LOG_FIRMWARE_SLOT; 966 break; 967 case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE: 968 lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED; 969 break; 970 case PCI_NVME_AEI_NOTICE_ANA_CHANGE: 971 lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; 972 break; 973 case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE: 974 lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE; 975 break; 976 case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT: 977 lid = NVME_LOG_LBA_STATUS_INFORMATION; 978 break; 979 case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE: 980 lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE; 981 break; 982 default: 983 lid = 0; 984 } 985 break; 986 default: 987 /* bad type?!? */ 988 EPRINTLN("%s unknown AEN type %u", __func__, atype); 989 status = NVME_SC_INTERNAL_DEVICE_ERROR; 990 break; 991 } 992 993 aer = pci_nvme_aer_get(sc); 994 assert(aer != NULL); 995 996 DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype); 997 pci_nvme_cq_update(sc, &sc->compl_queues[0], 998 (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */ 999 aer->cid, 1000 0, /* SQID */ 1001 status); 1002 1003 aen->event_data = 0; 1004 aen->posted = false; 1005 1006 pci_generate_msix(sc->nsc_pi, 0); 1007 } 1008 } 1009 1010 static void * 1011 aen_thr(void *arg) 1012 { 1013 struct pci_nvme_softc *sc; 1014 1015 sc = arg; 1016 1017 pthread_mutex_lock(&sc->aen_mtx); 1018 for (;;) { 1019 pci_nvme_aen_process(sc); 1020 pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx); 1021 } 1022 pthread_mutex_unlock(&sc->aen_mtx); 1023 1024 pthread_exit(NULL); 1025 return (NULL); 1026 } 1027 1028 static void 1029 pci_nvme_reset_locked(struct pci_nvme_softc *sc) 1030 { 1031 uint32_t i; 1032 1033 DPRINTF("%s", __func__); 1034 1035 sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | 1036 (1 << NVME_CAP_LO_REG_CQR_SHIFT) | 1037 (60 << NVME_CAP_LO_REG_TO_SHIFT); 1038 1039 sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT; 1040 1041 sc->regs.vs = NVME_REV(1,4); /* NVMe v1.4 */ 1042 1043 sc->regs.cc = 0; 1044 1045 assert(sc->submit_queues != NULL); 1046 1047 for (i = 0; i < sc->num_squeues + 1; i++) { 1048 sc->submit_queues[i].qbase = NULL; 1049 sc->submit_queues[i].size = 0; 1050 sc->submit_queues[i].cqid = 0; 1051 sc->submit_queues[i].tail = 0; 1052 sc->submit_queues[i].head = 0; 1053 } 1054 1055 assert(sc->compl_queues != NULL); 1056 1057 for (i = 0; i < sc->num_cqueues + 1; i++) { 1058 sc->compl_queues[i].qbase = NULL; 1059 sc->compl_queues[i].size = 0; 1060 sc->compl_queues[i].tail = 0; 1061 sc->compl_queues[i].head = 0; 1062 } 1063 1064 sc->num_q_is_set = false; 1065 1066 pci_nvme_aer_destroy(sc); 1067 pci_nvme_aen_destroy(sc); 1068 1069 /* 1070 * Clear CSTS.RDY last to prevent the host from enabling Controller 1071 * before cleanup completes 1072 */ 1073 sc->regs.csts = 0; 1074 } 1075 1076 static void 1077 pci_nvme_reset(struct pci_nvme_softc *sc) 1078 { 1079 pthread_mutex_lock(&sc->mtx); 1080 pci_nvme_reset_locked(sc); 1081 pthread_mutex_unlock(&sc->mtx); 1082 } 1083 1084 static void 1085 pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc) 1086 { 1087 uint16_t acqs, asqs; 1088 1089 DPRINTF("%s", __func__); 1090 1091 asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1; 1092 sc->submit_queues[0].size = asqs; 1093 sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq, 1094 sizeof(struct nvme_command) * asqs); 1095 1096 DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p", 1097 __func__, sc->regs.asq, sc->submit_queues[0].qbase); 1098 1099 acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) & 1100 NVME_AQA_REG_ACQS_MASK) + 1; 1101 sc->compl_queues[0].size = acqs; 1102 sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq, 1103 sizeof(struct nvme_completion) * acqs); 1104 sc->compl_queues[0].intr_en = NVME_CQ_INTEN; 1105 1106 DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p", 1107 __func__, sc->regs.acq, sc->compl_queues[0].qbase); 1108 } 1109 1110 static int 1111 nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b, 1112 size_t len, enum nvme_copy_dir dir) 1113 { 1114 uint8_t *p; 1115 size_t bytes; 1116 1117 if (len > (8 * 1024)) { 1118 return (-1); 1119 } 1120 1121 /* Copy from the start of prp1 to the end of the physical page */ 1122 bytes = PAGE_SIZE - (prp1 & PAGE_MASK); 1123 bytes = MIN(bytes, len); 1124 1125 p = vm_map_gpa(ctx, prp1, bytes); 1126 if (p == NULL) { 1127 return (-1); 1128 } 1129 1130 if (dir == NVME_COPY_TO_PRP) 1131 memcpy(p, b, bytes); 1132 else 1133 memcpy(b, p, bytes); 1134 1135 b += bytes; 1136 1137 len -= bytes; 1138 if (len == 0) { 1139 return (0); 1140 } 1141 1142 len = MIN(len, PAGE_SIZE); 1143 1144 p = vm_map_gpa(ctx, prp2, len); 1145 if (p == NULL) { 1146 return (-1); 1147 } 1148 1149 if (dir == NVME_COPY_TO_PRP) 1150 memcpy(p, b, len); 1151 else 1152 memcpy(b, p, len); 1153 1154 return (0); 1155 } 1156 1157 /* 1158 * Write a Completion Queue Entry update 1159 * 1160 * Write the completion and update the doorbell value 1161 */ 1162 static void 1163 pci_nvme_cq_update(struct pci_nvme_softc *sc, 1164 struct nvme_completion_queue *cq, 1165 uint32_t cdw0, 1166 uint16_t cid, 1167 uint16_t sqid, 1168 uint16_t status) 1169 { 1170 struct nvme_submission_queue *sq = &sc->submit_queues[sqid]; 1171 struct nvme_completion *cqe; 1172 1173 assert(cq->qbase != NULL); 1174 1175 pthread_mutex_lock(&cq->mtx); 1176 1177 cqe = &cq->qbase[cq->tail]; 1178 1179 /* Flip the phase bit */ 1180 status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK; 1181 1182 cqe->cdw0 = cdw0; 1183 cqe->sqhd = sq->head; 1184 cqe->sqid = sqid; 1185 cqe->cid = cid; 1186 cqe->status = status; 1187 1188 cq->tail++; 1189 if (cq->tail >= cq->size) { 1190 cq->tail = 0; 1191 } 1192 1193 pthread_mutex_unlock(&cq->mtx); 1194 } 1195 1196 static int 1197 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 1198 struct nvme_completion* compl) 1199 { 1200 uint16_t qid = command->cdw10 & 0xffff; 1201 1202 DPRINTF("%s DELETE_IO_SQ %u", __func__, qid); 1203 if (qid == 0 || qid > sc->num_squeues || 1204 (sc->submit_queues[qid].qbase == NULL)) { 1205 WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u", 1206 __func__, qid, sc->num_squeues); 1207 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1208 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1209 return (1); 1210 } 1211 1212 sc->submit_queues[qid].qbase = NULL; 1213 sc->submit_queues[qid].cqid = 0; 1214 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1215 return (1); 1216 } 1217 1218 static int 1219 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, 1220 struct nvme_completion* compl) 1221 { 1222 if (command->cdw11 & NVME_CMD_CDW11_PC) { 1223 uint16_t qid = command->cdw10 & 0xffff; 1224 struct nvme_submission_queue *nsq; 1225 1226 if ((qid == 0) || (qid > sc->num_squeues) || 1227 (sc->submit_queues[qid].qbase != NULL)) { 1228 WPRINTF("%s queue index %u > num_squeues %u", 1229 __func__, qid, sc->num_squeues); 1230 pci_nvme_status_tc(&compl->status, 1231 NVME_SCT_COMMAND_SPECIFIC, 1232 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1233 return (1); 1234 } 1235 1236 nsq = &sc->submit_queues[qid]; 1237 nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 1238 DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries); 1239 if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) { 1240 /* 1241 * Queues must specify at least two entries 1242 * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to 1243 * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec 1244 */ 1245 pci_nvme_status_tc(&compl->status, 1246 NVME_SCT_COMMAND_SPECIFIC, 1247 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); 1248 return (1); 1249 } 1250 nsq->head = nsq->tail = 0; 1251 1252 nsq->cqid = (command->cdw11 >> 16) & 0xffff; 1253 if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) { 1254 pci_nvme_status_tc(&compl->status, 1255 NVME_SCT_COMMAND_SPECIFIC, 1256 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1257 return (1); 1258 } 1259 1260 if (sc->compl_queues[nsq->cqid].qbase == NULL) { 1261 pci_nvme_status_tc(&compl->status, 1262 NVME_SCT_COMMAND_SPECIFIC, 1263 NVME_SC_COMPLETION_QUEUE_INVALID); 1264 return (1); 1265 } 1266 1267 nsq->qpriority = (command->cdw11 >> 1) & 0x03; 1268 1269 nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1270 sizeof(struct nvme_command) * (size_t)nsq->size); 1271 1272 DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__, 1273 qid, nsq->size, nsq->qbase, nsq->cqid); 1274 1275 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1276 1277 DPRINTF("%s completed creating IOSQ qid %u", 1278 __func__, qid); 1279 } else { 1280 /* 1281 * Guest sent non-cont submission queue request. 1282 * This setting is unsupported by this emulation. 1283 */ 1284 WPRINTF("%s unsupported non-contig (list-based) " 1285 "create i/o submission queue", __func__); 1286 1287 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1288 } 1289 return (1); 1290 } 1291 1292 static int 1293 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 1294 struct nvme_completion* compl) 1295 { 1296 uint16_t qid = command->cdw10 & 0xffff; 1297 uint16_t sqid; 1298 1299 DPRINTF("%s DELETE_IO_CQ %u", __func__, qid); 1300 if (qid == 0 || qid > sc->num_cqueues || 1301 (sc->compl_queues[qid].qbase == NULL)) { 1302 WPRINTF("%s queue index %u / num_cqueues %u", 1303 __func__, qid, sc->num_cqueues); 1304 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1305 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1306 return (1); 1307 } 1308 1309 /* Deleting an Active CQ is an error */ 1310 for (sqid = 1; sqid < sc->num_squeues + 1; sqid++) 1311 if (sc->submit_queues[sqid].cqid == qid) { 1312 pci_nvme_status_tc(&compl->status, 1313 NVME_SCT_COMMAND_SPECIFIC, 1314 NVME_SC_INVALID_QUEUE_DELETION); 1315 return (1); 1316 } 1317 1318 sc->compl_queues[qid].qbase = NULL; 1319 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1320 return (1); 1321 } 1322 1323 static int 1324 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, 1325 struct nvme_completion* compl) 1326 { 1327 struct nvme_completion_queue *ncq; 1328 uint16_t qid = command->cdw10 & 0xffff; 1329 1330 /* Only support Physically Contiguous queues */ 1331 if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) { 1332 WPRINTF("%s unsupported non-contig (list-based) " 1333 "create i/o completion queue", 1334 __func__); 1335 1336 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1337 return (1); 1338 } 1339 1340 if ((qid == 0) || (qid > sc->num_cqueues) || 1341 (sc->compl_queues[qid].qbase != NULL)) { 1342 WPRINTF("%s queue index %u > num_cqueues %u", 1343 __func__, qid, sc->num_cqueues); 1344 pci_nvme_status_tc(&compl->status, 1345 NVME_SCT_COMMAND_SPECIFIC, 1346 NVME_SC_INVALID_QUEUE_IDENTIFIER); 1347 return (1); 1348 } 1349 1350 ncq = &sc->compl_queues[qid]; 1351 ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1; 1352 ncq->intr_vec = (command->cdw11 >> 16) & 0xffff; 1353 if (ncq->intr_vec > (sc->max_queues + 1)) { 1354 pci_nvme_status_tc(&compl->status, 1355 NVME_SCT_COMMAND_SPECIFIC, 1356 NVME_SC_INVALID_INTERRUPT_VECTOR); 1357 return (1); 1358 } 1359 1360 ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); 1361 if ((ncq->size < 2) || (ncq->size > sc->max_qentries)) { 1362 /* 1363 * Queues must specify at least two entries 1364 * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to 1365 * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec 1366 */ 1367 pci_nvme_status_tc(&compl->status, 1368 NVME_SCT_COMMAND_SPECIFIC, 1369 NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); 1370 return (1); 1371 } 1372 ncq->head = ncq->tail = 0; 1373 ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, 1374 command->prp1, 1375 sizeof(struct nvme_command) * (size_t)ncq->size); 1376 1377 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1378 1379 1380 return (1); 1381 } 1382 1383 static int 1384 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command, 1385 struct nvme_completion* compl) 1386 { 1387 uint64_t logoff; 1388 uint32_t logsize; 1389 uint8_t logpage = command->cdw10 & 0xFF; 1390 1391 DPRINTF("%s log page %u len %u", __func__, logpage, logsize); 1392 1393 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1394 1395 /* 1396 * Command specifies the number of dwords to return in fields NUMDU 1397 * and NUMDL. This is a zero-based value. 1398 */ 1399 logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1; 1400 logsize *= sizeof(uint32_t); 1401 logoff = ((uint64_t)(command->cdw13) << 32) | command->cdw12; 1402 1403 switch (logpage) { 1404 case NVME_LOG_ERROR: 1405 if (logoff >= sizeof(sc->err_log)) { 1406 pci_nvme_status_genc(&compl->status, 1407 NVME_SC_INVALID_FIELD); 1408 break; 1409 } 1410 1411 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1412 command->prp2, (uint8_t *)&sc->err_log + logoff, 1413 MIN(logsize - logoff, sizeof(sc->err_log)), 1414 NVME_COPY_TO_PRP); 1415 break; 1416 case NVME_LOG_HEALTH_INFORMATION: 1417 if (logoff >= sizeof(sc->health_log)) { 1418 pci_nvme_status_genc(&compl->status, 1419 NVME_SC_INVALID_FIELD); 1420 break; 1421 } 1422 1423 pthread_mutex_lock(&sc->mtx); 1424 memcpy(&sc->health_log.data_units_read, &sc->read_data_units, 1425 sizeof(sc->health_log.data_units_read)); 1426 memcpy(&sc->health_log.data_units_written, &sc->write_data_units, 1427 sizeof(sc->health_log.data_units_written)); 1428 memcpy(&sc->health_log.host_read_commands, &sc->read_commands, 1429 sizeof(sc->health_log.host_read_commands)); 1430 memcpy(&sc->health_log.host_write_commands, &sc->write_commands, 1431 sizeof(sc->health_log.host_write_commands)); 1432 pthread_mutex_unlock(&sc->mtx); 1433 1434 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1435 command->prp2, (uint8_t *)&sc->health_log + logoff, 1436 MIN(logsize - logoff, sizeof(sc->health_log)), 1437 NVME_COPY_TO_PRP); 1438 break; 1439 case NVME_LOG_FIRMWARE_SLOT: 1440 if (logoff >= sizeof(sc->fw_log)) { 1441 pci_nvme_status_genc(&compl->status, 1442 NVME_SC_INVALID_FIELD); 1443 break; 1444 } 1445 1446 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1447 command->prp2, (uint8_t *)&sc->fw_log + logoff, 1448 MIN(logsize - logoff, sizeof(sc->fw_log)), 1449 NVME_COPY_TO_PRP); 1450 break; 1451 case NVME_LOG_CHANGED_NAMESPACE: 1452 if (logoff >= sizeof(sc->ns_log)) { 1453 pci_nvme_status_genc(&compl->status, 1454 NVME_SC_INVALID_FIELD); 1455 break; 1456 } 1457 1458 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1459 command->prp2, (uint8_t *)&sc->ns_log + logoff, 1460 MIN(logsize - logoff, sizeof(sc->ns_log)), 1461 NVME_COPY_TO_PRP); 1462 memset(&sc->ns_log, 0, sizeof(sc->ns_log)); 1463 break; 1464 default: 1465 DPRINTF("%s get log page %x command not supported", 1466 __func__, logpage); 1467 1468 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1469 NVME_SC_INVALID_LOG_PAGE); 1470 } 1471 1472 return (1); 1473 } 1474 1475 static int 1476 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command, 1477 struct nvme_completion* compl) 1478 { 1479 void *dest; 1480 uint16_t status; 1481 1482 DPRINTF("%s identify 0x%x nsid 0x%x", __func__, 1483 command->cdw10 & 0xFF, command->nsid); 1484 1485 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 1486 1487 switch (command->cdw10 & 0xFF) { 1488 case 0x00: /* return Identify Namespace data structure */ 1489 /* Global NS only valid with NS Management */ 1490 if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) { 1491 pci_nvme_status_genc(&status, 1492 NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 1493 break; 1494 } 1495 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1496 command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata), 1497 NVME_COPY_TO_PRP); 1498 break; 1499 case 0x01: /* return Identify Controller data structure */ 1500 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, 1501 command->prp2, (uint8_t *)&sc->ctrldata, 1502 sizeof(sc->ctrldata), 1503 NVME_COPY_TO_PRP); 1504 break; 1505 case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */ 1506 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1507 sizeof(uint32_t) * 1024); 1508 /* All unused entries shall be zero */ 1509 bzero(dest, sizeof(uint32_t) * 1024); 1510 ((uint32_t *)dest)[0] = 1; 1511 break; 1512 case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */ 1513 if (command->nsid != 1) { 1514 pci_nvme_status_genc(&status, 1515 NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 1516 break; 1517 } 1518 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, 1519 sizeof(uint32_t) * 1024); 1520 /* All bytes after the descriptor shall be zero */ 1521 bzero(dest, sizeof(uint32_t) * 1024); 1522 1523 /* Return NIDT=1 (i.e. EUI64) descriptor */ 1524 ((uint8_t *)dest)[0] = 1; 1525 ((uint8_t *)dest)[1] = sizeof(uint64_t); 1526 bcopy(sc->nsdata.eui64, ((uint8_t *)dest) + 4, sizeof(uint64_t)); 1527 break; 1528 default: 1529 DPRINTF("%s unsupported identify command requested 0x%x", 1530 __func__, command->cdw10 & 0xFF); 1531 pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD); 1532 break; 1533 } 1534 1535 compl->status = status; 1536 return (1); 1537 } 1538 1539 static const char * 1540 nvme_fid_to_name(uint8_t fid) 1541 { 1542 const char *name; 1543 1544 switch (fid) { 1545 case NVME_FEAT_ARBITRATION: 1546 name = "Arbitration"; 1547 break; 1548 case NVME_FEAT_POWER_MANAGEMENT: 1549 name = "Power Management"; 1550 break; 1551 case NVME_FEAT_LBA_RANGE_TYPE: 1552 name = "LBA Range Type"; 1553 break; 1554 case NVME_FEAT_TEMPERATURE_THRESHOLD: 1555 name = "Temperature Threshold"; 1556 break; 1557 case NVME_FEAT_ERROR_RECOVERY: 1558 name = "Error Recovery"; 1559 break; 1560 case NVME_FEAT_VOLATILE_WRITE_CACHE: 1561 name = "Volatile Write Cache"; 1562 break; 1563 case NVME_FEAT_NUMBER_OF_QUEUES: 1564 name = "Number of Queues"; 1565 break; 1566 case NVME_FEAT_INTERRUPT_COALESCING: 1567 name = "Interrupt Coalescing"; 1568 break; 1569 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: 1570 name = "Interrupt Vector Configuration"; 1571 break; 1572 case NVME_FEAT_WRITE_ATOMICITY: 1573 name = "Write Atomicity Normal"; 1574 break; 1575 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: 1576 name = "Asynchronous Event Configuration"; 1577 break; 1578 case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION: 1579 name = "Autonomous Power State Transition"; 1580 break; 1581 case NVME_FEAT_HOST_MEMORY_BUFFER: 1582 name = "Host Memory Buffer"; 1583 break; 1584 case NVME_FEAT_TIMESTAMP: 1585 name = "Timestamp"; 1586 break; 1587 case NVME_FEAT_KEEP_ALIVE_TIMER: 1588 name = "Keep Alive Timer"; 1589 break; 1590 case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT: 1591 name = "Host Controlled Thermal Management"; 1592 break; 1593 case NVME_FEAT_NON_OP_POWER_STATE_CONFIG: 1594 name = "Non-Operation Power State Config"; 1595 break; 1596 case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG: 1597 name = "Read Recovery Level Config"; 1598 break; 1599 case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: 1600 name = "Predictable Latency Mode Config"; 1601 break; 1602 case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW: 1603 name = "Predictable Latency Mode Window"; 1604 break; 1605 case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES: 1606 name = "LBA Status Information Report Interval"; 1607 break; 1608 case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: 1609 name = "Host Behavior Support"; 1610 break; 1611 case NVME_FEAT_SANITIZE_CONFIG: 1612 name = "Sanitize Config"; 1613 break; 1614 case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION: 1615 name = "Endurance Group Event Configuration"; 1616 break; 1617 case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: 1618 name = "Software Progress Marker"; 1619 break; 1620 case NVME_FEAT_HOST_IDENTIFIER: 1621 name = "Host Identifier"; 1622 break; 1623 case NVME_FEAT_RESERVATION_NOTIFICATION_MASK: 1624 name = "Reservation Notification Mask"; 1625 break; 1626 case NVME_FEAT_RESERVATION_PERSISTENCE: 1627 name = "Reservation Persistence"; 1628 break; 1629 case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG: 1630 name = "Namespace Write Protection Config"; 1631 break; 1632 default: 1633 name = "Unknown"; 1634 break; 1635 } 1636 1637 return (name); 1638 } 1639 1640 static void 1641 nvme_feature_invalid_cb(struct pci_nvme_softc *sc, 1642 struct nvme_feature_obj *feat, 1643 struct nvme_command *command, 1644 struct nvme_completion *compl) 1645 { 1646 1647 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1648 } 1649 1650 static void 1651 nvme_feature_iv_config(struct pci_nvme_softc *sc, 1652 struct nvme_feature_obj *feat, 1653 struct nvme_command *command, 1654 struct nvme_completion *compl) 1655 { 1656 uint32_t i; 1657 uint32_t cdw11 = command->cdw11; 1658 uint16_t iv; 1659 bool cd; 1660 1661 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1662 1663 iv = cdw11 & 0xffff; 1664 cd = cdw11 & (1 << 16); 1665 1666 if (iv > (sc->max_queues + 1)) { 1667 return; 1668 } 1669 1670 /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */ 1671 if ((iv == 0) && !cd) 1672 return; 1673 1674 /* Requested Interrupt Vector must be used by a CQ */ 1675 for (i = 0; i < sc->num_cqueues + 1; i++) { 1676 if (sc->compl_queues[i].intr_vec == iv) { 1677 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1678 } 1679 } 1680 } 1681 1682 #define NVME_ASYNC_EVENT_ENDURANCE_GROUP (0x4000) 1683 static void 1684 nvme_feature_async_event(struct pci_nvme_softc *sc, 1685 struct nvme_feature_obj *feat, 1686 struct nvme_command *command, 1687 struct nvme_completion *compl) 1688 { 1689 1690 if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP) 1691 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1692 } 1693 1694 #define NVME_TEMP_THRESH_OVER 0 1695 #define NVME_TEMP_THRESH_UNDER 1 1696 static void 1697 nvme_feature_temperature(struct pci_nvme_softc *sc, 1698 struct nvme_feature_obj *feat, 1699 struct nvme_command *command, 1700 struct nvme_completion *compl) 1701 { 1702 uint16_t tmpth; /* Temperature Threshold */ 1703 uint8_t tmpsel; /* Threshold Temperature Select */ 1704 uint8_t thsel; /* Threshold Type Select */ 1705 bool set_crit = false; 1706 1707 tmpth = command->cdw11 & 0xffff; 1708 tmpsel = (command->cdw11 >> 16) & 0xf; 1709 thsel = (command->cdw11 >> 20) & 0x3; 1710 1711 DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel); 1712 1713 /* Check for unsupported values */ 1714 if (((tmpsel != 0) && (tmpsel != 0xf)) || 1715 (thsel > NVME_TEMP_THRESH_UNDER)) { 1716 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1717 return; 1718 } 1719 1720 if (((thsel == NVME_TEMP_THRESH_OVER) && (NVME_TEMPERATURE >= tmpth)) || 1721 ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth))) 1722 set_crit = true; 1723 1724 pthread_mutex_lock(&sc->mtx); 1725 if (set_crit) 1726 sc->health_log.critical_warning |= 1727 NVME_CRIT_WARN_ST_TEMPERATURE; 1728 else 1729 sc->health_log.critical_warning &= 1730 ~NVME_CRIT_WARN_ST_TEMPERATURE; 1731 pthread_mutex_unlock(&sc->mtx); 1732 1733 if (set_crit) 1734 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART, 1735 sc->health_log.critical_warning); 1736 1737 1738 DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status); 1739 } 1740 1741 static void 1742 nvme_feature_num_queues(struct pci_nvme_softc *sc, 1743 struct nvme_feature_obj *feat, 1744 struct nvme_command *command, 1745 struct nvme_completion *compl) 1746 { 1747 uint16_t nqr; /* Number of Queues Requested */ 1748 1749 if (sc->num_q_is_set) { 1750 WPRINTF("%s: Number of Queues already set", __func__); 1751 pci_nvme_status_genc(&compl->status, 1752 NVME_SC_COMMAND_SEQUENCE_ERROR); 1753 return; 1754 } 1755 1756 nqr = command->cdw11 & 0xFFFF; 1757 if (nqr == 0xffff) { 1758 WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr); 1759 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1760 return; 1761 } 1762 1763 sc->num_squeues = ONE_BASED(nqr); 1764 if (sc->num_squeues > sc->max_queues) { 1765 DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues, 1766 sc->max_queues); 1767 sc->num_squeues = sc->max_queues; 1768 } 1769 1770 nqr = (command->cdw11 >> 16) & 0xFFFF; 1771 if (nqr == 0xffff) { 1772 WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr); 1773 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1774 return; 1775 } 1776 1777 sc->num_cqueues = ONE_BASED(nqr); 1778 if (sc->num_cqueues > sc->max_queues) { 1779 DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues, 1780 sc->max_queues); 1781 sc->num_cqueues = sc->max_queues; 1782 } 1783 1784 /* Patch the command value which will be saved on callback's return */ 1785 command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc); 1786 compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); 1787 1788 sc->num_q_is_set = true; 1789 } 1790 1791 static int 1792 nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command, 1793 struct nvme_completion *compl) 1794 { 1795 struct nvme_feature_obj *feat; 1796 uint32_t nsid = command->nsid; 1797 uint8_t fid = command->cdw10 & 0xFF; 1798 1799 DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); 1800 1801 if (fid >= NVME_FID_MAX) { 1802 DPRINTF("%s invalid feature 0x%x", __func__, fid); 1803 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1804 return (1); 1805 } 1806 feat = &sc->feat[fid]; 1807 1808 if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) { 1809 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1810 return (1); 1811 } 1812 1813 if (!feat->namespace_specific && 1814 !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) { 1815 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1816 NVME_SC_FEATURE_NOT_NS_SPECIFIC); 1817 return (1); 1818 } 1819 1820 compl->cdw0 = 0; 1821 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1822 1823 if (feat->set) 1824 feat->set(sc, feat, command, compl); 1825 1826 DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11); 1827 if (compl->status == NVME_SC_SUCCESS) { 1828 feat->cdw11 = command->cdw11; 1829 if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) && 1830 (command->cdw11 != 0)) 1831 pci_nvme_aen_notify(sc); 1832 } 1833 1834 return (0); 1835 } 1836 1837 #define NVME_FEATURES_SEL_SUPPORTED 0x3 1838 #define NVME_FEATURES_NS_SPECIFIC (1 << 1) 1839 1840 static int 1841 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command, 1842 struct nvme_completion* compl) 1843 { 1844 struct nvme_feature_obj *feat; 1845 uint8_t fid = command->cdw10 & 0xFF; 1846 uint8_t sel = (command->cdw10 >> 8) & 0x7; 1847 1848 DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); 1849 1850 if (fid >= NVME_FID_MAX) { 1851 DPRINTF("%s invalid feature 0x%x", __func__, fid); 1852 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1853 return (1); 1854 } 1855 1856 compl->cdw0 = 0; 1857 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1858 1859 feat = &sc->feat[fid]; 1860 if (feat->get) { 1861 feat->get(sc, feat, command, compl); 1862 } 1863 1864 if (compl->status == NVME_SC_SUCCESS) { 1865 if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific) 1866 compl->cdw0 = NVME_FEATURES_NS_SPECIFIC; 1867 else 1868 compl->cdw0 = feat->cdw11; 1869 } 1870 1871 return (0); 1872 } 1873 1874 static int 1875 nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command, 1876 struct nvme_completion* compl) 1877 { 1878 uint8_t ses, lbaf, pi; 1879 1880 /* Only supports Secure Erase Setting - User Data Erase */ 1881 ses = (command->cdw10 >> 9) & 0x7; 1882 if (ses > 0x1) { 1883 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1884 return (1); 1885 } 1886 1887 /* Only supports a single LBA Format */ 1888 lbaf = command->cdw10 & 0xf; 1889 if (lbaf != 0) { 1890 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1891 NVME_SC_INVALID_FORMAT); 1892 return (1); 1893 } 1894 1895 /* Doesn't support Protection Infomation */ 1896 pi = (command->cdw10 >> 5) & 0x7; 1897 if (pi != 0) { 1898 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); 1899 return (1); 1900 } 1901 1902 if (sc->nvstore.type == NVME_STOR_RAM) { 1903 if (sc->nvstore.ctx) 1904 free(sc->nvstore.ctx); 1905 sc->nvstore.ctx = calloc(1, sc->nvstore.size); 1906 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1907 } else { 1908 struct pci_nvme_ioreq *req; 1909 int err; 1910 1911 req = pci_nvme_get_ioreq(sc); 1912 if (req == NULL) { 1913 pci_nvme_status_genc(&compl->status, 1914 NVME_SC_INTERNAL_DEVICE_ERROR); 1915 WPRINTF("%s: unable to allocate IO req", __func__); 1916 return (1); 1917 } 1918 req->nvme_sq = &sc->submit_queues[0]; 1919 req->sqid = 0; 1920 req->opc = command->opc; 1921 req->cid = command->cid; 1922 req->nsid = command->nsid; 1923 1924 req->io_req.br_offset = 0; 1925 req->io_req.br_resid = sc->nvstore.size; 1926 req->io_req.br_callback = pci_nvme_io_done; 1927 1928 err = blockif_delete(sc->nvstore.ctx, &req->io_req); 1929 if (err) { 1930 pci_nvme_status_genc(&compl->status, 1931 NVME_SC_INTERNAL_DEVICE_ERROR); 1932 pci_nvme_release_ioreq(sc, req); 1933 } else 1934 compl->status = NVME_NO_STATUS; 1935 } 1936 1937 return (1); 1938 } 1939 1940 static int 1941 nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command, 1942 struct nvme_completion* compl) 1943 { 1944 DPRINTF("%s submission queue %u, command ID 0x%x", __func__, 1945 command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF); 1946 1947 /* TODO: search for the command ID and abort it */ 1948 1949 compl->cdw0 = 1; 1950 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); 1951 return (1); 1952 } 1953 1954 static int 1955 nvme_opc_async_event_req(struct pci_nvme_softc* sc, 1956 struct nvme_command* command, struct nvme_completion* compl) 1957 { 1958 DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__, 1959 sc->aer_count, sc->ctrldata.aerl, command->cid); 1960 1961 /* Don't exceed the Async Event Request Limit (AERL). */ 1962 if (pci_nvme_aer_limit_reached(sc)) { 1963 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, 1964 NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); 1965 return (1); 1966 } 1967 1968 if (pci_nvme_aer_add(sc, command->cid)) { 1969 pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC, 1970 NVME_SC_INTERNAL_DEVICE_ERROR); 1971 return (1); 1972 } 1973 1974 /* 1975 * Raise events when they happen based on the Set Features cmd. 1976 * These events happen async, so only set completion successful if 1977 * there is an event reflective of the request to get event. 1978 */ 1979 compl->status = NVME_NO_STATUS; 1980 pci_nvme_aen_notify(sc); 1981 1982 return (0); 1983 } 1984 1985 static void 1986 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) 1987 { 1988 struct nvme_completion compl; 1989 struct nvme_command *cmd; 1990 struct nvme_submission_queue *sq; 1991 struct nvme_completion_queue *cq; 1992 uint16_t sqhead; 1993 1994 DPRINTF("%s index %u", __func__, (uint32_t)value); 1995 1996 sq = &sc->submit_queues[0]; 1997 cq = &sc->compl_queues[0]; 1998 1999 pthread_mutex_lock(&sq->mtx); 2000 2001 sqhead = sq->head; 2002 DPRINTF("sqhead %u, tail %u", sqhead, sq->tail); 2003 2004 while (sqhead != atomic_load_acq_short(&sq->tail)) { 2005 cmd = &(sq->qbase)[sqhead]; 2006 compl.cdw0 = 0; 2007 compl.status = 0; 2008 2009 switch (cmd->opc) { 2010 case NVME_OPC_DELETE_IO_SQ: 2011 DPRINTF("%s command DELETE_IO_SQ", __func__); 2012 nvme_opc_delete_io_sq(sc, cmd, &compl); 2013 break; 2014 case NVME_OPC_CREATE_IO_SQ: 2015 DPRINTF("%s command CREATE_IO_SQ", __func__); 2016 nvme_opc_create_io_sq(sc, cmd, &compl); 2017 break; 2018 case NVME_OPC_DELETE_IO_CQ: 2019 DPRINTF("%s command DELETE_IO_CQ", __func__); 2020 nvme_opc_delete_io_cq(sc, cmd, &compl); 2021 break; 2022 case NVME_OPC_CREATE_IO_CQ: 2023 DPRINTF("%s command CREATE_IO_CQ", __func__); 2024 nvme_opc_create_io_cq(sc, cmd, &compl); 2025 break; 2026 case NVME_OPC_GET_LOG_PAGE: 2027 DPRINTF("%s command GET_LOG_PAGE", __func__); 2028 nvme_opc_get_log_page(sc, cmd, &compl); 2029 break; 2030 case NVME_OPC_IDENTIFY: 2031 DPRINTF("%s command IDENTIFY", __func__); 2032 nvme_opc_identify(sc, cmd, &compl); 2033 break; 2034 case NVME_OPC_ABORT: 2035 DPRINTF("%s command ABORT", __func__); 2036 nvme_opc_abort(sc, cmd, &compl); 2037 break; 2038 case NVME_OPC_SET_FEATURES: 2039 DPRINTF("%s command SET_FEATURES", __func__); 2040 nvme_opc_set_features(sc, cmd, &compl); 2041 break; 2042 case NVME_OPC_GET_FEATURES: 2043 DPRINTF("%s command GET_FEATURES", __func__); 2044 nvme_opc_get_features(sc, cmd, &compl); 2045 break; 2046 case NVME_OPC_FIRMWARE_ACTIVATE: 2047 DPRINTF("%s command FIRMWARE_ACTIVATE", __func__); 2048 pci_nvme_status_tc(&compl.status, 2049 NVME_SCT_COMMAND_SPECIFIC, 2050 NVME_SC_INVALID_FIRMWARE_SLOT); 2051 break; 2052 case NVME_OPC_ASYNC_EVENT_REQUEST: 2053 DPRINTF("%s command ASYNC_EVENT_REQ", __func__); 2054 nvme_opc_async_event_req(sc, cmd, &compl); 2055 break; 2056 case NVME_OPC_FORMAT_NVM: 2057 DPRINTF("%s command FORMAT_NVM", __func__); 2058 if ((sc->ctrldata.oacs & 2059 (1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) { 2060 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); 2061 break; 2062 } 2063 nvme_opc_format_nvm(sc, cmd, &compl); 2064 break; 2065 case NVME_OPC_SECURITY_SEND: 2066 case NVME_OPC_SECURITY_RECEIVE: 2067 case NVME_OPC_SANITIZE: 2068 case NVME_OPC_GET_LBA_STATUS: 2069 DPRINTF("%s command OPC=%#x (unsupported)", __func__, 2070 cmd->opc); 2071 /* Valid but unsupported opcodes */ 2072 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD); 2073 break; 2074 default: 2075 DPRINTF("%s command OPC=%#X (not implemented)", 2076 __func__, 2077 cmd->opc); 2078 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); 2079 } 2080 sqhead = (sqhead + 1) % sq->size; 2081 2082 if (NVME_COMPLETION_VALID(compl)) { 2083 pci_nvme_cq_update(sc, &sc->compl_queues[0], 2084 compl.cdw0, 2085 cmd->cid, 2086 0, /* SQID */ 2087 compl.status); 2088 } 2089 } 2090 2091 DPRINTF("setting sqhead %u", sqhead); 2092 sq->head = sqhead; 2093 2094 if (cq->head != cq->tail) 2095 pci_generate_msix(sc->nsc_pi, 0); 2096 2097 pthread_mutex_unlock(&sq->mtx); 2098 } 2099 2100 /* 2101 * Update the Write and Read statistics reported in SMART data 2102 * 2103 * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up. 2104 * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000 2105 * 512 byte blocks. Rounding up is acheived by initializing the remainder to 999. 2106 */ 2107 static void 2108 pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc, 2109 size_t bytes, uint16_t status) 2110 { 2111 2112 pthread_mutex_lock(&sc->mtx); 2113 switch (opc) { 2114 case NVME_OPC_WRITE: 2115 sc->write_commands++; 2116 if (status != NVME_SC_SUCCESS) 2117 break; 2118 sc->write_dunits_remainder += (bytes / 512); 2119 while (sc->write_dunits_remainder >= 1000) { 2120 sc->write_data_units++; 2121 sc->write_dunits_remainder -= 1000; 2122 } 2123 break; 2124 case NVME_OPC_READ: 2125 sc->read_commands++; 2126 if (status != NVME_SC_SUCCESS) 2127 break; 2128 sc->read_dunits_remainder += (bytes / 512); 2129 while (sc->read_dunits_remainder >= 1000) { 2130 sc->read_data_units++; 2131 sc->read_dunits_remainder -= 1000; 2132 } 2133 break; 2134 default: 2135 DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc); 2136 break; 2137 } 2138 pthread_mutex_unlock(&sc->mtx); 2139 } 2140 2141 /* 2142 * Check if the combination of Starting LBA (slba) and number of blocks 2143 * exceeds the range of the underlying storage. 2144 * 2145 * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores 2146 * the capacity in bytes as a uint64_t, care must be taken to avoid integer 2147 * overflow. 2148 */ 2149 static bool 2150 pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba, 2151 uint32_t nblocks) 2152 { 2153 size_t offset, bytes; 2154 2155 /* Overflow check of multiplying Starting LBA by the sector size */ 2156 if (slba >> (64 - nvstore->sectsz_bits)) 2157 return (true); 2158 2159 offset = slba << nvstore->sectsz_bits; 2160 bytes = nblocks << nvstore->sectsz_bits; 2161 2162 /* Overflow check of Number of Logical Blocks */ 2163 if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes)) 2164 return (true); 2165 2166 return (false); 2167 } 2168 2169 static int 2170 pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req, 2171 uint64_t gpaddr, size_t size, int do_write, uint64_t lba) 2172 { 2173 int iovidx; 2174 2175 if (req == NULL) 2176 return (-1); 2177 2178 if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) { 2179 return (-1); 2180 } 2181 2182 /* concatenate contig block-iovs to minimize number of iovs */ 2183 if ((req->prev_gpaddr + req->prev_size) == gpaddr) { 2184 iovidx = req->io_req.br_iovcnt - 1; 2185 2186 req->io_req.br_iov[iovidx].iov_base = 2187 paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 2188 req->prev_gpaddr, size); 2189 2190 req->prev_size += size; 2191 req->io_req.br_resid += size; 2192 2193 req->io_req.br_iov[iovidx].iov_len = req->prev_size; 2194 } else { 2195 iovidx = req->io_req.br_iovcnt; 2196 if (iovidx == 0) { 2197 req->io_req.br_offset = lba; 2198 req->io_req.br_resid = 0; 2199 req->io_req.br_param = req; 2200 } 2201 2202 req->io_req.br_iov[iovidx].iov_base = 2203 paddr_guest2host(req->sc->nsc_pi->pi_vmctx, 2204 gpaddr, size); 2205 2206 req->io_req.br_iov[iovidx].iov_len = size; 2207 2208 req->prev_gpaddr = gpaddr; 2209 req->prev_size = size; 2210 req->io_req.br_resid += size; 2211 2212 req->io_req.br_iovcnt++; 2213 } 2214 2215 return (0); 2216 } 2217 2218 static void 2219 pci_nvme_set_completion(struct pci_nvme_softc *sc, 2220 struct nvme_submission_queue *sq, int sqid, uint16_t cid, 2221 uint32_t cdw0, uint16_t status) 2222 { 2223 struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid]; 2224 2225 DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x", 2226 __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status), 2227 NVME_STATUS_GET_SC(status)); 2228 2229 pci_nvme_cq_update(sc, cq, 2230 0, /* CDW0 */ 2231 cid, 2232 sqid, 2233 status); 2234 2235 if (cq->head != cq->tail) { 2236 if (cq->intr_en & NVME_CQ_INTEN) { 2237 pci_generate_msix(sc->nsc_pi, cq->intr_vec); 2238 } else { 2239 DPRINTF("%s: CQ%u interrupt disabled", 2240 __func__, sq->cqid); 2241 } 2242 } 2243 } 2244 2245 static void 2246 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req) 2247 { 2248 req->sc = NULL; 2249 req->nvme_sq = NULL; 2250 req->sqid = 0; 2251 2252 pthread_mutex_lock(&sc->mtx); 2253 2254 STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link); 2255 sc->pending_ios--; 2256 2257 /* when no more IO pending, can set to ready if device reset/enabled */ 2258 if (sc->pending_ios == 0 && 2259 NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts))) 2260 sc->regs.csts |= NVME_CSTS_RDY; 2261 2262 pthread_mutex_unlock(&sc->mtx); 2263 2264 sem_post(&sc->iosemlock); 2265 } 2266 2267 static struct pci_nvme_ioreq * 2268 pci_nvme_get_ioreq(struct pci_nvme_softc *sc) 2269 { 2270 struct pci_nvme_ioreq *req = NULL; 2271 2272 sem_wait(&sc->iosemlock); 2273 pthread_mutex_lock(&sc->mtx); 2274 2275 req = STAILQ_FIRST(&sc->ioreqs_free); 2276 assert(req != NULL); 2277 STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link); 2278 2279 req->sc = sc; 2280 2281 sc->pending_ios++; 2282 2283 pthread_mutex_unlock(&sc->mtx); 2284 2285 req->io_req.br_iovcnt = 0; 2286 req->io_req.br_offset = 0; 2287 req->io_req.br_resid = 0; 2288 req->io_req.br_param = req; 2289 req->prev_gpaddr = 0; 2290 req->prev_size = 0; 2291 2292 return req; 2293 } 2294 2295 static void 2296 pci_nvme_io_done(struct blockif_req *br, int err) 2297 { 2298 struct pci_nvme_ioreq *req = br->br_param; 2299 struct nvme_submission_queue *sq = req->nvme_sq; 2300 uint16_t code, status; 2301 2302 DPRINTF("%s error %d %s", __func__, err, strerror(err)); 2303 2304 /* TODO return correct error */ 2305 code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS; 2306 pci_nvme_status_genc(&status, code); 2307 2308 pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status); 2309 pci_nvme_stats_write_read_update(req->sc, req->opc, 2310 req->bytes, status); 2311 pci_nvme_release_ioreq(req->sc, req); 2312 } 2313 2314 /* 2315 * Implements the Flush command. The specification states: 2316 * If a volatile write cache is not present, Flush commands complete 2317 * successfully and have no effect 2318 * in the description of the Volatile Write Cache (VWC) field of the Identify 2319 * Controller data. Therefore, set status to Success if the command is 2320 * not supported (i.e. RAM or as indicated by the blockif). 2321 */ 2322 static bool 2323 nvme_opc_flush(struct pci_nvme_softc *sc, 2324 struct nvme_command *cmd, 2325 struct pci_nvme_blockstore *nvstore, 2326 struct pci_nvme_ioreq *req, 2327 uint16_t *status) 2328 { 2329 bool pending = false; 2330 2331 if (nvstore->type == NVME_STOR_RAM) { 2332 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2333 } else { 2334 int err; 2335 2336 req->io_req.br_callback = pci_nvme_io_done; 2337 2338 err = blockif_flush(nvstore->ctx, &req->io_req); 2339 switch (err) { 2340 case 0: 2341 pending = true; 2342 break; 2343 case EOPNOTSUPP: 2344 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2345 break; 2346 default: 2347 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2348 } 2349 } 2350 2351 return (pending); 2352 } 2353 2354 static uint16_t 2355 nvme_write_read_ram(struct pci_nvme_softc *sc, 2356 struct pci_nvme_blockstore *nvstore, 2357 uint64_t prp1, uint64_t prp2, 2358 size_t offset, uint64_t bytes, 2359 bool is_write) 2360 { 2361 uint8_t *buf = nvstore->ctx; 2362 enum nvme_copy_dir dir; 2363 uint16_t status; 2364 2365 if (is_write) 2366 dir = NVME_COPY_TO_PRP; 2367 else 2368 dir = NVME_COPY_FROM_PRP; 2369 2370 if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2, 2371 buf + offset, bytes, dir)) 2372 pci_nvme_status_genc(&status, 2373 NVME_SC_DATA_TRANSFER_ERROR); 2374 else 2375 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 2376 2377 return (status); 2378 } 2379 2380 static uint16_t 2381 nvme_write_read_blockif(struct pci_nvme_softc *sc, 2382 struct pci_nvme_blockstore *nvstore, 2383 struct pci_nvme_ioreq *req, 2384 uint64_t prp1, uint64_t prp2, 2385 size_t offset, uint64_t bytes, 2386 bool is_write) 2387 { 2388 uint64_t size; 2389 int err; 2390 uint16_t status = NVME_NO_STATUS; 2391 2392 size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes); 2393 if (pci_nvme_append_iov_req(sc, req, prp1, 2394 size, is_write, offset)) { 2395 pci_nvme_status_genc(&status, 2396 NVME_SC_DATA_TRANSFER_ERROR); 2397 goto out; 2398 } 2399 2400 offset += size; 2401 bytes -= size; 2402 2403 if (bytes == 0) { 2404 ; 2405 } else if (bytes <= PAGE_SIZE) { 2406 size = bytes; 2407 if (pci_nvme_append_iov_req(sc, req, prp2, 2408 size, is_write, offset)) { 2409 pci_nvme_status_genc(&status, 2410 NVME_SC_DATA_TRANSFER_ERROR); 2411 goto out; 2412 } 2413 } else { 2414 void *vmctx = sc->nsc_pi->pi_vmctx; 2415 uint64_t *prp_list = &prp2; 2416 uint64_t *last = prp_list; 2417 2418 /* PRP2 is pointer to a physical region page list */ 2419 while (bytes) { 2420 /* Last entry in list points to the next list */ 2421 if ((prp_list == last) && (bytes > PAGE_SIZE)) { 2422 uint64_t prp = *prp_list; 2423 2424 prp_list = paddr_guest2host(vmctx, prp, 2425 PAGE_SIZE - (prp % PAGE_SIZE)); 2426 last = prp_list + (NVME_PRP2_ITEMS - 1); 2427 } 2428 2429 size = MIN(bytes, PAGE_SIZE); 2430 2431 if (pci_nvme_append_iov_req(sc, req, *prp_list, 2432 size, is_write, offset)) { 2433 pci_nvme_status_genc(&status, 2434 NVME_SC_DATA_TRANSFER_ERROR); 2435 goto out; 2436 } 2437 2438 offset += size; 2439 bytes -= size; 2440 2441 prp_list++; 2442 } 2443 } 2444 req->io_req.br_callback = pci_nvme_io_done; 2445 if (is_write) 2446 err = blockif_write(nvstore->ctx, &req->io_req); 2447 else 2448 err = blockif_read(nvstore->ctx, &req->io_req); 2449 2450 if (err) 2451 pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR); 2452 out: 2453 return (status); 2454 } 2455 2456 static bool 2457 nvme_opc_write_read(struct pci_nvme_softc *sc, 2458 struct nvme_command *cmd, 2459 struct pci_nvme_blockstore *nvstore, 2460 struct pci_nvme_ioreq *req, 2461 uint16_t *status) 2462 { 2463 uint64_t lba, nblocks, bytes; 2464 size_t offset; 2465 bool is_write = cmd->opc == NVME_OPC_WRITE; 2466 bool pending = false; 2467 2468 lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; 2469 nblocks = (cmd->cdw12 & 0xFFFF) + 1; 2470 2471 if (pci_nvme_out_of_range(nvstore, lba, nblocks)) { 2472 WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)", 2473 __func__, lba, nblocks); 2474 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); 2475 goto out; 2476 } 2477 2478 bytes = nblocks << nvstore->sectsz_bits; 2479 if (bytes > NVME_MAX_DATA_SIZE) { 2480 WPRINTF("%s command would exceed MDTS", __func__); 2481 pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD); 2482 goto out; 2483 } 2484 2485 offset = lba << nvstore->sectsz_bits; 2486 2487 req->bytes = bytes; 2488 req->io_req.br_offset = lba; 2489 2490 /* PRP bits 1:0 must be zero */ 2491 cmd->prp1 &= ~0x3UL; 2492 cmd->prp2 &= ~0x3UL; 2493 2494 if (nvstore->type == NVME_STOR_RAM) { 2495 *status = nvme_write_read_ram(sc, nvstore, cmd->prp1, 2496 cmd->prp2, offset, bytes, is_write); 2497 } else { 2498 *status = nvme_write_read_blockif(sc, nvstore, req, 2499 cmd->prp1, cmd->prp2, offset, bytes, is_write); 2500 2501 if (*status == NVME_NO_STATUS) 2502 pending = true; 2503 } 2504 out: 2505 if (!pending) 2506 pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status); 2507 2508 return (pending); 2509 } 2510 2511 static void 2512 pci_nvme_dealloc_sm(struct blockif_req *br, int err) 2513 { 2514 struct pci_nvme_ioreq *req = br->br_param; 2515 struct pci_nvme_softc *sc = req->sc; 2516 bool done = true; 2517 uint16_t status; 2518 2519 if (err) { 2520 pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); 2521 } else if ((req->prev_gpaddr + 1) == (req->prev_size)) { 2522 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 2523 } else { 2524 struct iovec *iov = req->io_req.br_iov; 2525 2526 req->prev_gpaddr++; 2527 iov += req->prev_gpaddr; 2528 2529 /* The iov_* values already include the sector size */ 2530 req->io_req.br_offset = (off_t)iov->iov_base; 2531 req->io_req.br_resid = iov->iov_len; 2532 if (blockif_delete(sc->nvstore.ctx, &req->io_req)) { 2533 pci_nvme_status_genc(&status, 2534 NVME_SC_INTERNAL_DEVICE_ERROR); 2535 } else 2536 done = false; 2537 } 2538 2539 if (done) { 2540 pci_nvme_set_completion(sc, req->nvme_sq, req->sqid, 2541 req->cid, 0, status); 2542 pci_nvme_release_ioreq(sc, req); 2543 } 2544 } 2545 2546 static bool 2547 nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc, 2548 struct nvme_command *cmd, 2549 struct pci_nvme_blockstore *nvstore, 2550 struct pci_nvme_ioreq *req, 2551 uint16_t *status) 2552 { 2553 struct nvme_dsm_range *range; 2554 uint32_t nr, r, non_zero, dr; 2555 int err; 2556 bool pending = false; 2557 2558 if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) { 2559 pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE); 2560 goto out; 2561 } 2562 2563 nr = cmd->cdw10 & 0xff; 2564 2565 /* copy locally because a range entry could straddle PRPs */ 2566 range = calloc(1, NVME_MAX_DSM_TRIM); 2567 if (range == NULL) { 2568 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2569 goto out; 2570 } 2571 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2, 2572 (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP); 2573 2574 /* Check for invalid ranges and the number of non-zero lengths */ 2575 non_zero = 0; 2576 for (r = 0; r <= nr; r++) { 2577 if (pci_nvme_out_of_range(nvstore, 2578 range[r].starting_lba, range[r].length)) { 2579 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); 2580 goto out; 2581 } 2582 if (range[r].length != 0) 2583 non_zero++; 2584 } 2585 2586 if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) { 2587 size_t offset, bytes; 2588 int sectsz_bits = sc->nvstore.sectsz_bits; 2589 2590 /* 2591 * DSM calls are advisory only, and compliant controllers 2592 * may choose to take no actions (i.e. return Success). 2593 */ 2594 if (!nvstore->deallocate) { 2595 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2596 goto out; 2597 } 2598 2599 /* If all ranges have a zero length, return Success */ 2600 if (non_zero == 0) { 2601 pci_nvme_status_genc(status, NVME_SC_SUCCESS); 2602 goto out; 2603 } 2604 2605 if (req == NULL) { 2606 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2607 goto out; 2608 } 2609 2610 offset = range[0].starting_lba << sectsz_bits; 2611 bytes = range[0].length << sectsz_bits; 2612 2613 /* 2614 * If the request is for more than a single range, store 2615 * the ranges in the br_iov. Optimize for the common case 2616 * of a single range. 2617 * 2618 * Note that NVMe Number of Ranges is a zero based value 2619 */ 2620 req->io_req.br_iovcnt = 0; 2621 req->io_req.br_offset = offset; 2622 req->io_req.br_resid = bytes; 2623 2624 if (nr == 0) { 2625 req->io_req.br_callback = pci_nvme_io_done; 2626 } else { 2627 struct iovec *iov = req->io_req.br_iov; 2628 2629 for (r = 0, dr = 0; r <= nr; r++) { 2630 offset = range[r].starting_lba << sectsz_bits; 2631 bytes = range[r].length << sectsz_bits; 2632 if (bytes == 0) 2633 continue; 2634 2635 if ((nvstore->size - offset) < bytes) { 2636 pci_nvme_status_genc(status, 2637 NVME_SC_LBA_OUT_OF_RANGE); 2638 goto out; 2639 } 2640 iov[dr].iov_base = (void *)offset; 2641 iov[dr].iov_len = bytes; 2642 dr++; 2643 } 2644 req->io_req.br_callback = pci_nvme_dealloc_sm; 2645 2646 /* 2647 * Use prev_gpaddr to track the current entry and 2648 * prev_size to track the number of entries 2649 */ 2650 req->prev_gpaddr = 0; 2651 req->prev_size = dr; 2652 } 2653 2654 err = blockif_delete(nvstore->ctx, &req->io_req); 2655 if (err) 2656 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); 2657 else 2658 pending = true; 2659 } 2660 out: 2661 free(range); 2662 return (pending); 2663 } 2664 2665 static void 2666 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx) 2667 { 2668 struct nvme_submission_queue *sq; 2669 uint16_t status; 2670 uint16_t sqhead; 2671 2672 /* handle all submissions up to sq->tail index */ 2673 sq = &sc->submit_queues[idx]; 2674 2675 pthread_mutex_lock(&sq->mtx); 2676 2677 sqhead = sq->head; 2678 DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p", 2679 idx, sqhead, sq->tail, sq->qbase); 2680 2681 while (sqhead != atomic_load_acq_short(&sq->tail)) { 2682 struct nvme_command *cmd; 2683 struct pci_nvme_ioreq *req; 2684 uint32_t nsid; 2685 bool pending; 2686 2687 pending = false; 2688 req = NULL; 2689 status = 0; 2690 2691 cmd = &sq->qbase[sqhead]; 2692 sqhead = (sqhead + 1) % sq->size; 2693 2694 nsid = le32toh(cmd->nsid); 2695 if ((nsid == 0) || (nsid > sc->ctrldata.nn)) { 2696 pci_nvme_status_genc(&status, 2697 NVME_SC_INVALID_NAMESPACE_OR_FORMAT); 2698 status |= 2699 NVME_STATUS_DNR_MASK << NVME_STATUS_DNR_SHIFT; 2700 goto complete; 2701 } 2702 2703 req = pci_nvme_get_ioreq(sc); 2704 if (req == NULL) { 2705 pci_nvme_status_genc(&status, 2706 NVME_SC_INTERNAL_DEVICE_ERROR); 2707 WPRINTF("%s: unable to allocate IO req", __func__); 2708 goto complete; 2709 } 2710 req->nvme_sq = sq; 2711 req->sqid = idx; 2712 req->opc = cmd->opc; 2713 req->cid = cmd->cid; 2714 req->nsid = cmd->nsid; 2715 2716 switch (cmd->opc) { 2717 case NVME_OPC_FLUSH: 2718 pending = nvme_opc_flush(sc, cmd, &sc->nvstore, 2719 req, &status); 2720 break; 2721 case NVME_OPC_WRITE: 2722 case NVME_OPC_READ: 2723 pending = nvme_opc_write_read(sc, cmd, &sc->nvstore, 2724 req, &status); 2725 break; 2726 case NVME_OPC_WRITE_ZEROES: 2727 /* TODO: write zeroes 2728 WPRINTF("%s write zeroes lba 0x%lx blocks %u", 2729 __func__, lba, cmd->cdw12 & 0xFFFF); */ 2730 pci_nvme_status_genc(&status, NVME_SC_SUCCESS); 2731 break; 2732 case NVME_OPC_DATASET_MANAGEMENT: 2733 pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore, 2734 req, &status); 2735 break; 2736 default: 2737 WPRINTF("%s unhandled io command 0x%x", 2738 __func__, cmd->opc); 2739 pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE); 2740 } 2741 complete: 2742 if (!pending) { 2743 pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, 2744 status); 2745 if (req != NULL) 2746 pci_nvme_release_ioreq(sc, req); 2747 } 2748 } 2749 2750 sq->head = sqhead; 2751 2752 pthread_mutex_unlock(&sq->mtx); 2753 } 2754 2755 static void 2756 pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc, 2757 uint64_t idx, int is_sq, uint64_t value) 2758 { 2759 DPRINTF("nvme doorbell %lu, %s, val 0x%lx", 2760 idx, is_sq ? "SQ" : "CQ", value & 0xFFFF); 2761 2762 if (is_sq) { 2763 if (idx > sc->num_squeues) { 2764 WPRINTF("%s queue index %lu overflow from " 2765 "guest (max %u)", 2766 __func__, idx, sc->num_squeues); 2767 return; 2768 } 2769 2770 atomic_store_short(&sc->submit_queues[idx].tail, 2771 (uint16_t)value); 2772 2773 if (idx == 0) { 2774 pci_nvme_handle_admin_cmd(sc, value); 2775 } else { 2776 /* submission queue; handle new entries in SQ */ 2777 if (idx > sc->num_squeues) { 2778 WPRINTF("%s SQ index %lu overflow from " 2779 "guest (max %u)", 2780 __func__, idx, sc->num_squeues); 2781 return; 2782 } 2783 pci_nvme_handle_io_cmd(sc, (uint16_t)idx); 2784 } 2785 } else { 2786 if (idx > sc->num_cqueues) { 2787 WPRINTF("%s queue index %lu overflow from " 2788 "guest (max %u)", 2789 __func__, idx, sc->num_cqueues); 2790 return; 2791 } 2792 2793 atomic_store_short(&sc->compl_queues[idx].head, 2794 (uint16_t)value); 2795 } 2796 } 2797 2798 static void 2799 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite) 2800 { 2801 const char *s = iswrite ? "WRITE" : "READ"; 2802 2803 switch (offset) { 2804 case NVME_CR_CAP_LOW: 2805 DPRINTF("%s %s NVME_CR_CAP_LOW", func, s); 2806 break; 2807 case NVME_CR_CAP_HI: 2808 DPRINTF("%s %s NVME_CR_CAP_HI", func, s); 2809 break; 2810 case NVME_CR_VS: 2811 DPRINTF("%s %s NVME_CR_VS", func, s); 2812 break; 2813 case NVME_CR_INTMS: 2814 DPRINTF("%s %s NVME_CR_INTMS", func, s); 2815 break; 2816 case NVME_CR_INTMC: 2817 DPRINTF("%s %s NVME_CR_INTMC", func, s); 2818 break; 2819 case NVME_CR_CC: 2820 DPRINTF("%s %s NVME_CR_CC", func, s); 2821 break; 2822 case NVME_CR_CSTS: 2823 DPRINTF("%s %s NVME_CR_CSTS", func, s); 2824 break; 2825 case NVME_CR_NSSR: 2826 DPRINTF("%s %s NVME_CR_NSSR", func, s); 2827 break; 2828 case NVME_CR_AQA: 2829 DPRINTF("%s %s NVME_CR_AQA", func, s); 2830 break; 2831 case NVME_CR_ASQ_LOW: 2832 DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s); 2833 break; 2834 case NVME_CR_ASQ_HI: 2835 DPRINTF("%s %s NVME_CR_ASQ_HI", func, s); 2836 break; 2837 case NVME_CR_ACQ_LOW: 2838 DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s); 2839 break; 2840 case NVME_CR_ACQ_HI: 2841 DPRINTF("%s %s NVME_CR_ACQ_HI", func, s); 2842 break; 2843 default: 2844 DPRINTF("unknown nvme bar-0 offset 0x%lx", offset); 2845 } 2846 2847 } 2848 2849 static void 2850 pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc, 2851 uint64_t offset, int size, uint64_t value) 2852 { 2853 uint32_t ccreg; 2854 2855 if (offset >= NVME_DOORBELL_OFFSET) { 2856 uint64_t belloffset = offset - NVME_DOORBELL_OFFSET; 2857 uint64_t idx = belloffset / 8; /* door bell size = 2*int */ 2858 int is_sq = (belloffset % 8) < 4; 2859 2860 if (belloffset > ((sc->max_queues+1) * 8 - 4)) { 2861 WPRINTF("guest attempted an overflow write offset " 2862 "0x%lx, val 0x%lx in %s", 2863 offset, value, __func__); 2864 return; 2865 } 2866 2867 pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value); 2868 return; 2869 } 2870 2871 DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx", 2872 offset, size, value); 2873 2874 if (size != 4) { 2875 WPRINTF("guest wrote invalid size %d (offset 0x%lx, " 2876 "val 0x%lx) to bar0 in %s", 2877 size, offset, value, __func__); 2878 /* TODO: shutdown device */ 2879 return; 2880 } 2881 2882 pci_nvme_bar0_reg_dumps(__func__, offset, 1); 2883 2884 pthread_mutex_lock(&sc->mtx); 2885 2886 switch (offset) { 2887 case NVME_CR_CAP_LOW: 2888 case NVME_CR_CAP_HI: 2889 /* readonly */ 2890 break; 2891 case NVME_CR_VS: 2892 /* readonly */ 2893 break; 2894 case NVME_CR_INTMS: 2895 /* MSI-X, so ignore */ 2896 break; 2897 case NVME_CR_INTMC: 2898 /* MSI-X, so ignore */ 2899 break; 2900 case NVME_CR_CC: 2901 ccreg = (uint32_t)value; 2902 2903 DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u " 2904 "iocqes %u", 2905 __func__, 2906 NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg), 2907 NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg), 2908 NVME_CC_GET_IOCQES(ccreg)); 2909 2910 if (NVME_CC_GET_SHN(ccreg)) { 2911 /* perform shutdown - flush out data to backend */ 2912 sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK << 2913 NVME_CSTS_REG_SHST_SHIFT); 2914 sc->regs.csts |= NVME_SHST_COMPLETE << 2915 NVME_CSTS_REG_SHST_SHIFT; 2916 } 2917 if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) { 2918 if (NVME_CC_GET_EN(ccreg) == 0) 2919 /* transition 1-> causes controller reset */ 2920 pci_nvme_reset_locked(sc); 2921 else 2922 pci_nvme_init_controller(ctx, sc); 2923 } 2924 2925 /* Insert the iocqes, iosqes and en bits from the write */ 2926 sc->regs.cc &= ~NVME_CC_WRITE_MASK; 2927 sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK; 2928 if (NVME_CC_GET_EN(ccreg) == 0) { 2929 /* Insert the ams, mps and css bit fields */ 2930 sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK; 2931 sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK; 2932 sc->regs.csts &= ~NVME_CSTS_RDY; 2933 } else if (sc->pending_ios == 0) { 2934 sc->regs.csts |= NVME_CSTS_RDY; 2935 } 2936 break; 2937 case NVME_CR_CSTS: 2938 break; 2939 case NVME_CR_NSSR: 2940 /* ignore writes; don't support subsystem reset */ 2941 break; 2942 case NVME_CR_AQA: 2943 sc->regs.aqa = (uint32_t)value; 2944 break; 2945 case NVME_CR_ASQ_LOW: 2946 sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) | 2947 (0xFFFFF000 & value); 2948 break; 2949 case NVME_CR_ASQ_HI: 2950 sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) | 2951 (value << 32); 2952 break; 2953 case NVME_CR_ACQ_LOW: 2954 sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) | 2955 (0xFFFFF000 & value); 2956 break; 2957 case NVME_CR_ACQ_HI: 2958 sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) | 2959 (value << 32); 2960 break; 2961 default: 2962 DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d", 2963 __func__, offset, value, size); 2964 } 2965 pthread_mutex_unlock(&sc->mtx); 2966 } 2967 2968 static void 2969 pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, 2970 int baridx, uint64_t offset, int size, uint64_t value) 2971 { 2972 struct pci_nvme_softc* sc = pi->pi_arg; 2973 2974 if (baridx == pci_msix_table_bar(pi) || 2975 baridx == pci_msix_pba_bar(pi)) { 2976 DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, " 2977 " value 0x%lx", baridx, offset, size, value); 2978 2979 pci_emul_msix_twrite(pi, offset, size, value); 2980 return; 2981 } 2982 2983 switch (baridx) { 2984 case 0: 2985 pci_nvme_write_bar_0(ctx, sc, offset, size, value); 2986 break; 2987 2988 default: 2989 DPRINTF("%s unknown baridx %d, val 0x%lx", 2990 __func__, baridx, value); 2991 } 2992 } 2993 2994 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc, 2995 uint64_t offset, int size) 2996 { 2997 uint64_t value; 2998 2999 pci_nvme_bar0_reg_dumps(__func__, offset, 0); 3000 3001 if (offset < NVME_DOORBELL_OFFSET) { 3002 void *p = &(sc->regs); 3003 pthread_mutex_lock(&sc->mtx); 3004 memcpy(&value, (void *)((uintptr_t)p + offset), size); 3005 pthread_mutex_unlock(&sc->mtx); 3006 } else { 3007 value = 0; 3008 WPRINTF("pci_nvme: read invalid offset %ld", offset); 3009 } 3010 3011 switch (size) { 3012 case 1: 3013 value &= 0xFF; 3014 break; 3015 case 2: 3016 value &= 0xFFFF; 3017 break; 3018 case 4: 3019 value &= 0xFFFFFFFF; 3020 break; 3021 } 3022 3023 DPRINTF(" nvme-read offset 0x%lx, size %d -> value 0x%x", 3024 offset, size, (uint32_t)value); 3025 3026 return (value); 3027 } 3028 3029 3030 3031 static uint64_t 3032 pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, 3033 uint64_t offset, int size) 3034 { 3035 struct pci_nvme_softc* sc = pi->pi_arg; 3036 3037 if (baridx == pci_msix_table_bar(pi) || 3038 baridx == pci_msix_pba_bar(pi)) { 3039 DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d", 3040 baridx, offset, size); 3041 3042 return pci_emul_msix_tread(pi, offset, size); 3043 } 3044 3045 switch (baridx) { 3046 case 0: 3047 return pci_nvme_read_bar_0(sc, offset, size); 3048 3049 default: 3050 DPRINTF("unknown bar %d, 0x%lx", baridx, offset); 3051 } 3052 3053 return (0); 3054 } 3055 3056 static int 3057 pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl) 3058 { 3059 char bident[sizeof("XX:X:X")]; 3060 const char *value; 3061 uint32_t sectsz; 3062 3063 sc->max_queues = NVME_QUEUES; 3064 sc->max_qentries = NVME_MAX_QENTRIES; 3065 sc->ioslots = NVME_IOSLOTS; 3066 sc->num_squeues = sc->max_queues; 3067 sc->num_cqueues = sc->max_queues; 3068 sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; 3069 sectsz = 0; 3070 snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn), 3071 "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 3072 3073 value = get_config_value_node(nvl, "maxq"); 3074 if (value != NULL) 3075 sc->max_queues = atoi(value); 3076 value = get_config_value_node(nvl, "qsz"); 3077 if (value != NULL) { 3078 sc->max_qentries = atoi(value); 3079 if (sc->max_qentries <= 0) { 3080 EPRINTLN("nvme: Invalid qsz option %d", 3081 sc->max_qentries); 3082 return (-1); 3083 } 3084 } 3085 value = get_config_value_node(nvl, "ioslots"); 3086 if (value != NULL) { 3087 sc->ioslots = atoi(value); 3088 if (sc->ioslots <= 0) { 3089 EPRINTLN("Invalid ioslots option %d", sc->ioslots); 3090 return (-1); 3091 } 3092 } 3093 value = get_config_value_node(nvl, "sectsz"); 3094 if (value != NULL) 3095 sectsz = atoi(value); 3096 value = get_config_value_node(nvl, "ser"); 3097 if (value != NULL) { 3098 /* 3099 * This field indicates the Product Serial Number in 3100 * 7-bit ASCII, unused bytes should be space characters. 3101 * Ref: NVMe v1.3c. 3102 */ 3103 cpywithpad((char *)sc->ctrldata.sn, 3104 sizeof(sc->ctrldata.sn), value, ' '); 3105 } 3106 value = get_config_value_node(nvl, "eui64"); 3107 if (value != NULL) 3108 sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0)); 3109 value = get_config_value_node(nvl, "dsm"); 3110 if (value != NULL) { 3111 if (strcmp(value, "auto") == 0) 3112 sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; 3113 else if (strcmp(value, "enable") == 0) 3114 sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE; 3115 else if (strcmp(value, "disable") == 0) 3116 sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE; 3117 } 3118 3119 value = get_config_value_node(nvl, "ram"); 3120 if (value != NULL) { 3121 uint64_t sz = strtoull(value, NULL, 10); 3122 3123 sc->nvstore.type = NVME_STOR_RAM; 3124 sc->nvstore.size = sz * 1024 * 1024; 3125 sc->nvstore.ctx = calloc(1, sc->nvstore.size); 3126 sc->nvstore.sectsz = 4096; 3127 sc->nvstore.sectsz_bits = 12; 3128 if (sc->nvstore.ctx == NULL) { 3129 EPRINTLN("nvme: Unable to allocate RAM"); 3130 return (-1); 3131 } 3132 } else { 3133 snprintf(bident, sizeof(bident), "%d:%d", 3134 sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); 3135 sc->nvstore.ctx = blockif_open(nvl, bident); 3136 if (sc->nvstore.ctx == NULL) { 3137 EPRINTLN("nvme: Could not open backing file: %s", 3138 strerror(errno)); 3139 return (-1); 3140 } 3141 sc->nvstore.type = NVME_STOR_BLOCKIF; 3142 sc->nvstore.size = blockif_size(sc->nvstore.ctx); 3143 } 3144 3145 if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) 3146 sc->nvstore.sectsz = sectsz; 3147 else if (sc->nvstore.type != NVME_STOR_RAM) 3148 sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); 3149 for (sc->nvstore.sectsz_bits = 9; 3150 (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; 3151 sc->nvstore.sectsz_bits++); 3152 3153 if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) 3154 sc->max_queues = NVME_QUEUES; 3155 3156 return (0); 3157 } 3158 3159 static void 3160 pci_nvme_resized(struct blockif_ctxt *bctxt, void *arg, size_t new_size) 3161 { 3162 struct pci_nvme_softc *sc; 3163 struct pci_nvme_blockstore *nvstore; 3164 struct nvme_namespace_data *nd; 3165 3166 sc = arg; 3167 nvstore = &sc->nvstore; 3168 nd = &sc->nsdata; 3169 3170 nvstore->size = new_size; 3171 pci_nvme_init_nsdata_size(nvstore, nd); 3172 3173 /* Add changed NSID to list */ 3174 sc->ns_log.ns[0] = 1; 3175 sc->ns_log.ns[1] = 0; 3176 3177 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE, 3178 PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED); 3179 } 3180 3181 static int 3182 pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl) 3183 { 3184 struct pci_nvme_softc *sc; 3185 uint32_t pci_membar_sz; 3186 int error; 3187 3188 error = 0; 3189 3190 sc = calloc(1, sizeof(struct pci_nvme_softc)); 3191 pi->pi_arg = sc; 3192 sc->nsc_pi = pi; 3193 3194 error = pci_nvme_parse_config(sc, nvl); 3195 if (error < 0) 3196 goto done; 3197 else 3198 error = 0; 3199 3200 STAILQ_INIT(&sc->ioreqs_free); 3201 sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq)); 3202 for (int i = 0; i < sc->ioslots; i++) { 3203 STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link); 3204 } 3205 3206 pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A); 3207 pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); 3208 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); 3209 pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM); 3210 pci_set_cfgdata8(pi, PCIR_PROGIF, 3211 PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); 3212 3213 /* 3214 * Allocate size of NVMe registers + doorbell space for all queues. 3215 * 3216 * The specification requires a minimum memory I/O window size of 16K. 3217 * The Windows driver will refuse to start a device with a smaller 3218 * window. 3219 */ 3220 pci_membar_sz = sizeof(struct nvme_registers) + 3221 2 * sizeof(uint32_t) * (sc->max_queues + 1); 3222 pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); 3223 3224 DPRINTF("nvme membar size: %u", pci_membar_sz); 3225 3226 error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz); 3227 if (error) { 3228 WPRINTF("%s pci alloc mem bar failed", __func__); 3229 goto done; 3230 } 3231 3232 error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR); 3233 if (error) { 3234 WPRINTF("%s pci add msixcap failed", __func__); 3235 goto done; 3236 } 3237 3238 error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP); 3239 if (error) { 3240 WPRINTF("%s pci add Express capability failed", __func__); 3241 goto done; 3242 } 3243 3244 pthread_mutex_init(&sc->mtx, NULL); 3245 sem_init(&sc->iosemlock, 0, sc->ioslots); 3246 blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc); 3247 3248 pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues); 3249 /* 3250 * Controller data depends on Namespace data so initialize Namespace 3251 * data first. 3252 */ 3253 pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore); 3254 pci_nvme_init_ctrldata(sc); 3255 pci_nvme_init_logpages(sc); 3256 pci_nvme_init_features(sc); 3257 3258 pci_nvme_aer_init(sc); 3259 pci_nvme_aen_init(sc); 3260 3261 pci_nvme_reset(sc); 3262 3263 pci_lintr_request(pi); 3264 3265 done: 3266 return (error); 3267 } 3268 3269 static int 3270 pci_nvme_legacy_config(nvlist_t *nvl, const char *opts) 3271 { 3272 char *cp, *ram; 3273 3274 if (opts == NULL) 3275 return (0); 3276 3277 if (strncmp(opts, "ram=", 4) == 0) { 3278 cp = strchr(opts, ','); 3279 if (cp == NULL) { 3280 set_config_value_node(nvl, "ram", opts + 4); 3281 return (0); 3282 } 3283 ram = strndup(opts + 4, cp - opts - 4); 3284 set_config_value_node(nvl, "ram", ram); 3285 free(ram); 3286 return (pci_parse_legacy_config(nvl, cp + 1)); 3287 } else 3288 return (blockif_legacy_config(nvl, opts)); 3289 } 3290 3291 struct pci_devemu pci_de_nvme = { 3292 .pe_emu = "nvme", 3293 .pe_init = pci_nvme_init, 3294 .pe_legacy_config = pci_nvme_legacy_config, 3295 .pe_barwrite = pci_nvme_write, 3296 .pe_barread = pci_nvme_read 3297 }; 3298 PCI_EMUL_SET(pci_de_nvme); 3299