13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 138834f7acSYouzhong Yang * Copyright 2016 The MathWorks, Inc. All rights reserved. 14e89be50aSRob Johnston * Copyright 2019 Joyent, Inc. 1534331de3SHans Rosenfeld * Copyright 2019 Unix Software Ltd. 16533affcbSRobert Mustacchi * Copyright 2024 Oxide Computer Company. 1704be5853SAndy Fiddaman * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. 18353d89b0SHans Rosenfeld * Copyright 2022 Tintri by DDN, Inc. All rights reserved. 193c9168faSHans Rosenfeld */ 203c9168faSHans Rosenfeld 213c9168faSHans Rosenfeld #ifndef _NVME_VAR_H 223c9168faSHans Rosenfeld #define _NVME_VAR_H 233c9168faSHans Rosenfeld 243c9168faSHans Rosenfeld #include <sys/ddi.h> 253c9168faSHans Rosenfeld #include <sys/sunddi.h> 263c9168faSHans Rosenfeld #include <sys/blkdev.h> 273c9168faSHans Rosenfeld #include <sys/taskq_impl.h> 28e984c70bSHans Rosenfeld #include <sys/list.h> 29533affcbSRobert Mustacchi #include <sys/ddi_ufm.h> 30533affcbSRobert Mustacchi #include <nvme_common.h> 313c9168faSHans Rosenfeld 323c9168faSHans Rosenfeld /* 333c9168faSHans Rosenfeld * NVMe driver state 343c9168faSHans Rosenfeld */ 353c9168faSHans Rosenfeld 363c9168faSHans Rosenfeld #ifdef __cplusplus 373d9b1a2aSHans Rosenfeld extern "C" { 383c9168faSHans Rosenfeld #endif 393c9168faSHans Rosenfeld 40e6dd7845SAndy Fiddaman typedef enum { 41e6dd7845SAndy Fiddaman NVME_PCI_CONFIG = 1 << 0, 42e6dd7845SAndy Fiddaman NVME_FMA_INIT = 1 << 1, 43e6dd7845SAndy Fiddaman NVME_REGS_MAPPED = 1 << 2, 44e6dd7845SAndy Fiddaman NVME_ADMIN_QUEUE = 1 << 3, 45e6dd7845SAndy Fiddaman NVME_CTRL_LIMITS = 1 << 4, 46e6dd7845SAndy Fiddaman NVME_INTERRUPTS = 1 << 5, 47e6dd7845SAndy Fiddaman NVME_UFM_INIT = 1 << 6, 48e6dd7845SAndy Fiddaman NVME_MUTEX_INIT = 1 << 7, 49533affcbSRobert Mustacchi NVME_MGMT_INIT = 1 << 8 50e6dd7845SAndy Fiddaman } nvme_progress_t; 51e6dd7845SAndy Fiddaman 52e6dd7845SAndy Fiddaman typedef enum { 53533affcbSRobert Mustacchi NVME_NS_LOCK = 1 << 0 54533affcbSRobert Mustacchi } nvme_ns_progress_t; 55533affcbSRobert Mustacchi 56533affcbSRobert Mustacchi typedef enum { 57e6dd7845SAndy Fiddaman /* 58e6dd7845SAndy Fiddaman * The controller fails to properly process commands on the admin queue 59e6dd7845SAndy Fiddaman * if the first one has CID 0. Subsequent use of CID 0 doesn't present 60e6dd7845SAndy Fiddaman * a problem. 61e6dd7845SAndy Fiddaman */ 62e6dd7845SAndy Fiddaman NVME_QUIRK_START_CID = 1 << 0, 63e6dd7845SAndy Fiddaman } nvme_quirk_t; 643c9168faSHans Rosenfeld 653c9168faSHans Rosenfeld #define NVME_MIN_ADMIN_QUEUE_LEN 16 663c9168faSHans Rosenfeld #define NVME_MIN_IO_QUEUE_LEN 16 673c9168faSHans Rosenfeld #define NVME_DEFAULT_ADMIN_QUEUE_LEN 256 683c9168faSHans Rosenfeld #define NVME_DEFAULT_IO_QUEUE_LEN 1024 693c9168faSHans Rosenfeld #define NVME_DEFAULT_ASYNC_EVENT_LIMIT 10 703c9168faSHans Rosenfeld #define NVME_MIN_ASYNC_EVENT_LIMIT 1 716801591eSHans Rosenfeld #define NVME_DEFAULT_MIN_BLOCK_SIZE 512 723c9168faSHans Rosenfeld 733c9168faSHans Rosenfeld 743c9168faSHans Rosenfeld typedef struct nvme nvme_t; 753c9168faSHans Rosenfeld typedef struct nvme_namespace nvme_namespace_t; 76533affcbSRobert Mustacchi typedef struct nvme_minor nvme_minor_t; 77533affcbSRobert Mustacchi typedef struct nvme_lock nvme_lock_t; 78533affcbSRobert Mustacchi typedef struct nvme_minor_lock_info nvme_minor_lock_info_t; 793c9168faSHans Rosenfeld typedef struct nvme_dma nvme_dma_t; 803c9168faSHans Rosenfeld typedef struct nvme_cmd nvme_cmd_t; 810999c112SPaul Winder typedef struct nvme_cq nvme_cq_t; 823c9168faSHans Rosenfeld typedef struct nvme_qpair nvme_qpair_t; 833c9168faSHans Rosenfeld typedef struct nvme_task_arg nvme_task_arg_t; 843c9168faSHans Rosenfeld 85533affcbSRobert Mustacchi /* 86533affcbSRobert Mustacchi * These states represent the minor's perspective. That is, of a minor's 87533affcbSRobert Mustacchi * namespace and controller lock, where is it? 88533affcbSRobert Mustacchi */ 89533affcbSRobert Mustacchi typedef enum { 90533affcbSRobert Mustacchi NVME_LOCK_STATE_UNLOCKED = 0, 91533affcbSRobert Mustacchi NVME_LOCK_STATE_BLOCKED, 92533affcbSRobert Mustacchi NVME_LOCK_STATE_ACQUIRED 93533affcbSRobert Mustacchi } nvme_minor_lock_state_t; 94533affcbSRobert Mustacchi 95533affcbSRobert Mustacchi struct nvme_minor_lock_info { 96533affcbSRobert Mustacchi list_node_t nli_node; 97533affcbSRobert Mustacchi nvme_lock_t *nli_lock; 98533affcbSRobert Mustacchi nvme_minor_lock_state_t nli_state; 99533affcbSRobert Mustacchi nvme_lock_level_t nli_curlevel; 100533affcbSRobert Mustacchi /* 101533affcbSRobert Mustacchi * While the minor points back to itself and the nvme_t should always 102533affcbSRobert Mustacchi * point to the current controller, the namespace should only point to 103533affcbSRobert Mustacchi * one if this is a particular namespace lock. The former two are 104533affcbSRobert Mustacchi * initialized at minor initialization time. 105533affcbSRobert Mustacchi */ 106533affcbSRobert Mustacchi nvme_minor_t *nli_minor; 107533affcbSRobert Mustacchi nvme_t *nli_nvme; 108533affcbSRobert Mustacchi nvme_namespace_t *nli_ns; 109533affcbSRobert Mustacchi /* 110533affcbSRobert Mustacchi * This is the common ioctl information that should be filled in when 111533affcbSRobert Mustacchi * we're being woken up for any reason other than an interrupted signal. 112533affcbSRobert Mustacchi * This should only be set while blocking. 113533affcbSRobert Mustacchi */ 114533affcbSRobert Mustacchi nvme_ioctl_common_t *nli_ioc; 115533affcbSRobert Mustacchi /* 116533affcbSRobert Mustacchi * The following are provided for debugging purposes. In particular, 117533affcbSRobert Mustacchi * information like the kthread_t and related that performed this should 118533affcbSRobert Mustacchi * be considered suspect as it represents who took the operation, not 119533affcbSRobert Mustacchi * who performed the operation (unless we're actively blocking). 120533affcbSRobert Mustacchi */ 121533affcbSRobert Mustacchi hrtime_t nli_last_change; 122533affcbSRobert Mustacchi uintptr_t nli_acq_kthread; 123533affcbSRobert Mustacchi pid_t nli_acq_pid; 124533affcbSRobert Mustacchi }; 125533affcbSRobert Mustacchi 126533affcbSRobert Mustacchi struct nvme_minor { 127533affcbSRobert Mustacchi /* 128533affcbSRobert Mustacchi * The following three fields are set when this is created. 129533affcbSRobert Mustacchi */ 130533affcbSRobert Mustacchi id_t nm_minor; 131533affcbSRobert Mustacchi nvme_t *nm_ctrl; 132533affcbSRobert Mustacchi nvme_namespace_t *nm_ns; 133533affcbSRobert Mustacchi /* 134533affcbSRobert Mustacchi * This link is used to index this minor on the global list of active 135533affcbSRobert Mustacchi * open-related minors. This is only manipulated under the 136533affcbSRobert Mustacchi * nvme_open_minors_mutex. 137533affcbSRobert Mustacchi */ 138533affcbSRobert Mustacchi avl_node_t nm_avl; 139533affcbSRobert Mustacchi /* 140533affcbSRobert Mustacchi * Information related to locking. Note, there is no pointer to a locked 141533affcbSRobert Mustacchi * controller as the only one can be the one specified here. This data 142533affcbSRobert Mustacchi * is protected by the controller's n_minor_mutex. 143533affcbSRobert Mustacchi */ 144533affcbSRobert Mustacchi kcondvar_t nm_cv; 145533affcbSRobert Mustacchi nvme_minor_lock_info_t nm_ctrl_lock; 146533affcbSRobert Mustacchi nvme_minor_lock_info_t nm_ns_lock; 147533affcbSRobert Mustacchi }; 148533affcbSRobert Mustacchi 149533affcbSRobert Mustacchi struct nvme_lock { 150533affcbSRobert Mustacchi nvme_minor_lock_info_t *nl_writer; 151533affcbSRobert Mustacchi list_t nl_readers; 152533affcbSRobert Mustacchi list_t nl_pend_readers; 153533affcbSRobert Mustacchi list_t nl_pend_writers; 154533affcbSRobert Mustacchi /* 155533affcbSRobert Mustacchi * The following are stats to indicate how often certain locking 156533affcbSRobert Mustacchi * activities have occurred for debugging purposes. 157533affcbSRobert Mustacchi */ 158533affcbSRobert Mustacchi uint32_t nl_nwrite_locks; 159533affcbSRobert Mustacchi uint32_t nl_nread_locks; 160533affcbSRobert Mustacchi uint32_t nl_npend_writes; 161533affcbSRobert Mustacchi uint32_t nl_npend_reads; 162533affcbSRobert Mustacchi uint32_t nl_nnonblock; 163533affcbSRobert Mustacchi uint32_t nl_nsignals; 164533affcbSRobert Mustacchi uint32_t nl_nsig_unlock; 165533affcbSRobert Mustacchi uint32_t nl_nsig_blocks; 166533affcbSRobert Mustacchi uint32_t nl_nsig_acq; 1673d9b1a2aSHans Rosenfeld }; 1683d9b1a2aSHans Rosenfeld 1693c9168faSHans Rosenfeld struct nvme_dma { 1703c9168faSHans Rosenfeld ddi_dma_handle_t nd_dmah; 1713c9168faSHans Rosenfeld ddi_acc_handle_t nd_acch; 1723c9168faSHans Rosenfeld ddi_dma_cookie_t nd_cookie; 1733c9168faSHans Rosenfeld uint_t nd_ncookie; 1743c9168faSHans Rosenfeld caddr_t nd_memp; 1753c9168faSHans Rosenfeld size_t nd_len; 1768834f7acSYouzhong Yang boolean_t nd_cached; 1773c9168faSHans Rosenfeld }; 1783c9168faSHans Rosenfeld 1793c9168faSHans Rosenfeld struct nvme_cmd { 180e984c70bSHans Rosenfeld struct list_node nc_list; 181e984c70bSHans Rosenfeld 1823c9168faSHans Rosenfeld nvme_sqe_t nc_sqe; 1833c9168faSHans Rosenfeld nvme_cqe_t nc_cqe; 1843c9168faSHans Rosenfeld 1853c9168faSHans Rosenfeld void (*nc_callback)(void *); 1863c9168faSHans Rosenfeld bd_xfer_t *nc_xfer; 1873c9168faSHans Rosenfeld boolean_t nc_completed; 1883d9b1a2aSHans Rosenfeld boolean_t nc_dontpanic; 1893c9168faSHans Rosenfeld uint16_t nc_sqid; 1903c9168faSHans Rosenfeld 1913c9168faSHans Rosenfeld nvme_dma_t *nc_dma; 192eb7b3ff7SBenjamin Naecker nvme_dma_t *nc_prp; /* DMA for PRP lists */ 1933c9168faSHans Rosenfeld 1943c9168faSHans Rosenfeld kmutex_t nc_mutex; 1953c9168faSHans Rosenfeld kcondvar_t nc_cv; 1963c9168faSHans Rosenfeld 1973c9168faSHans Rosenfeld taskq_ent_t nc_tqent; 1983c9168faSHans Rosenfeld nvme_t *nc_nvme; 1993c9168faSHans Rosenfeld }; 2003c9168faSHans Rosenfeld 2010999c112SPaul Winder struct nvme_cq { 2020999c112SPaul Winder size_t ncq_nentry; 2030999c112SPaul Winder uint16_t ncq_id; 2040999c112SPaul Winder 2050999c112SPaul Winder nvme_dma_t *ncq_dma; 2060999c112SPaul Winder nvme_cqe_t *ncq_cq; 2070999c112SPaul Winder uint_t ncq_head; 2080999c112SPaul Winder uint_t ncq_tail; 2090999c112SPaul Winder uintptr_t ncq_hdbl; 2100999c112SPaul Winder int ncq_phase; 2110999c112SPaul Winder 2124d95620bSPaul Winder taskq_t *ncq_cmd_taskq; 2134d95620bSPaul Winder 2140999c112SPaul Winder kmutex_t ncq_mutex; 2150999c112SPaul Winder }; 2160999c112SPaul Winder 2173c9168faSHans Rosenfeld struct nvme_qpair { 2183c9168faSHans Rosenfeld size_t nq_nentry; 2193c9168faSHans Rosenfeld 220ffb64830SJordan Paige Hendricks /* submission fields */ 2213c9168faSHans Rosenfeld nvme_dma_t *nq_sqdma; 2223c9168faSHans Rosenfeld nvme_sqe_t *nq_sq; 2233c9168faSHans Rosenfeld uint_t nq_sqhead; 2243c9168faSHans Rosenfeld uint_t nq_sqtail; 2253c9168faSHans Rosenfeld uintptr_t nq_sqtdbl; 2263c9168faSHans Rosenfeld 227ffb64830SJordan Paige Hendricks /* completion */ 2280999c112SPaul Winder nvme_cq_t *nq_cq; 2293c9168faSHans Rosenfeld 230ffb64830SJordan Paige Hendricks /* shared structures for completion and submission */ 231ffb64830SJordan Paige Hendricks nvme_cmd_t **nq_cmd; /* active command array */ 232ffb64830SJordan Paige Hendricks uint16_t nq_next_cmd; /* next potential empty queue slot */ 233ffb64830SJordan Paige Hendricks uint_t nq_active_cmds; /* number of active cmds */ 2343c9168faSHans Rosenfeld 235ffb64830SJordan Paige Hendricks kmutex_t nq_mutex; /* protects shared state */ 236ffb64830SJordan Paige Hendricks ksema_t nq_sema; /* semaphore to ensure q always has >= 1 empty slot */ 2373c9168faSHans Rosenfeld }; 2383c9168faSHans Rosenfeld 239*50d757e7SRobert Mustacchi typedef struct nvme_mgmt_lock { 240*50d757e7SRobert Mustacchi kmutex_t nml_lock; 241*50d757e7SRobert Mustacchi kcondvar_t nml_cv; 242*50d757e7SRobert Mustacchi uintptr_t nml_bd_own; 243*50d757e7SRobert Mustacchi } nvme_mgmt_lock_t; 244*50d757e7SRobert Mustacchi 2453c9168faSHans Rosenfeld struct nvme { 2463c9168faSHans Rosenfeld dev_info_t *n_dip; 247e6dd7845SAndy Fiddaman nvme_progress_t n_progress; 248e6dd7845SAndy Fiddaman nvme_quirk_t n_quirks; 2493c9168faSHans Rosenfeld 2503c9168faSHans Rosenfeld caddr_t n_regs; 2513c9168faSHans Rosenfeld ddi_acc_handle_t n_regh; 2523c9168faSHans Rosenfeld 2533c9168faSHans Rosenfeld kmem_cache_t *n_cmd_cache; 2548834f7acSYouzhong Yang kmem_cache_t *n_prp_cache; 2553c9168faSHans Rosenfeld 2563c9168faSHans Rosenfeld size_t n_inth_sz; 2573c9168faSHans Rosenfeld ddi_intr_handle_t *n_inth; 2583c9168faSHans Rosenfeld int n_intr_cnt; 2593c9168faSHans Rosenfeld uint_t n_intr_pri; 2603c9168faSHans Rosenfeld int n_intr_cap; 2613c9168faSHans Rosenfeld int n_intr_type; 2623c9168faSHans Rosenfeld int n_intr_types; 2633c9168faSHans Rosenfeld 264e6dd7845SAndy Fiddaman ddi_acc_handle_t n_pcicfg_handle; 265e6dd7845SAndy Fiddaman uint16_t n_vendor_id; 266e6dd7845SAndy Fiddaman uint16_t n_device_id; 267e6dd7845SAndy Fiddaman uint16_t n_subsystem_vendor_id; 268e6dd7845SAndy Fiddaman uint16_t n_subsystem_device_id; 269e6dd7845SAndy Fiddaman uint8_t n_revision_id; 270e6dd7845SAndy Fiddaman 271510a6847SHans Rosenfeld char *n_product; 272510a6847SHans Rosenfeld char *n_vendor; 273510a6847SHans Rosenfeld 27424979ca3SHans Rosenfeld nvme_version_t n_version; 2753c9168faSHans Rosenfeld boolean_t n_dead; 276533affcbSRobert Mustacchi nvme_ioctl_errno_t n_dead_status; 277533affcbSRobert Mustacchi taskq_ent_t n_dead_tqent; 2783c9168faSHans Rosenfeld boolean_t n_strict_version; 2793c9168faSHans Rosenfeld boolean_t n_ignore_unknown_vendor_status; 2803c9168faSHans Rosenfeld uint32_t n_admin_queue_len; 2810999c112SPaul Winder uint32_t n_io_squeue_len; 2820999c112SPaul Winder uint32_t n_io_cqueue_len; 2833c9168faSHans Rosenfeld uint16_t n_async_event_limit; 2846801591eSHans Rosenfeld uint_t n_min_block_size; 2853c9168faSHans Rosenfeld uint16_t n_abort_command_limit; 2863c9168faSHans Rosenfeld uint64_t n_max_data_transfer_size; 287d148d46eSHans Rosenfeld boolean_t n_write_cache_present; 288d148d46eSHans Rosenfeld boolean_t n_write_cache_enabled; 2893c9168faSHans Rosenfeld int n_error_log_len; 29008139162SToomas Soome boolean_t n_async_event_supported; 2910999c112SPaul Winder int n_submission_queues; 2920999c112SPaul Winder int n_completion_queues; 2933c9168faSHans Rosenfeld 2943c9168faSHans Rosenfeld int n_nssr_supported; 2953c9168faSHans Rosenfeld int n_doorbell_stride; 2963c9168faSHans Rosenfeld int n_timeout; 2973c9168faSHans Rosenfeld int n_arbitration_mechanisms; 2983c9168faSHans Rosenfeld int n_cont_queues_reqd; 2993c9168faSHans Rosenfeld int n_max_queue_entries; 3003c9168faSHans Rosenfeld int n_pageshift; 3013c9168faSHans Rosenfeld int n_pagesize; 3023c9168faSHans Rosenfeld 303533affcbSRobert Mustacchi uint32_t n_namespace_count; 3044d95620bSPaul Winder uint_t n_namespaces_attachable; 3050999c112SPaul Winder uint_t n_ioq_count; 3060999c112SPaul Winder uint_t n_cq_count; 3073c9168faSHans Rosenfeld 308533affcbSRobert Mustacchi /* 309533affcbSRobert Mustacchi * This is cached identify controller and common namespace data that 310533affcbSRobert Mustacchi * exists in the system. This generally can be used in the kernel; 311533affcbSRobert Mustacchi * however, we have to be careful about what we use here because these 312533affcbSRobert Mustacchi * values are not refreshed after attach. Therefore these are good for 313533affcbSRobert Mustacchi * answering the question what does the controller support or what is in 314533affcbSRobert Mustacchi * the common namespace information, but not otherwise. That means you 315533affcbSRobert Mustacchi * shouldn't use this to try to answer how much capacity is still in the 316533affcbSRobert Mustacchi * controller because this information is just cached. 317533affcbSRobert Mustacchi */ 3183c9168faSHans Rosenfeld nvme_identify_ctrl_t *n_idctl; 319533affcbSRobert Mustacchi nvme_identify_nsid_t *n_idcomns; 3203c9168faSHans Rosenfeld 321ffb64830SJordan Paige Hendricks /* Pointer to the admin queue, which is always queue 0 in n_ioq. */ 3223c9168faSHans Rosenfeld nvme_qpair_t *n_adminq; 323ffb64830SJordan Paige Hendricks /* 324ffb64830SJordan Paige Hendricks * All command queues, including the admin queue. 325ffb64830SJordan Paige Hendricks * Its length is: n_ioq_count + 1. 326ffb64830SJordan Paige Hendricks */ 3273c9168faSHans Rosenfeld nvme_qpair_t **n_ioq; 3280999c112SPaul Winder nvme_cq_t **n_cq; 3293c9168faSHans Rosenfeld 3303c9168faSHans Rosenfeld nvme_namespace_t *n_ns; 3313c9168faSHans Rosenfeld 3323c9168faSHans Rosenfeld ddi_dma_attr_t n_queue_dma_attr; 3333c9168faSHans Rosenfeld ddi_dma_attr_t n_prp_dma_attr; 3343c9168faSHans Rosenfeld ddi_dma_attr_t n_sgl_dma_attr; 3353c9168faSHans Rosenfeld ddi_device_acc_attr_t n_reg_acc_attr; 3363c9168faSHans Rosenfeld ddi_iblock_cookie_t n_fm_ibc; 3373c9168faSHans Rosenfeld int n_fm_cap; 3383c9168faSHans Rosenfeld 3393c9168faSHans Rosenfeld ksema_t n_abort_sema; 3403c9168faSHans Rosenfeld 34134331de3SHans Rosenfeld /* protects namespace management operations */ 342*50d757e7SRobert Mustacchi nvme_mgmt_lock_t n_mgmt; 34334331de3SHans Rosenfeld 344533affcbSRobert Mustacchi /* 345533affcbSRobert Mustacchi * This lock protects the minor node locking state across the controller 346533affcbSRobert Mustacchi * and all related namespaces. 347533affcbSRobert Mustacchi */ 348353d89b0SHans Rosenfeld kmutex_t n_minor_mutex; 349533affcbSRobert Mustacchi nvme_lock_t n_lock; 3503c9168faSHans Rosenfeld 3513c9168faSHans Rosenfeld /* errors detected by driver */ 3523c9168faSHans Rosenfeld uint32_t n_dma_bind_err; 3533c9168faSHans Rosenfeld uint32_t n_abort_failed; 3543c9168faSHans Rosenfeld uint32_t n_cmd_timeout; 3553c9168faSHans Rosenfeld uint32_t n_cmd_aborted; 3563c9168faSHans Rosenfeld uint32_t n_wrong_logpage; 3573c9168faSHans Rosenfeld uint32_t n_unknown_logpage; 3583c9168faSHans Rosenfeld uint32_t n_too_many_cookies; 359e6dd7845SAndy Fiddaman uint32_t n_unknown_cid; 3603c9168faSHans Rosenfeld 3613c9168faSHans Rosenfeld /* errors detected by hardware */ 3623c9168faSHans Rosenfeld uint32_t n_data_xfr_err; 3633c9168faSHans Rosenfeld uint32_t n_internal_err; 3643c9168faSHans Rosenfeld uint32_t n_abort_rq_err; 3653c9168faSHans Rosenfeld uint32_t n_abort_sq_del; 3663c9168faSHans Rosenfeld uint32_t n_nvm_cap_exc; 3673c9168faSHans Rosenfeld uint32_t n_nvm_ns_notrdy; 3683eaaeb3dSAndy Fiddaman uint32_t n_nvm_ns_formatting; 3693c9168faSHans Rosenfeld uint32_t n_inv_cq_err; 3703c9168faSHans Rosenfeld uint32_t n_inv_qid_err; 3713c9168faSHans Rosenfeld uint32_t n_max_qsz_exc; 3723c9168faSHans Rosenfeld uint32_t n_inv_int_vect; 3733c9168faSHans Rosenfeld uint32_t n_inv_log_page; 3743c9168faSHans Rosenfeld uint32_t n_inv_format; 3753c9168faSHans Rosenfeld uint32_t n_inv_q_del; 3763c9168faSHans Rosenfeld uint32_t n_cnfl_attr; 3773c9168faSHans Rosenfeld uint32_t n_inv_prot; 3783c9168faSHans Rosenfeld uint32_t n_readonly; 3793c9168faSHans Rosenfeld 3803c9168faSHans Rosenfeld /* errors reported by asynchronous events */ 3813c9168faSHans Rosenfeld uint32_t n_diagfail_event; 3823c9168faSHans Rosenfeld uint32_t n_persistent_event; 3833c9168faSHans Rosenfeld uint32_t n_transient_event; 3843c9168faSHans Rosenfeld uint32_t n_fw_load_event; 3853c9168faSHans Rosenfeld uint32_t n_reliability_event; 3863c9168faSHans Rosenfeld uint32_t n_temperature_event; 3873c9168faSHans Rosenfeld uint32_t n_spare_event; 3883c9168faSHans Rosenfeld uint32_t n_vendor_event; 38904be5853SAndy Fiddaman uint32_t n_notice_event; 3903c9168faSHans Rosenfeld uint32_t n_unknown_event; 3913c9168faSHans Rosenfeld 392ffb64830SJordan Paige Hendricks /* hot removal NDI event handling */ 393ffb64830SJordan Paige Hendricks ddi_eventcookie_t n_rm_cookie; 394ffb64830SJordan Paige Hendricks ddi_callback_id_t n_ev_rm_cb_id; 395ffb64830SJordan Paige Hendricks 396e89be50aSRob Johnston /* DDI UFM handle */ 397e89be50aSRob Johnston ddi_ufm_handle_t *n_ufmh; 398e89be50aSRob Johnston /* Cached Firmware Slot Information log page */ 399e89be50aSRob Johnston nvme_fwslot_log_t *n_fwslot; 400e89be50aSRob Johnston /* Lock protecting the cached firmware slot info */ 401e89be50aSRob Johnston kmutex_t n_fwslot_mutex; 4023c9168faSHans Rosenfeld }; 4033c9168faSHans Rosenfeld 4043c9168faSHans Rosenfeld struct nvme_namespace { 4053c9168faSHans Rosenfeld nvme_t *ns_nvme; 406533affcbSRobert Mustacchi nvme_ns_progress_t ns_progress; 40724979ca3SHans Rosenfeld uint8_t ns_eui64[8]; 408b8f43eb6SHans Rosenfeld uint8_t ns_nguid[16]; 409670f080bSHans Rosenfeld char ns_name[11]; 41024979ca3SHans Rosenfeld 4113c9168faSHans Rosenfeld bd_handle_t ns_bd_hdl; 4123c9168faSHans Rosenfeld 4133c9168faSHans Rosenfeld uint32_t ns_id; 4143c9168faSHans Rosenfeld size_t ns_block_count; 4153c9168faSHans Rosenfeld size_t ns_block_size; 4163c9168faSHans Rosenfeld size_t ns_best_block_size; 4173c9168faSHans Rosenfeld 418baf9a850SHans Rosenfeld boolean_t ns_allocated; 419baf9a850SHans Rosenfeld boolean_t ns_active; 4203c9168faSHans Rosenfeld boolean_t ns_ignore; 42134331de3SHans Rosenfeld boolean_t ns_attached; 4223c9168faSHans Rosenfeld 4233c9168faSHans Rosenfeld nvme_identify_nsid_t *ns_idns; 4243c9168faSHans Rosenfeld 425533affcbSRobert Mustacchi /* 426533affcbSRobert Mustacchi * Namespace lock, see the theory statement for more information. 427533affcbSRobert Mustacchi */ 428533affcbSRobert Mustacchi nvme_lock_t ns_lock; 4293d9b1a2aSHans Rosenfeld 4303c9168faSHans Rosenfeld /* 431b8f43eb6SHans Rosenfeld * If a namespace has neither NGUID nor EUI64, we create a devid in 43224979ca3SHans Rosenfeld * nvme_prepare_devid(). 4333c9168faSHans Rosenfeld */ 43424979ca3SHans Rosenfeld char *ns_devid; 4353c9168faSHans Rosenfeld }; 4363c9168faSHans Rosenfeld 4373c9168faSHans Rosenfeld struct nvme_task_arg { 4383c9168faSHans Rosenfeld nvme_t *nt_nvme; 4393c9168faSHans Rosenfeld nvme_cmd_t *nt_cmd; 4403c9168faSHans Rosenfeld }; 4413c9168faSHans Rosenfeld 442533affcbSRobert Mustacchi typedef enum { 443533affcbSRobert Mustacchi /* 444533affcbSRobert Mustacchi * This indicates that there is no exclusive access required for this 445533affcbSRobert Mustacchi * operation. However, this operation will fail if someone attempts to 446533affcbSRobert Mustacchi * perform this operation and someone else holds a write lock. 447533affcbSRobert Mustacchi */ 448533affcbSRobert Mustacchi NVME_IOCTL_EXCL_NONE = 0, 449533affcbSRobert Mustacchi /* 450533affcbSRobert Mustacchi * This indicates that a write lock is required to perform the 451533affcbSRobert Mustacchi * operation. 452533affcbSRobert Mustacchi */ 453533affcbSRobert Mustacchi NVME_IOCTL_EXCL_WRITE, 454533affcbSRobert Mustacchi /* 455533affcbSRobert Mustacchi * This indicates that the exclusive check should be skipped. The only 456533affcbSRobert Mustacchi * case this should be used in is the lock and unlock ioctls as they 457533affcbSRobert Mustacchi * should be able to proceed even when the controller is being used 458533affcbSRobert Mustacchi * exclusively. 459533affcbSRobert Mustacchi */ 460533affcbSRobert Mustacchi NVME_IOCTL_EXCL_SKIP 461533affcbSRobert Mustacchi } nvme_ioctl_excl_t; 462533affcbSRobert Mustacchi 463533affcbSRobert Mustacchi /* 464533affcbSRobert Mustacchi * This structure represents the set of checks that we apply to ioctl's using 465533affcbSRobert Mustacchi * the nvme_ioctl_common_t structure as part of validation. 466533affcbSRobert Mustacchi */ 467533affcbSRobert Mustacchi typedef struct nvme_ioctl_check { 468533affcbSRobert Mustacchi /* 469533affcbSRobert Mustacchi * This indicates whether or not the command in question allows a 470533affcbSRobert Mustacchi * namespace to be specified at all. If this is false, a namespace minor 471533affcbSRobert Mustacchi * cannot be used and a controller minor must leave the nsid set to 472533affcbSRobert Mustacchi * zero. 473533affcbSRobert Mustacchi */ 474533affcbSRobert Mustacchi boolean_t nck_ns_ok; 475533affcbSRobert Mustacchi /* 476533affcbSRobert Mustacchi * This indicates that a minor node corresponding to a namespace is 477533affcbSRobert Mustacchi * allowed to issue this. 478533affcbSRobert Mustacchi */ 479533affcbSRobert Mustacchi boolean_t nck_ns_minor_ok; 480533affcbSRobert Mustacchi /* 481533affcbSRobert Mustacchi * This indicates that the controller should be skipped from all of the 482533affcbSRobert Mustacchi * following processing behavior. That is, it's allowed to specify 483533affcbSRobert Mustacchi * whatever it wants in the nsid field, regardless if it is valid or 484533affcbSRobert Mustacchi * not. This is required for some of the Identify Command options that 485533affcbSRobert Mustacchi * list endpoints. This should generally not be used and the driver 486533affcbSRobert Mustacchi * should still validate the nuance here. 487533affcbSRobert Mustacchi */ 488533affcbSRobert Mustacchi boolean_t nck_skip_ctrl; 489533affcbSRobert Mustacchi /* 490533affcbSRobert Mustacchi * This indicates that if we're on the controller's minor and we don't 491533affcbSRobert Mustacchi * have an explicit namespace ID (i.e. 0), should the namespace be 492533affcbSRobert Mustacchi * rewritten to be the broadcast namespace. 493533affcbSRobert Mustacchi */ 494533affcbSRobert Mustacchi boolean_t nck_ctrl_rewrite; 495533affcbSRobert Mustacchi /* 496533affcbSRobert Mustacchi * This indicates whether or not the broadcast NSID is acceptable for 497533affcbSRobert Mustacchi * the controller node. 498533affcbSRobert Mustacchi */ 499533affcbSRobert Mustacchi boolean_t nck_bcast_ok; 500533affcbSRobert Mustacchi 501533affcbSRobert Mustacchi /* 502533affcbSRobert Mustacchi * This indicates to the lock checking code what kind of exclusive 503533affcbSRobert Mustacchi * access is required. This check occurs after any namespace rewriting 504533affcbSRobert Mustacchi * has occurred. When looking at exclusivity, a broadcast namespace or 505533affcbSRobert Mustacchi * namespace 0 indicate that the controller is the target, otherwise the 506533affcbSRobert Mustacchi * target namespace will be checked for a write lock. 507533affcbSRobert Mustacchi */ 508533affcbSRobert Mustacchi nvme_ioctl_excl_t nck_excl; 509533affcbSRobert Mustacchi } nvme_ioctl_check_t; 510533affcbSRobert Mustacchi 511533affcbSRobert Mustacchi /* 512533affcbSRobert Mustacchi * Constants 513533affcbSRobert Mustacchi */ 514533affcbSRobert Mustacchi extern uint_t nvme_vendor_specific_admin_cmd_max_timeout; 515533affcbSRobert Mustacchi extern uint32_t nvme_vendor_specific_admin_cmd_size; 516533affcbSRobert Mustacchi 517533affcbSRobert Mustacchi /* 518533affcbSRobert Mustacchi * Common functions. 519533affcbSRobert Mustacchi */ 520533affcbSRobert Mustacchi extern nvme_namespace_t *nvme_nsid2ns(nvme_t *, uint32_t); 521533affcbSRobert Mustacchi extern boolean_t nvme_ioctl_error(nvme_ioctl_common_t *, nvme_ioctl_errno_t, 522533affcbSRobert Mustacchi uint32_t, uint32_t); 523533affcbSRobert Mustacchi extern boolean_t nvme_ctrl_atleast(nvme_t *, const nvme_version_t *); 524533affcbSRobert Mustacchi extern void nvme_ioctl_success(nvme_ioctl_common_t *); 525533affcbSRobert Mustacchi 526533affcbSRobert Mustacchi /* 527533affcbSRobert Mustacchi * Validation related functions and kernel tunable limits. 528533affcbSRobert Mustacchi */ 529533affcbSRobert Mustacchi extern boolean_t nvme_validate_logpage(nvme_t *, nvme_ioctl_get_logpage_t *); 530533affcbSRobert Mustacchi extern boolean_t nvme_validate_identify(nvme_t *, nvme_ioctl_identify_t *, 531533affcbSRobert Mustacchi boolean_t); 532533affcbSRobert Mustacchi extern boolean_t nvme_validate_get_feature(nvme_t *, 533533affcbSRobert Mustacchi nvme_ioctl_get_feature_t *); 534533affcbSRobert Mustacchi extern boolean_t nvme_validate_vuc(nvme_t *, nvme_ioctl_passthru_t *); 535533affcbSRobert Mustacchi extern boolean_t nvme_validate_format(nvme_t *, nvme_ioctl_format_t *); 536533affcbSRobert Mustacchi extern boolean_t nvme_validate_fw_load(nvme_t *, nvme_ioctl_fw_load_t *); 537533affcbSRobert Mustacchi extern boolean_t nvme_validate_fw_commit(nvme_t *, nvme_ioctl_fw_commit_t *); 538533affcbSRobert Mustacchi 539533affcbSRobert Mustacchi /* 540533affcbSRobert Mustacchi * Locking functions 541533affcbSRobert Mustacchi */ 542533affcbSRobert Mustacchi extern void nvme_rwlock(nvme_minor_t *, nvme_ioctl_lock_t *); 543533affcbSRobert Mustacchi extern void nvme_rwunlock(nvme_minor_lock_info_t *, nvme_lock_t *); 544533affcbSRobert Mustacchi extern void nvme_rwlock_ctrl_dead(void *); 545533affcbSRobert Mustacchi extern void nvme_lock_init(nvme_lock_t *); 546533affcbSRobert Mustacchi extern void nvme_lock_fini(nvme_lock_t *); 547533affcbSRobert Mustacchi 5483c9168faSHans Rosenfeld #ifdef __cplusplus 5493d9b1a2aSHans Rosenfeld } 5503c9168faSHans Rosenfeld #endif 5513c9168faSHans Rosenfeld 5523c9168faSHans Rosenfeld #endif /* _NVME_VAR_H */ 553