1 /* 2 * Copyright (c) 2016 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * Primary header file 36 * NVME softc and structural definitions 37 */ 38 39 #if defined(__DragonFly__) 40 #include "nvme_dragonfly.h" 41 #else 42 #error "build for OS unknown" 43 #endif 44 #include "nvme_fw.h" 45 #include "nvme_log.h" 46 #include "nvme_ident.h" 47 #include "nvme_ns.h" 48 #include "nvme_chipset.h" 49 #include "nvme_ioctl.h" 50 51 MALLOC_DECLARE(M_NVME); 52 53 /* 54 * Choose some reasonable limit, even if the hardware supports more. 55 */ 56 #define NVME_MAX_QUEUES 1024 57 #define NVME_MAX_NAMESPACES 1024 58 59 struct nvme_queue; 60 struct nvme_softc; 61 struct nvme_softns; 62 63 /* 64 * Device matching array for special attach/detach codings. 65 */ 66 typedef struct { 67 pci_vendor_id_t vendor; 68 pci_product_id_t product; 69 int (*attach)(device_t dev); 70 int (*detach)(device_t dev); 71 char *name; 72 } nvme_device_t; 73 74 /* 75 * Kernel-level request structure. This structure allows the driver to 76 * construct, issue, wait for, and process responses to commands. Each 77 * queue manages its own request bank. 78 * 79 * In order to disconnect the request structure from the hardware queue 80 * mechanism itself, to reduce SMP conflicts and interactions, and allow 81 * command/response processing to block without interfering with queue 82 * operations, this structure embeds a copy of the HW command and response 83 * structures instead of referencing the ones in the actual hardware queues. 84 * These will be copied to/from the actual queue entries by lower-level 85 * chipset code. 86 * 87 * Requests are associated with particular queues, completions can occur on 88 * any queue. Requests associated with the admin queue conveniently include 89 * an additional 4K 'info' block suitable for DMA. 90 */ 91 typedef struct nvme_request { 92 struct nvme_request *next_avail; 93 struct nvme_subqueue *subq; /* which queue is submission on */ 94 struct nvme_comqueue *comq; /* which queue is completion on */ 95 uint32_t state; 96 uint32_t cmd_id; /* reqary[] index */ 97 int waiting; 98 nvme_allcmd_t cmd; /* hw submit structure for entry */ 99 nvme_allres_t res; /* hw completion structure for entry */ 100 nvme_admin_data_t *info; /* DMA data (admin request only) */ 101 bus_addr_t pinfo; /* phys address for PRP */ 102 103 /* 104 * Aux fields to keep track of bio and other data, depending on 105 * the callback. If the callback is NULL the caller will poll for 106 * completion. 107 */ 108 void (*callback)(struct nvme_request *req, struct lock *lk); 109 struct nvme_softns *nsc; 110 struct bio *bio; 111 } nvme_request_t; 112 113 #define NVME_REQ_AVAIL 0 114 #define NVME_REQ_ALLOCATED 1 115 #define NVME_REQ_SUBMITTED 2 116 #define NVME_REQ_COMPLETED 3 117 118 typedef struct nvme_subqueue { 119 /* 120 * Driver stuff 121 */ 122 struct lock lk; /* queue lock controls access */ 123 struct nvme_softc *sc; 124 nvme_request_t *first_avail; 125 nvme_request_t *reqary; 126 uint32_t nqe; /* #of queue entries */ 127 uint16_t qid; /* which queue# are we on? */ 128 uint16_t comqid; /* we are tied to this completion qu */ 129 uint32_t subq_doorbell_reg; 130 uint32_t subq_head; /* start of active requests */ 131 uint32_t subq_tail; /* new requests */ 132 uint32_t unsubmitted; /* #unsubmitted requests */ 133 int unused01; 134 int signal_requeue; 135 136 /* 137 * DMA resources 138 */ 139 bus_dmamap_t sque_map; 140 bus_dmamap_t prps_map; 141 nvme_allcmd_t *ksubq; /* kernel-mapped addresses */ 142 uint64_t *kprps; /* enough PRPs per request for */ 143 /* MAXPHYS bytes worth of mappings */ 144 bus_addr_t psubq; /* physical addresses */ 145 bus_addr_t pprps; 146 147 /* 148 * Additional DMA resources for admin queue (A NVME_MAX_ADMIN_BUFFER 149 * sized buffer for each queue entry). 150 */ 151 bus_dmamap_t adm_map; 152 bus_addr_t pdatapgs; 153 nvme_admin_data_t *kdatapgs; 154 } nvme_subqueue_t; 155 156 typedef struct nvme_comqueue { 157 /* 158 * Driver stuff 159 */ 160 struct lock lk; /* queue lock controls access */ 161 struct nvme_softc *sc; 162 uint32_t nqe; /* #of queue entries */ 163 uint16_t phase; /* phase to match (res->tail.status) */ 164 uint16_t qid; /* which queue# are we on? */ 165 uint32_t comq_doorbell_reg; 166 uint32_t comq_head; /* consume responses */ 167 uint32_t comq_tail; 168 169 /* 170 * DMA resources 171 */ 172 bus_dmamap_t cque_map; 173 nvme_allres_t *kcomq; 174 bus_addr_t pcomq; 175 } nvme_comqueue_t; 176 177 typedef struct nvme_softns { 178 struct nvme_softc *sc; 179 nvme_ident_ns_data_t idns; 180 struct bio_queue_head bioq; /* excess BIOs */ 181 struct lock lk; /* mostly for bioq handling */ 182 int state; 183 int unit; 184 uint32_t nsid; 185 uint32_t blksize; 186 struct disk disk; /* disk attachment */ 187 struct devstat stats; 188 cdev_t cdev; /* disk device (cdev) */ 189 } nvme_softns_t; 190 191 #define NVME_NSC_STATE_UNATTACHED 0 192 #define NVME_NSC_STATE_ATTACHED 1 193 194 195 typedef struct nvme_softc { 196 TAILQ_ENTRY(nvme_softc) entry; /* global list */ 197 device_t dev; /* core device */ 198 const nvme_device_t *ad; /* quirk detection etc */ 199 thread_t admintd; 200 uint32_t maxqe; 201 uint64_t cap; 202 uint32_t vers; 203 uint32_t dstrd4; /* doorbell stride */ 204 uint32_t entimo; /* enable timeout in ticks */ 205 uint16_t niosubqs; /* #of I/O submission queues */ 206 uint16_t niocomqs; /* #of I/O completion queues */ 207 uint16_t dumpqno; 208 uint16_t eventqno; 209 uint16_t qmap[SMP_MAXCPU][2]; 210 uint32_t flags; 211 nvme_subqueue_t subqueues[NVME_MAX_QUEUES]; 212 nvme_comqueue_t comqueues[NVME_MAX_QUEUES]; 213 int cputovect[SMP_MAXCPU]; 214 215 /* 216 * bio/disk layer tracking 217 */ 218 ulong opencnt; 219 220 /* 221 * admin queue irq resources 222 * register map resources 223 */ 224 struct resource *regs; 225 struct resource *bar4; 226 bus_space_tag_t iot; 227 bus_space_handle_t ioh; 228 int rid_regs; 229 int rid_bar4; 230 231 int nirqs; 232 int irq_type; 233 struct resource *irq[NVME_MAX_QUEUES]; 234 int rid_irq[NVME_MAX_QUEUES]; 235 void *irq_handle[NVME_MAX_QUEUES]; 236 237 /* 238 * dma tags 239 */ 240 bus_dma_tag_t prps_tag; /* worst-case PRP table(s) per queue */ 241 bus_dma_tag_t sque_tag; /* submission queue */ 242 bus_dma_tag_t cque_tag; /* completion queue */ 243 bus_dma_tag_t adm_tag; /* DMA data buffers for admin cmds */ 244 size_t prp_bytes; 245 size_t cmd_bytes; 246 size_t res_bytes; 247 size_t adm_bytes; 248 249 /* 250 * Admin thread and controller identify data 251 */ 252 uint32_t admin_signal; 253 struct lock admin_lk; 254 struct lock ioctl_lk; 255 int (*admin_func)(struct nvme_softc *); 256 nvme_ident_ctlr_data_t idctlr; 257 nvme_softns_t *nscary[NVME_MAX_NAMESPACES]; 258 int nscmax; 259 } nvme_softc_t; 260 261 #define NVME_SC_ATTACHED 0x00000001 262 #define NVME_SC_UNLOADING 0x00000002 263 264 #define ADMIN_SIG_STOP 0x00000001 265 #define ADMIN_SIG_RUNNING 0x00000002 266 #define ADMIN_SIG_PROBED 0x00000004 267 #define ADMIN_SIG_REQUEUE 0x00000008 268 #define ADMIN_SIG_RUN_MASK (ADMIN_SIG_STOP | ADMIN_SIG_REQUEUE) 269 270 #define NVME_QMAP_RD 0 271 #define NVME_QMAP_WR 1 272 273 /* 274 * Prototypes 275 */ 276 const nvme_device_t *nvme_lookup_device(device_t dev); 277 void nvme_os_sleep(int ms); 278 int nvme_os_softsleep(void); 279 void nvme_os_hardsleep(int us); 280 u_int32_t nvme_read(nvme_softc_t *sc, bus_size_t r); 281 u_int64_t nvme_read8(nvme_softc_t *sc, bus_size_t r); 282 void nvme_write(nvme_softc_t *sc, bus_size_t r, u_int32_t v); 283 void nvme_write8(nvme_softc_t *sc, bus_size_t r, u_int64_t v); 284 int nvme_enable(nvme_softc_t *sc, int enable); 285 int nvme_alloc_subqueue(nvme_softc_t *sc, uint16_t qid); 286 int nvme_alloc_comqueue(nvme_softc_t *sc, uint16_t qid); 287 void nvme_free_subqueue(nvme_softc_t *sc, uint16_t qid); 288 void nvme_free_comqueue(nvme_softc_t *sc, uint16_t qid); 289 290 nvme_request_t *nvme_get_admin_request(nvme_softc_t *sc, uint8_t opcode); 291 nvme_request_t *nvme_get_request(nvme_subqueue_t *queue, uint8_t opcode, 292 char *kva, size_t bytes); 293 void nvme_submit_request(nvme_request_t *req); 294 int nvme_wait_request(nvme_request_t *req, int ticks); 295 void nvme_put_request(nvme_request_t *req); 296 void nvme_poll_completions(nvme_comqueue_t *queue, struct lock *lk); 297 298 int nvme_start_admin_thread(nvme_softc_t *sc); 299 void nvme_stop_admin_thread(nvme_softc_t *sc); 300 301 int nvme_create_subqueue(nvme_softc_t *sc, uint16_t qid); 302 int nvme_create_comqueue(nvme_softc_t *sc, uint16_t qid); 303 int nvme_delete_subqueue(nvme_softc_t *sc, uint16_t qid); 304 int nvme_delete_comqueue(nvme_softc_t *sc, uint16_t qid); 305 int nvme_issue_shutdown(nvme_softc_t *sc); 306 307 void nvme_disk_attach(nvme_softns_t *nsc); 308 void nvme_disk_detach(nvme_softns_t *nsc); 309 void nvme_disk_requeues(nvme_softc_t *sc); 310 int nvme_alloc_disk_unit(void); 311 312 int nvme_getlog_ioctl(nvme_softc_t *sc, nvme_getlog_ioctl_t *ioc); 313 314 void nvme_intr(void *arg); 315 size_t string_cleanup(char *str, int domiddle); 316