1*0f19ad86Skrw /* $OpenBSD: nvme.c,v 1.69 2020/03/01 14:10:56 krw Exp $ */ 2282c0692Sdlg 3282c0692Sdlg /* 4282c0692Sdlg * Copyright (c) 2014 David Gwynne <dlg@openbsd.org> 5282c0692Sdlg * 6282c0692Sdlg * Permission to use, copy, modify, and distribute this software for any 7282c0692Sdlg * purpose with or without fee is hereby granted, provided that the above 8282c0692Sdlg * copyright notice and this permission notice appear in all copies. 9282c0692Sdlg * 10282c0692Sdlg * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11282c0692Sdlg * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12282c0692Sdlg * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13282c0692Sdlg * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14282c0692Sdlg * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15282c0692Sdlg * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16282c0692Sdlg * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17282c0692Sdlg */ 18282c0692Sdlg 19282c0692Sdlg #include <sys/param.h> 20282c0692Sdlg #include <sys/systm.h> 21282c0692Sdlg #include <sys/buf.h> 22282c0692Sdlg #include <sys/kernel.h> 23282c0692Sdlg #include <sys/malloc.h> 24282c0692Sdlg #include <sys/device.h> 25282c0692Sdlg #include <sys/queue.h> 26282c0692Sdlg #include <sys/mutex.h> 27282c0692Sdlg #include <sys/pool.h> 28282c0692Sdlg 2903d86467Sjmatthew #include <sys/atomic.h> 3003d86467Sjmatthew 31282c0692Sdlg #include <machine/bus.h> 32282c0692Sdlg 33282c0692Sdlg #include <scsi/scsi_all.h> 34c947e044Sdlg #include <scsi/scsi_disk.h> 35282c0692Sdlg #include <scsi/scsiconf.h> 36282c0692Sdlg 37448b3c09Sdlg #include <dev/ic/nvmereg.h> 38448b3c09Sdlg #include <dev/ic/nvmevar.h> 39448b3c09Sdlg 40282c0692Sdlg struct cfdriver nvme_cd = { 41282c0692Sdlg NULL, 42282c0692Sdlg "nvme", 43282c0692Sdlg DV_DULL 44282c0692Sdlg }; 45282c0692Sdlg 46282c0692Sdlg int nvme_ready(struct nvme_softc *, u_int32_t); 47ee8b2d53Skrw int nvme_enable(struct nvme_softc *); 48282c0692Sdlg int nvme_disable(struct nvme_softc *); 495313ab17Sdlg int nvme_shutdown(struct nvme_softc *); 50aa5ddcf9Ssf int nvme_resume(struct nvme_softc *); 51282c0692Sdlg 52282c0692Sdlg void nvme_dumpregs(struct nvme_softc *); 5388aa9192Sdlg int nvme_identify(struct nvme_softc *, u_int); 54e623ca5aSdlg void nvme_fill_identify(struct nvme_softc *, struct nvme_ccb *, void *); 55282c0692Sdlg 56448b3c09Sdlg int nvme_ccbs_alloc(struct nvme_softc *, u_int); 57a1141c40Stedu void nvme_ccbs_free(struct nvme_softc *, u_int); 58448b3c09Sdlg 59448b3c09Sdlg void * nvme_ccb_get(void *); 60448b3c09Sdlg void nvme_ccb_put(void *, void *); 61448b3c09Sdlg 62e623ca5aSdlg int nvme_poll(struct nvme_softc *, struct nvme_queue *, struct nvme_ccb *, 63cca2dfedSdlg void (*)(struct nvme_softc *, struct nvme_ccb *, void *)); 64e623ca5aSdlg void nvme_poll_fill(struct nvme_softc *, struct nvme_ccb *, void *); 65e623ca5aSdlg void nvme_poll_done(struct nvme_softc *, struct nvme_ccb *, 66e623ca5aSdlg struct nvme_cqe *); 67ad9e5681Sdlg void nvme_sqe_fill(struct nvme_softc *, struct nvme_ccb *, void *); 68e623ca5aSdlg void nvme_empty_done(struct nvme_softc *, struct nvme_ccb *, 69e623ca5aSdlg struct nvme_cqe *); 70448b3c09Sdlg 71e623ca5aSdlg struct nvme_queue * 7259968badSdlg nvme_q_alloc(struct nvme_softc *, u_int16_t, u_int, u_int); 73cd15a86fSdlg int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 74aa5ddcf9Ssf int nvme_q_reset(struct nvme_softc *, struct nvme_queue *); 755313ab17Sdlg int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 76448b3c09Sdlg void nvme_q_submit(struct nvme_softc *, 77448b3c09Sdlg struct nvme_queue *, struct nvme_ccb *, 78e623ca5aSdlg void (*)(struct nvme_softc *, struct nvme_ccb *, void *)); 79e623ca5aSdlg int nvme_q_complete(struct nvme_softc *, struct nvme_queue *); 80e623ca5aSdlg void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 81282c0692Sdlg 82e623ca5aSdlg struct nvme_dmamem * 83e623ca5aSdlg nvme_dmamem_alloc(struct nvme_softc *, size_t); 84e623ca5aSdlg void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 85e623ca5aSdlg void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int); 86282c0692Sdlg 8711624e4cSdlg void nvme_scsi_cmd(struct scsi_xfer *); 8811624e4cSdlg int nvme_scsi_probe(struct scsi_link *); 8911624e4cSdlg void nvme_scsi_free(struct scsi_link *); 9011624e4cSdlg 9103d86467Sjmatthew #ifdef HIBERNATE 9203d86467Sjmatthew #include <uvm/uvm_extern.h> 9303d86467Sjmatthew #include <sys/hibernate.h> 9403d86467Sjmatthew #include <sys/disk.h> 9503d86467Sjmatthew #include <sys/disklabel.h> 9603d86467Sjmatthew 9703d86467Sjmatthew int nvme_hibernate_io(dev_t, daddr_t, vaddr_t, size_t, int, void *); 9803d86467Sjmatthew #endif 9903d86467Sjmatthew 10011624e4cSdlg struct scsi_adapter nvme_switch = { 10121ceeee0Skrw nvme_scsi_cmd, NULL, nvme_scsi_probe, nvme_scsi_free, NULL 10211624e4cSdlg }; 10311624e4cSdlg 1041eef25a1Sdlg void nvme_scsi_io(struct scsi_xfer *, int); 1051eef25a1Sdlg void nvme_scsi_io_fill(struct nvme_softc *, struct nvme_ccb *, void *); 106a1e87500Sdlg void nvme_scsi_io_done(struct nvme_softc *, struct nvme_ccb *, 107a1e87500Sdlg struct nvme_cqe *); 108a1e87500Sdlg 109689c2c68Sdlg void nvme_scsi_sync(struct scsi_xfer *); 110689c2c68Sdlg void nvme_scsi_sync_fill(struct nvme_softc *, struct nvme_ccb *, void *); 111689c2c68Sdlg void nvme_scsi_sync_done(struct nvme_softc *, struct nvme_ccb *, 112689c2c68Sdlg struct nvme_cqe *); 113689c2c68Sdlg 1149856392eSdlg void nvme_scsi_inq(struct scsi_xfer *); 1159856392eSdlg void nvme_scsi_inquiry(struct scsi_xfer *); 116c947e044Sdlg void nvme_scsi_capacity16(struct scsi_xfer *); 117c947e044Sdlg void nvme_scsi_capacity(struct scsi_xfer *); 1189856392eSdlg 119282c0692Sdlg #define nvme_read4(_s, _r) \ 120282c0692Sdlg bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 121282c0692Sdlg #define nvme_write4(_s, _r, _v) \ 122282c0692Sdlg bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 123343e9e5aSmpi /* 124343e9e5aSmpi * Some controllers, at least Apple NVMe, always require split 125343e9e5aSmpi * transfers, so don't use bus_space_{read,write}_8() on LP64. 126343e9e5aSmpi */ 127282c0692Sdlg static inline u_int64_t 128282c0692Sdlg nvme_read8(struct nvme_softc *sc, bus_size_t r) 129282c0692Sdlg { 130282c0692Sdlg u_int64_t v; 131282c0692Sdlg u_int32_t *a = (u_int32_t *)&v; 132282c0692Sdlg 133282c0692Sdlg #if _BYTE_ORDER == _LITTLE_ENDIAN 134282c0692Sdlg a[0] = nvme_read4(sc, r); 135282c0692Sdlg a[1] = nvme_read4(sc, r + 4); 136282c0692Sdlg #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 137282c0692Sdlg a[1] = nvme_read4(sc, r); 138282c0692Sdlg a[0] = nvme_read4(sc, r + 4); 139282c0692Sdlg #endif 140282c0692Sdlg 141282c0692Sdlg return (v); 142282c0692Sdlg } 143282c0692Sdlg 144282c0692Sdlg static inline void 145282c0692Sdlg nvme_write8(struct nvme_softc *sc, bus_size_t r, u_int64_t v) 146282c0692Sdlg { 147282c0692Sdlg u_int32_t *a = (u_int32_t *)&v; 148282c0692Sdlg 149282c0692Sdlg #if _BYTE_ORDER == _LITTLE_ENDIAN 150282c0692Sdlg nvme_write4(sc, r, a[0]); 151282c0692Sdlg nvme_write4(sc, r + 4, a[1]); 152282c0692Sdlg #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 153282c0692Sdlg nvme_write4(sc, r, a[1]); 154282c0692Sdlg nvme_write4(sc, r + 4, a[0]); 155282c0692Sdlg #endif 156282c0692Sdlg } 157282c0692Sdlg #define nvme_barrier(_s, _r, _l, _f) \ 158282c0692Sdlg bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 159282c0692Sdlg 160282c0692Sdlg void 161282c0692Sdlg nvme_dumpregs(struct nvme_softc *sc) 162282c0692Sdlg { 163282c0692Sdlg u_int64_t r8; 164282c0692Sdlg u_int32_t r4; 165282c0692Sdlg 166282c0692Sdlg r8 = nvme_read8(sc, NVME_CAP); 167282c0692Sdlg printf("%s: cap 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 168282c0692Sdlg printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 169282c0692Sdlg (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 170282c0692Sdlg printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 171282c0692Sdlg (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 172282c0692Sdlg printf("%s: css %llu\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 173282c0692Sdlg printf("%s: nssrs %llu\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 174a7f13332Sdlg printf("%s: dstrd %u\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 175282c0692Sdlg printf("%s: to %llu msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 176282c0692Sdlg printf("%s: ams %llu\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 177282c0692Sdlg printf("%s: cqr %llu\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 178282c0692Sdlg printf("%s: mqes %llu\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 179282c0692Sdlg 180a7f13332Sdlg printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 181282c0692Sdlg 182282c0692Sdlg r4 = nvme_read4(sc, NVME_CC); 183a7f13332Sdlg printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 184282c0692Sdlg printf("%s: iocqes %u\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4)); 185282c0692Sdlg printf("%s: iosqes %u\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4)); 186282c0692Sdlg printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 187282c0692Sdlg printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 188282c0692Sdlg printf("%s: mps %u\n", DEVNAME(sc), NVME_CC_MPS_R(r4)); 189282c0692Sdlg printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 190282c0692Sdlg printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN)); 191282c0692Sdlg 192a7f13332Sdlg printf("%s: csts 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_CSTS)); 193a7f13332Sdlg printf("%s: aqa 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_AQA)); 194282c0692Sdlg printf("%s: asq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 195282c0692Sdlg printf("%s: acq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 196282c0692Sdlg } 197282c0692Sdlg 198282c0692Sdlg int 199282c0692Sdlg nvme_ready(struct nvme_softc *sc, u_int32_t rdy) 200282c0692Sdlg { 201282c0692Sdlg u_int i = 0; 202282c0692Sdlg 203282c0692Sdlg while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 204282c0692Sdlg if (i++ > sc->sc_rdy_to) 205282c0692Sdlg return (1); 206282c0692Sdlg 207282c0692Sdlg delay(1000); 208282c0692Sdlg nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 209282c0692Sdlg } 210282c0692Sdlg 211282c0692Sdlg return (0); 212282c0692Sdlg } 213282c0692Sdlg 214282c0692Sdlg int 215ee8b2d53Skrw nvme_enable(struct nvme_softc *sc) 216282c0692Sdlg { 217282c0692Sdlg u_int32_t cc; 218282c0692Sdlg 219282c0692Sdlg cc = nvme_read4(sc, NVME_CC); 220282c0692Sdlg if (ISSET(cc, NVME_CC_EN)) 221282c0692Sdlg return (nvme_ready(sc, NVME_CSTS_RDY)); 222282c0692Sdlg 22383d8579cSdlg nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 22483d8579cSdlg NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 22583d8579cSdlg nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 22683d8579cSdlg 227282c0692Sdlg nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 228282c0692Sdlg nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 229282c0692Sdlg nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 230282c0692Sdlg nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 231282c0692Sdlg 232282c0692Sdlg CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 233282c0692Sdlg NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 234*0f19ad86Skrw SET(cc, NVME_CC_IOSQES(6)); /* Submission queue size == 2**6 (64) */ 235*0f19ad86Skrw SET(cc, NVME_CC_IOCQES(4)); /* Completion queue size == 2**4 (16) */ 236282c0692Sdlg SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 237282c0692Sdlg SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 238282c0692Sdlg SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 239*0f19ad86Skrw SET(cc, NVME_CC_MPS(ffs(sc->sc_mps) - 1)); 240282c0692Sdlg SET(cc, NVME_CC_EN); 241282c0692Sdlg 242282c0692Sdlg nvme_write4(sc, NVME_CC, cc); 243282c0692Sdlg nvme_barrier(sc, 0, sc->sc_ios, 244282c0692Sdlg BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 245282c0692Sdlg 24683d8579cSdlg return (nvme_ready(sc, NVME_CSTS_RDY)); 247282c0692Sdlg } 248282c0692Sdlg 249282c0692Sdlg int 250282c0692Sdlg nvme_disable(struct nvme_softc *sc) 251282c0692Sdlg { 252282c0692Sdlg u_int32_t cc, csts; 253282c0692Sdlg 254282c0692Sdlg cc = nvme_read4(sc, NVME_CC); 255282c0692Sdlg if (ISSET(cc, NVME_CC_EN)) { 256282c0692Sdlg csts = nvme_read4(sc, NVME_CSTS); 257282c0692Sdlg if (!ISSET(csts, NVME_CSTS_CFS) && 258448b3c09Sdlg nvme_ready(sc, NVME_CSTS_RDY) != 0) 259282c0692Sdlg return (1); 260282c0692Sdlg } 261282c0692Sdlg 262282c0692Sdlg CLR(cc, NVME_CC_EN); 263282c0692Sdlg 264282c0692Sdlg nvme_write4(sc, NVME_CC, cc); 265282c0692Sdlg nvme_barrier(sc, 0, sc->sc_ios, 266282c0692Sdlg BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 267282c0692Sdlg 268282c0692Sdlg return (nvme_ready(sc, 0)); 269282c0692Sdlg } 270282c0692Sdlg 271282c0692Sdlg int 272282c0692Sdlg nvme_attach(struct nvme_softc *sc) 273282c0692Sdlg { 27401b059acSdlg struct scsibus_attach_args saa; 275282c0692Sdlg u_int64_t cap; 276282c0692Sdlg u_int32_t reg; 277282c0692Sdlg u_int mps = PAGE_SHIFT; 278a1141c40Stedu u_int nccbs = 0; 279282c0692Sdlg 2803f4555f0Sdlg mtx_init(&sc->sc_ccb_mtx, IPL_BIO); 2813f4555f0Sdlg SIMPLEQ_INIT(&sc->sc_ccb_list); 2823f4555f0Sdlg scsi_iopool_init(&sc->sc_iopool, sc, nvme_ccb_get, nvme_ccb_put); 2833f4555f0Sdlg 284282c0692Sdlg reg = nvme_read4(sc, NVME_VS); 285282c0692Sdlg if (reg == 0xffffffff) { 286282c0692Sdlg printf(", invalid mapping\n"); 287282c0692Sdlg return (1); 288282c0692Sdlg } 289282c0692Sdlg 290a4fcfefeSmpi printf(", NVMe %d.%d\n", NVME_VS_MJR(reg), NVME_VS_MNR(reg)); 291282c0692Sdlg 292282c0692Sdlg cap = nvme_read8(sc, NVME_CAP); 293aa5ddcf9Ssf sc->sc_dstrd = NVME_CAP_DSTRD(cap); 2941be1268fSdlg if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 2951be1268fSdlg printf("%s: NVMe minimum page size %u " 2961be1268fSdlg "is greater than CPU page size %u\n", DEVNAME(sc), 2971be1268fSdlg 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 2981be1268fSdlg return (1); 2991be1268fSdlg } 3001be1268fSdlg if (NVME_CAP_MPSMAX(cap) < mps) 301282c0692Sdlg mps = NVME_CAP_MPSMAX(cap); 302282c0692Sdlg 303282c0692Sdlg sc->sc_rdy_to = NVME_CAP_TO(cap); 304448b3c09Sdlg sc->sc_mps = 1 << mps; 305448b3c09Sdlg sc->sc_mdts = MAXPHYS; 306448b3c09Sdlg sc->sc_max_sgl = 2; 3073f4555f0Sdlg 3083f4555f0Sdlg if (nvme_disable(sc) != 0) { 3093f4555f0Sdlg printf("%s: unable to disable controller\n", DEVNAME(sc)); 3103f4555f0Sdlg return (1); 3113f4555f0Sdlg } 312282c0692Sdlg 313aa5ddcf9Ssf sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, 128, sc->sc_dstrd); 314282c0692Sdlg if (sc->sc_admin_q == NULL) { 315282c0692Sdlg printf("%s: unable to allocate admin queue\n", DEVNAME(sc)); 316282c0692Sdlg return (1); 317282c0692Sdlg } 318282c0692Sdlg 319448b3c09Sdlg if (nvme_ccbs_alloc(sc, 16) != 0) { 320448b3c09Sdlg printf("%s: unable to allocate initial ccbs\n", DEVNAME(sc)); 321282c0692Sdlg goto free_admin_q; 322282c0692Sdlg } 323a1141c40Stedu nccbs = 16; 324282c0692Sdlg 325ee8b2d53Skrw if (nvme_enable(sc) != 0) { 326448b3c09Sdlg printf("%s: unable to enable controller\n", DEVNAME(sc)); 327448b3c09Sdlg goto free_ccbs; 328448b3c09Sdlg } 329448b3c09Sdlg 33088aa9192Sdlg if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 331448b3c09Sdlg printf("%s: unable to identify controller\n", DEVNAME(sc)); 332448b3c09Sdlg goto disable; 333448b3c09Sdlg } 334282c0692Sdlg 3351926545cSdlg /* we know how big things are now */ 3361926545cSdlg sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 3371926545cSdlg 338a1141c40Stedu nvme_ccbs_free(sc, nccbs); 3391926545cSdlg if (nvme_ccbs_alloc(sc, 64) != 0) { 3401926545cSdlg printf("%s: unable to allocate ccbs\n", DEVNAME(sc)); 3411926545cSdlg goto free_admin_q; 3421926545cSdlg } 343a1141c40Stedu nccbs = 64; 3441926545cSdlg 34503d86467Sjmatthew sc->sc_q = nvme_q_alloc(sc, NVME_IO_Q, 128, sc->sc_dstrd); 34660c144b3Sdlg if (sc->sc_q == NULL) { 34760c144b3Sdlg printf("%s: unable to allocate io q\n", DEVNAME(sc)); 34860c144b3Sdlg goto disable; 34960c144b3Sdlg } 35060c144b3Sdlg 35160c144b3Sdlg if (nvme_q_create(sc, sc->sc_q) != 0) { 35260c144b3Sdlg printf("%s: unable to create io q\n", DEVNAME(sc)); 35360c144b3Sdlg goto free_q; 35460c144b3Sdlg } 35560c144b3Sdlg 35603d86467Sjmatthew sc->sc_hib_q = nvme_q_alloc(sc, NVME_HIB_Q, 4, sc->sc_dstrd); 35703d86467Sjmatthew if (sc->sc_hib_q == NULL) { 35803d86467Sjmatthew printf("%s: unable to allocate hibernate io queue\n", DEVNAME(sc)); 35989715fd5Sjsg goto free_q; 36003d86467Sjmatthew } 36103d86467Sjmatthew 362ffeaebd1Sdlg nvme_write4(sc, NVME_INTMC, 1); 363ffeaebd1Sdlg 364397f5692Skettenis sc->sc_namespaces = mallocarray(sc->sc_nn + 1, 365397f5692Skettenis sizeof(*sc->sc_namespaces), M_DEVBUF, M_WAITOK|M_ZERO); 366bc407f60Sdlg 36701b059acSdlg sc->sc_link.adapter = &nvme_switch; 36801b059acSdlg sc->sc_link.adapter_softc = sc; 369397f5692Skettenis sc->sc_link.adapter_buswidth = sc->sc_nn + 1; 37001b059acSdlg sc->sc_link.luns = 1; 371397f5692Skettenis sc->sc_link.adapter_target = 0; 3723aa07c40Sdlg sc->sc_link.openings = 64; 37301b059acSdlg sc->sc_link.pool = &sc->sc_iopool; 37401b059acSdlg 37501b059acSdlg memset(&saa, 0, sizeof(saa)); 37601b059acSdlg saa.saa_sc_link = &sc->sc_link; 37701b059acSdlg 37801b059acSdlg sc->sc_scsibus = (struct scsibus_softc *)config_found(&sc->sc_dev, 37901b059acSdlg &saa, scsiprint); 38001b059acSdlg 381282c0692Sdlg return (0); 382282c0692Sdlg 38360c144b3Sdlg free_q: 38460c144b3Sdlg nvme_q_free(sc, sc->sc_q); 385448b3c09Sdlg disable: 386448b3c09Sdlg nvme_disable(sc); 387448b3c09Sdlg free_ccbs: 388a1141c40Stedu nvme_ccbs_free(sc, nccbs); 389282c0692Sdlg free_admin_q: 390448b3c09Sdlg nvme_q_free(sc, sc->sc_admin_q); 391282c0692Sdlg 392282c0692Sdlg return (1); 393282c0692Sdlg } 394282c0692Sdlg 39511624e4cSdlg int 396aa5ddcf9Ssf nvme_resume(struct nvme_softc *sc) 397aa5ddcf9Ssf { 398aa5ddcf9Ssf if (nvme_disable(sc) != 0) { 399aa5ddcf9Ssf printf("%s: unable to disable controller\n", DEVNAME(sc)); 400aa5ddcf9Ssf return (1); 401aa5ddcf9Ssf } 402aa5ddcf9Ssf 403aa5ddcf9Ssf if (nvme_q_reset(sc, sc->sc_admin_q) != 0) { 404aa5ddcf9Ssf printf("%s: unable to reset admin queue\n", DEVNAME(sc)); 405aa5ddcf9Ssf return (1); 406aa5ddcf9Ssf } 407aa5ddcf9Ssf 408ee8b2d53Skrw if (nvme_enable(sc) != 0) { 409aa5ddcf9Ssf printf("%s: unable to enable controller\n", DEVNAME(sc)); 410aa5ddcf9Ssf return (1); 411aa5ddcf9Ssf } 412aa5ddcf9Ssf 41303d86467Sjmatthew sc->sc_q = nvme_q_alloc(sc, NVME_IO_Q, 128, sc->sc_dstrd); 414aa5ddcf9Ssf if (sc->sc_q == NULL) { 415aa5ddcf9Ssf printf("%s: unable to allocate io q\n", DEVNAME(sc)); 416aa5ddcf9Ssf goto disable; 417aa5ddcf9Ssf } 418aa5ddcf9Ssf 419aa5ddcf9Ssf if (nvme_q_create(sc, sc->sc_q) != 0) { 420aa5ddcf9Ssf printf("%s: unable to create io q\n", DEVNAME(sc)); 421aa5ddcf9Ssf goto free_q; 422aa5ddcf9Ssf } 423aa5ddcf9Ssf 424aa5ddcf9Ssf nvme_write4(sc, NVME_INTMC, 1); 425aa5ddcf9Ssf 426aa5ddcf9Ssf return (0); 427aa5ddcf9Ssf 428aa5ddcf9Ssf free_q: 429aa5ddcf9Ssf nvme_q_free(sc, sc->sc_q); 430aa5ddcf9Ssf disable: 431aa5ddcf9Ssf nvme_disable(sc); 432aa5ddcf9Ssf 433aa5ddcf9Ssf return (1); 434aa5ddcf9Ssf } 435aa5ddcf9Ssf 436aa5ddcf9Ssf int 43711624e4cSdlg nvme_scsi_probe(struct scsi_link *link) 43811624e4cSdlg { 43965dd51f8Sdlg struct nvme_softc *sc = link->adapter_softc; 44065dd51f8Sdlg struct nvme_sqe sqe; 44165dd51f8Sdlg struct nvm_identify_namespace *identify; 44265dd51f8Sdlg struct nvme_dmamem *mem; 44365dd51f8Sdlg struct nvme_ccb *ccb; 44465dd51f8Sdlg int rv; 44565dd51f8Sdlg 44665dd51f8Sdlg ccb = scsi_io_get(&sc->sc_iopool, 0); 44765dd51f8Sdlg KASSERT(ccb != NULL); 44865dd51f8Sdlg 44965dd51f8Sdlg mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 45065dd51f8Sdlg if (mem == NULL) 45165dd51f8Sdlg return (ENOMEM); 45265dd51f8Sdlg 45365dd51f8Sdlg memset(&sqe, 0, sizeof(sqe)); 45465dd51f8Sdlg sqe.opcode = NVM_ADMIN_IDENTIFY; 455397f5692Skettenis htolem32(&sqe.nsid, link->target); 45665dd51f8Sdlg htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 45765dd51f8Sdlg htolem32(&sqe.cdw10, 0); 45865dd51f8Sdlg 45965dd51f8Sdlg ccb->ccb_done = nvme_empty_done; 46065dd51f8Sdlg ccb->ccb_cookie = &sqe; 46165dd51f8Sdlg 46265dd51f8Sdlg nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 46365dd51f8Sdlg rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 46465dd51f8Sdlg nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 46565dd51f8Sdlg 46665dd51f8Sdlg scsi_io_put(&sc->sc_iopool, ccb); 46765dd51f8Sdlg 46865dd51f8Sdlg if (rv != 0) { 46965dd51f8Sdlg rv = EIO; 47065dd51f8Sdlg goto done; 47165dd51f8Sdlg } 47265dd51f8Sdlg 47365dd51f8Sdlg /* commit */ 47465dd51f8Sdlg 47565dd51f8Sdlg identify = malloc(sizeof(*identify), M_DEVBUF, M_WAITOK|M_ZERO); 47665dd51f8Sdlg memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 47765dd51f8Sdlg 47865dd51f8Sdlg sc->sc_namespaces[link->target].ident = identify; 47965dd51f8Sdlg 48065dd51f8Sdlg done: 48165dd51f8Sdlg nvme_dmamem_free(sc, mem); 48265dd51f8Sdlg 48365dd51f8Sdlg return (rv); 48411624e4cSdlg } 48511624e4cSdlg 4865313ab17Sdlg int 4875313ab17Sdlg nvme_shutdown(struct nvme_softc *sc) 4885313ab17Sdlg { 4895313ab17Sdlg u_int32_t cc, csts; 4905313ab17Sdlg int i; 4915313ab17Sdlg 4925313ab17Sdlg nvme_write4(sc, NVME_INTMC, 0); 4935313ab17Sdlg 4945313ab17Sdlg if (nvme_q_delete(sc, sc->sc_q) != 0) { 4955313ab17Sdlg printf("%s: unable to delete q, disabling\n", DEVNAME(sc)); 4965313ab17Sdlg goto disable; 4975313ab17Sdlg } 4985313ab17Sdlg 4995313ab17Sdlg cc = nvme_read4(sc, NVME_CC); 5005313ab17Sdlg CLR(cc, NVME_CC_SHN_MASK); 5015313ab17Sdlg SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 5025313ab17Sdlg nvme_write4(sc, NVME_CC, cc); 5035313ab17Sdlg 5045313ab17Sdlg for (i = 0; i < 4000; i++) { 5055313ab17Sdlg nvme_barrier(sc, 0, sc->sc_ios, 5065313ab17Sdlg BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 5075313ab17Sdlg csts = nvme_read4(sc, NVME_CSTS); 5085313ab17Sdlg if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 5095313ab17Sdlg return (0); 5105313ab17Sdlg 5115313ab17Sdlg delay(1000); 5125313ab17Sdlg } 5135313ab17Sdlg 5147f0b5f10Sderaadt printf("%s: unable to shutdown, disabling\n", DEVNAME(sc)); 5155313ab17Sdlg 5165313ab17Sdlg disable: 5175313ab17Sdlg nvme_disable(sc); 5185313ab17Sdlg return (0); 5195313ab17Sdlg } 5205313ab17Sdlg 5215313ab17Sdlg int 5225313ab17Sdlg nvme_activate(struct nvme_softc *sc, int act) 5235313ab17Sdlg { 5245313ab17Sdlg int rv; 5255313ab17Sdlg 5265313ab17Sdlg switch (act) { 5275313ab17Sdlg case DVACT_POWERDOWN: 5285313ab17Sdlg rv = config_activate_children(&sc->sc_dev, act); 5295313ab17Sdlg nvme_shutdown(sc); 5305313ab17Sdlg break; 531aa5ddcf9Ssf case DVACT_RESUME: 532aa5ddcf9Ssf rv = nvme_resume(sc); 533aa5ddcf9Ssf if (rv == 0) 534aa5ddcf9Ssf rv = config_activate_children(&sc->sc_dev, act); 535aa5ddcf9Ssf break; 5365313ab17Sdlg default: 5375313ab17Sdlg rv = config_activate_children(&sc->sc_dev, act); 5385313ab17Sdlg break; 5395313ab17Sdlg } 5405313ab17Sdlg 5415313ab17Sdlg return (rv); 5425313ab17Sdlg } 5435313ab17Sdlg 54411624e4cSdlg void 54511624e4cSdlg nvme_scsi_cmd(struct scsi_xfer *xs) 54611624e4cSdlg { 5479856392eSdlg switch (xs->cmd->opcode) { 548a1e87500Sdlg case READ_COMMAND: 549a1e87500Sdlg case READ_BIG: 550a1e87500Sdlg case READ_12: 551a1e87500Sdlg case READ_16: 5521eef25a1Sdlg nvme_scsi_io(xs, SCSI_DATA_IN); 553a1e87500Sdlg return; 554a1e87500Sdlg case WRITE_COMMAND: 555a1e87500Sdlg case WRITE_BIG: 556a1e87500Sdlg case WRITE_12: 557a1e87500Sdlg case WRITE_16: 5581eef25a1Sdlg nvme_scsi_io(xs, SCSI_DATA_OUT); 559d3f19a0bSdlg return; 560a1e87500Sdlg 561689c2c68Sdlg case SYNCHRONIZE_CACHE: 562689c2c68Sdlg nvme_scsi_sync(xs); 563689c2c68Sdlg return; 564689c2c68Sdlg 5659856392eSdlg case INQUIRY: 5669856392eSdlg nvme_scsi_inq(xs); 5679856392eSdlg return; 568c947e044Sdlg case READ_CAPACITY_16: 569c947e044Sdlg nvme_scsi_capacity16(xs); 570c947e044Sdlg return; 571c947e044Sdlg case READ_CAPACITY: 572c947e044Sdlg nvme_scsi_capacity(xs); 573c947e044Sdlg return; 574c947e044Sdlg 575a50b57eaSdlg case TEST_UNIT_READY: 576a50b57eaSdlg case PREVENT_ALLOW: 577a50b57eaSdlg case START_STOP: 578a50b57eaSdlg xs->error = XS_NOERROR; 579a50b57eaSdlg scsi_done(xs); 580a50b57eaSdlg return; 581a50b57eaSdlg 5829856392eSdlg default: 5839856392eSdlg break; 5849856392eSdlg } 5859856392eSdlg 58611624e4cSdlg xs->error = XS_DRIVER_STUFFUP; 58711624e4cSdlg scsi_done(xs); 58811624e4cSdlg } 58911624e4cSdlg 59011624e4cSdlg void 5911eef25a1Sdlg nvme_scsi_io(struct scsi_xfer *xs, int dir) 592a1e87500Sdlg { 593a1e87500Sdlg struct scsi_link *link = xs->sc_link; 594a1e87500Sdlg struct nvme_softc *sc = link->adapter_softc; 595a1e87500Sdlg struct nvme_ccb *ccb = xs->io; 596a1e87500Sdlg bus_dmamap_t dmap = ccb->ccb_dmamap; 597ecfae151Sdlg int i; 598a1e87500Sdlg 5991eef25a1Sdlg if ((xs->flags & (SCSI_DATA_IN|SCSI_DATA_OUT)) != dir) 6001eef25a1Sdlg goto stuffup; 6011eef25a1Sdlg 602a1e87500Sdlg ccb->ccb_done = nvme_scsi_io_done; 603a1e87500Sdlg ccb->ccb_cookie = xs; 604a1e87500Sdlg 605a1e87500Sdlg if (bus_dmamap_load(sc->sc_dmat, dmap, 606a1e87500Sdlg xs->data, xs->datalen, NULL, ISSET(xs->flags, SCSI_NOSLEEP) ? 607a1e87500Sdlg BUS_DMA_NOWAIT : BUS_DMA_WAITOK) != 0) 608a1e87500Sdlg goto stuffup; 609a1e87500Sdlg 610a1e87500Sdlg bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 611a1e87500Sdlg ISSET(xs->flags, SCSI_DATA_IN) ? 612a1e87500Sdlg BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 613a1e87500Sdlg 614ecfae151Sdlg if (dmap->dm_nsegs > 2) { 615ecfae151Sdlg for (i = 1; i < dmap->dm_nsegs; i++) { 616ecfae151Sdlg htolem64(&ccb->ccb_prpl[i - 1], 617ecfae151Sdlg dmap->dm_segs[i].ds_addr); 618ecfae151Sdlg } 619ecfae151Sdlg bus_dmamap_sync(sc->sc_dmat, 620ecfae151Sdlg NVME_DMA_MAP(sc->sc_ccb_prpls), 621ecfae151Sdlg ccb->ccb_prpl_off, 622ecfae151Sdlg sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 623ecfae151Sdlg BUS_DMASYNC_PREWRITE); 624ecfae151Sdlg } 625ecfae151Sdlg 626a1e87500Sdlg if (ISSET(xs->flags, SCSI_POLL)) { 6271eef25a1Sdlg nvme_poll(sc, sc->sc_q, ccb, nvme_scsi_io_fill); 628a1e87500Sdlg return; 629a1e87500Sdlg } 630a1e87500Sdlg 6311eef25a1Sdlg nvme_q_submit(sc, sc->sc_q, ccb, nvme_scsi_io_fill); 632a1e87500Sdlg return; 633a1e87500Sdlg 634a1e87500Sdlg stuffup: 635a1e87500Sdlg xs->error = XS_DRIVER_STUFFUP; 636a1e87500Sdlg scsi_done(xs); 637a1e87500Sdlg } 638a1e87500Sdlg 639a1e87500Sdlg void 6401eef25a1Sdlg nvme_scsi_io_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 641a1e87500Sdlg { 642a1e87500Sdlg struct nvme_sqe_io *sqe = slot; 643a1e87500Sdlg struct scsi_xfer *xs = ccb->ccb_cookie; 644a1e87500Sdlg struct scsi_link *link = xs->sc_link; 645a1e87500Sdlg bus_dmamap_t dmap = ccb->ccb_dmamap; 646a1e87500Sdlg u_int64_t lba; 647a1e87500Sdlg u_int32_t blocks; 648a1e87500Sdlg 649a1e87500Sdlg scsi_cmd_rw_decode(xs->cmd, &lba, &blocks); 650a1e87500Sdlg 6511eef25a1Sdlg sqe->opcode = ISSET(xs->flags, SCSI_DATA_IN) ? 6521eef25a1Sdlg NVM_CMD_READ : NVM_CMD_WRITE; 653397f5692Skettenis htolem32(&sqe->nsid, link->target); 654a1e87500Sdlg 655a1e87500Sdlg htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 656a1e87500Sdlg switch (dmap->dm_nsegs) { 657a1e87500Sdlg case 1: 658a1e87500Sdlg break; 659a1e87500Sdlg case 2: 660a1e87500Sdlg htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 661a1e87500Sdlg break; 662a1e87500Sdlg default: 663ecfae151Sdlg /* the prp list is already set up and synced */ 664ecfae151Sdlg htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 665ecfae151Sdlg break; 666a1e87500Sdlg } 667a1e87500Sdlg 668a1e87500Sdlg htolem64(&sqe->slba, lba); 669a1e87500Sdlg htolem16(&sqe->nlb, blocks - 1); 670a1e87500Sdlg } 671a1e87500Sdlg 672a1e87500Sdlg void 673a1e87500Sdlg nvme_scsi_io_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 674a1e87500Sdlg struct nvme_cqe *cqe) 675a1e87500Sdlg { 676a1e87500Sdlg struct scsi_xfer *xs = ccb->ccb_cookie; 677a1e87500Sdlg bus_dmamap_t dmap = ccb->ccb_dmamap; 678a1e87500Sdlg u_int16_t flags; 679a1e87500Sdlg 680ecfae151Sdlg if (dmap->dm_nsegs > 2) { 681ecfae151Sdlg bus_dmamap_sync(sc->sc_dmat, 682ecfae151Sdlg NVME_DMA_MAP(sc->sc_ccb_prpls), 683ecfae151Sdlg ccb->ccb_prpl_off, 684ecfae151Sdlg sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 685ecfae151Sdlg BUS_DMASYNC_POSTWRITE); 686ecfae151Sdlg } 687ecfae151Sdlg 688a1e87500Sdlg bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 689a1e87500Sdlg ISSET(xs->flags, SCSI_DATA_IN) ? 690a1e87500Sdlg BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 691a1e87500Sdlg 692a1e87500Sdlg bus_dmamap_unload(sc->sc_dmat, dmap); 693a1e87500Sdlg 694a1e87500Sdlg flags = lemtoh16(&cqe->flags); 695a1e87500Sdlg 696a1e87500Sdlg xs->error = (NVME_CQE_SC(flags) == NVME_CQE_SC_SUCCESS) ? 697a1e87500Sdlg XS_NOERROR : XS_DRIVER_STUFFUP; 6988cfbc7f4Sdlg xs->status = SCSI_OK; 699a1e87500Sdlg xs->resid = 0; 700a1e87500Sdlg scsi_done(xs); 701a1e87500Sdlg } 702a1e87500Sdlg 703a1e87500Sdlg void 704689c2c68Sdlg nvme_scsi_sync(struct scsi_xfer *xs) 705689c2c68Sdlg { 706689c2c68Sdlg struct scsi_link *link = xs->sc_link; 707689c2c68Sdlg struct nvme_softc *sc = link->adapter_softc; 708689c2c68Sdlg struct nvme_ccb *ccb = xs->io; 709689c2c68Sdlg 710689c2c68Sdlg ccb->ccb_done = nvme_scsi_sync_done; 711689c2c68Sdlg ccb->ccb_cookie = xs; 712689c2c68Sdlg 713689c2c68Sdlg if (ISSET(xs->flags, SCSI_POLL)) { 714689c2c68Sdlg nvme_poll(sc, sc->sc_q, ccb, nvme_scsi_sync_fill); 715689c2c68Sdlg return; 716689c2c68Sdlg } 717689c2c68Sdlg 718689c2c68Sdlg nvme_q_submit(sc, sc->sc_q, ccb, nvme_scsi_sync_fill); 719689c2c68Sdlg } 720689c2c68Sdlg 721689c2c68Sdlg void 722689c2c68Sdlg nvme_scsi_sync_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 723689c2c68Sdlg { 724689c2c68Sdlg struct nvme_sqe *sqe = slot; 725689c2c68Sdlg struct scsi_xfer *xs = ccb->ccb_cookie; 726689c2c68Sdlg struct scsi_link *link = xs->sc_link; 727689c2c68Sdlg 728689c2c68Sdlg sqe->opcode = NVM_CMD_FLUSH; 729397f5692Skettenis htolem32(&sqe->nsid, link->target); 730689c2c68Sdlg } 731689c2c68Sdlg 732689c2c68Sdlg void 733689c2c68Sdlg nvme_scsi_sync_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 734689c2c68Sdlg struct nvme_cqe *cqe) 735689c2c68Sdlg { 736689c2c68Sdlg struct scsi_xfer *xs = ccb->ccb_cookie; 737689c2c68Sdlg u_int16_t flags; 738689c2c68Sdlg 739689c2c68Sdlg flags = lemtoh16(&cqe->flags); 740689c2c68Sdlg 741689c2c68Sdlg xs->error = (NVME_CQE_SC(flags) == NVME_CQE_SC_SUCCESS) ? 742689c2c68Sdlg XS_NOERROR : XS_DRIVER_STUFFUP; 743689c2c68Sdlg xs->status = SCSI_OK; 744689c2c68Sdlg xs->resid = 0; 745689c2c68Sdlg scsi_done(xs); 746689c2c68Sdlg } 747689c2c68Sdlg 748689c2c68Sdlg void 7499856392eSdlg nvme_scsi_inq(struct scsi_xfer *xs) 7509856392eSdlg { 7519856392eSdlg struct scsi_inquiry *inq = (struct scsi_inquiry *)xs->cmd; 7529856392eSdlg 7539856392eSdlg if (!ISSET(inq->flags, SI_EVPD)) { 7549856392eSdlg nvme_scsi_inquiry(xs); 7559856392eSdlg return; 7569856392eSdlg } 7579856392eSdlg 7589856392eSdlg switch (inq->pagecode) { 7599856392eSdlg default: 7609856392eSdlg /* printf("%s: %d\n", __func__, inq->pagecode); */ 7619856392eSdlg break; 7629856392eSdlg } 7639856392eSdlg 7649856392eSdlg xs->error = XS_DRIVER_STUFFUP; 7659856392eSdlg scsi_done(xs); 7669856392eSdlg } 7679856392eSdlg 7689856392eSdlg void 7699856392eSdlg nvme_scsi_inquiry(struct scsi_xfer *xs) 7709856392eSdlg { 7719856392eSdlg struct scsi_inquiry_data inq; 7729856392eSdlg struct scsi_link *link = xs->sc_link; 7739856392eSdlg struct nvme_softc *sc = link->adapter_softc; 7749856392eSdlg struct nvm_identify_namespace *ns; 7759856392eSdlg 7769856392eSdlg ns = sc->sc_namespaces[link->target].ident; 7779856392eSdlg 7789856392eSdlg memset(&inq, 0, sizeof(inq)); 7799856392eSdlg 7809856392eSdlg inq.device = T_DIRECT; 7819856392eSdlg inq.version = 0x06; /* SPC-4 */ 7829856392eSdlg inq.response_format = 2; 7839856392eSdlg inq.additional_length = 32; 7849856392eSdlg inq.flags |= SID_CmdQue; 7859856392eSdlg memcpy(inq.vendor, "NVMe ", sizeof(inq.vendor)); 7869856392eSdlg memcpy(inq.product, sc->sc_identify.mn, sizeof(inq.product)); 7879856392eSdlg memcpy(inq.revision, sc->sc_identify.fr, sizeof(inq.revision)); 7889856392eSdlg 7899856392eSdlg memcpy(xs->data, &inq, MIN(sizeof(inq), xs->datalen)); 7909856392eSdlg 7919856392eSdlg xs->error = XS_NOERROR; 7929856392eSdlg scsi_done(xs); 7939856392eSdlg } 7949856392eSdlg 7959856392eSdlg void 796c947e044Sdlg nvme_scsi_capacity16(struct scsi_xfer *xs) 797c947e044Sdlg { 798c947e044Sdlg struct scsi_read_cap_data_16 rcd; 799c947e044Sdlg struct scsi_link *link = xs->sc_link; 800c947e044Sdlg struct nvme_softc *sc = link->adapter_softc; 801c947e044Sdlg struct nvm_identify_namespace *ns; 802c947e044Sdlg struct nvm_namespace_format *f; 803c947e044Sdlg u_int64_t nsze; 804c947e044Sdlg u_int16_t tpe = READ_CAP_16_TPE; 805c947e044Sdlg 806c947e044Sdlg ns = sc->sc_namespaces[link->target].ident; 807c947e044Sdlg 808c947e044Sdlg if (xs->cmdlen != sizeof(struct scsi_read_capacity_16)) { 809c947e044Sdlg xs->error = XS_DRIVER_STUFFUP; 810c947e044Sdlg scsi_done(xs); 811c947e044Sdlg return; 812c947e044Sdlg } 813c947e044Sdlg 814dc8298f6Sjcs /* sd_read_cap_16() will add one */ 815dc8298f6Sjcs nsze = lemtoh64(&ns->nsze) - 1; 816c947e044Sdlg f = &ns->lbaf[NVME_ID_NS_FLBAS(ns->flbas)]; 817c947e044Sdlg 818c947e044Sdlg memset(&rcd, 0, sizeof(rcd)); 819c947e044Sdlg _lto8b(nsze, rcd.addr); 820c947e044Sdlg _lto4b(1 << f->lbads, rcd.length); 821c947e044Sdlg _lto2b(tpe, rcd.lowest_aligned); 822c947e044Sdlg 823c947e044Sdlg memcpy(xs->data, &rcd, MIN(sizeof(rcd), xs->datalen)); 824c947e044Sdlg 825c947e044Sdlg xs->error = XS_NOERROR; 826c947e044Sdlg scsi_done(xs); 827c947e044Sdlg } 828c947e044Sdlg 829c947e044Sdlg void 830c947e044Sdlg nvme_scsi_capacity(struct scsi_xfer *xs) 831c947e044Sdlg { 832c947e044Sdlg struct scsi_read_cap_data rcd; 833c947e044Sdlg struct scsi_link *link = xs->sc_link; 834c947e044Sdlg struct nvme_softc *sc = link->adapter_softc; 835c947e044Sdlg struct nvm_identify_namespace *ns; 836c947e044Sdlg struct nvm_namespace_format *f; 837c947e044Sdlg u_int64_t nsze; 838c947e044Sdlg 839c947e044Sdlg ns = sc->sc_namespaces[link->target].ident; 840c947e044Sdlg 841c947e044Sdlg if (xs->cmdlen != sizeof(struct scsi_read_capacity)) { 842c947e044Sdlg xs->error = XS_DRIVER_STUFFUP; 843c947e044Sdlg scsi_done(xs); 844c947e044Sdlg return; 845c947e044Sdlg } 846c947e044Sdlg 847dc8298f6Sjcs /* sd_read_cap_10() will add one */ 848dc8298f6Sjcs nsze = lemtoh64(&ns->nsze) - 1; 849c947e044Sdlg if (nsze > 0xffffffff) 850c947e044Sdlg nsze = 0xffffffff; 851c947e044Sdlg 852c947e044Sdlg f = &ns->lbaf[NVME_ID_NS_FLBAS(ns->flbas)]; 853c947e044Sdlg 854c947e044Sdlg memset(&rcd, 0, sizeof(rcd)); 855c947e044Sdlg _lto4b(nsze, rcd.addr); 856c947e044Sdlg _lto4b(1 << f->lbads, rcd.length); 857c947e044Sdlg 858c947e044Sdlg memcpy(xs->data, &rcd, MIN(sizeof(rcd), xs->datalen)); 859c947e044Sdlg 860c947e044Sdlg xs->error = XS_NOERROR; 861c947e044Sdlg scsi_done(xs); 862c947e044Sdlg } 863c947e044Sdlg 864c947e044Sdlg void 86511624e4cSdlg nvme_scsi_free(struct scsi_link *link) 86611624e4cSdlg { 86765dd51f8Sdlg struct nvme_softc *sc = link->adapter_softc; 86865dd51f8Sdlg struct nvm_identify_namespace *identify; 86911624e4cSdlg 87065dd51f8Sdlg identify = sc->sc_namespaces[link->target].ident; 87165dd51f8Sdlg sc->sc_namespaces[link->target].ident = NULL; 87265dd51f8Sdlg 87365dd51f8Sdlg free(identify, M_DEVBUF, sizeof(*identify)); 87411624e4cSdlg } 87511624e4cSdlg 876448b3c09Sdlg void 877448b3c09Sdlg nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 878448b3c09Sdlg void (*fill)(struct nvme_softc *, struct nvme_ccb *, void *)) 879448b3c09Sdlg { 880448b3c09Sdlg struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 881448b3c09Sdlg u_int32_t tail; 882448b3c09Sdlg 883448b3c09Sdlg mtx_enter(&q->q_sq_mtx); 884448b3c09Sdlg tail = q->q_sq_tail; 885448b3c09Sdlg if (++q->q_sq_tail >= q->q_entries) 886448b3c09Sdlg q->q_sq_tail = 0; 887448b3c09Sdlg 888448b3c09Sdlg sqe += tail; 889448b3c09Sdlg 890448b3c09Sdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 891448b3c09Sdlg sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 892448b3c09Sdlg memset(sqe, 0, sizeof(*sqe)); 893448b3c09Sdlg (*fill)(sc, ccb, sqe); 89424aede5fSdlg sqe->cid = ccb->ccb_id; 895448b3c09Sdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 896448b3c09Sdlg sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 897448b3c09Sdlg 898448b3c09Sdlg nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 899448b3c09Sdlg mtx_leave(&q->q_sq_mtx); 900448b3c09Sdlg } 901448b3c09Sdlg 902448b3c09Sdlg struct nvme_poll_state { 903448b3c09Sdlg struct nvme_sqe s; 904448b3c09Sdlg struct nvme_cqe c; 905448b3c09Sdlg }; 906448b3c09Sdlg 907448b3c09Sdlg int 908448b3c09Sdlg nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 909448b3c09Sdlg void (*fill)(struct nvme_softc *, struct nvme_ccb *, void *)) 910448b3c09Sdlg { 911448b3c09Sdlg struct nvme_poll_state state; 912448b3c09Sdlg void (*done)(struct nvme_softc *, struct nvme_ccb *, struct nvme_cqe *); 913448b3c09Sdlg void *cookie; 9149f81ea3bSdlg u_int16_t flags; 915448b3c09Sdlg 916448b3c09Sdlg memset(&state, 0, sizeof(state)); 917448b3c09Sdlg (*fill)(sc, ccb, &state.s); 918448b3c09Sdlg 919448b3c09Sdlg done = ccb->ccb_done; 920448b3c09Sdlg cookie = ccb->ccb_cookie; 921448b3c09Sdlg 922448b3c09Sdlg ccb->ccb_done = nvme_poll_done; 923448b3c09Sdlg ccb->ccb_cookie = &state; 924448b3c09Sdlg 925448b3c09Sdlg nvme_q_submit(sc, q, ccb, nvme_poll_fill); 926448b3c09Sdlg while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 9275fac9627Sdlg if (nvme_q_complete(sc, q) == 0) 928448b3c09Sdlg delay(10); 929448b3c09Sdlg 930448b3c09Sdlg /* XXX no timeout? */ 931448b3c09Sdlg } 932448b3c09Sdlg 933448b3c09Sdlg ccb->ccb_cookie = cookie; 934448b3c09Sdlg done(sc, ccb, &state.c); 935448b3c09Sdlg 9369f81ea3bSdlg flags = lemtoh16(&state.c.flags); 9379f81ea3bSdlg 938c05818fdSdlg return (flags & ~NVME_CQE_PHASE); 939448b3c09Sdlg } 940448b3c09Sdlg 941448b3c09Sdlg void 942448b3c09Sdlg nvme_poll_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 943448b3c09Sdlg { 944448b3c09Sdlg struct nvme_sqe *sqe = slot; 945448b3c09Sdlg struct nvme_poll_state *state = ccb->ccb_cookie; 946448b3c09Sdlg 947448b3c09Sdlg *sqe = state->s; 948448b3c09Sdlg } 949448b3c09Sdlg 950448b3c09Sdlg void 951448b3c09Sdlg nvme_poll_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 952448b3c09Sdlg struct nvme_cqe *cqe) 953448b3c09Sdlg { 954448b3c09Sdlg struct nvme_poll_state *state = ccb->ccb_cookie; 955448b3c09Sdlg 956448b3c09Sdlg state->c = *cqe; 957300a229bSyasuoka SET(state->c.flags, htole16(NVME_CQE_PHASE)); 958448b3c09Sdlg } 959448b3c09Sdlg 9609f81ea3bSdlg void 961ad9e5681Sdlg nvme_sqe_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 962ad9e5681Sdlg { 963ad9e5681Sdlg struct nvme_sqe *src = ccb->ccb_cookie; 964ad9e5681Sdlg struct nvme_sqe *dst = slot; 965ad9e5681Sdlg 966ad9e5681Sdlg *dst = *src; 967ad9e5681Sdlg } 968ad9e5681Sdlg 969ad9e5681Sdlg void 9709f81ea3bSdlg nvme_empty_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 9719f81ea3bSdlg struct nvme_cqe *cqe) 9729f81ea3bSdlg { 9739f81ea3bSdlg } 9749f81ea3bSdlg 975448b3c09Sdlg int 976448b3c09Sdlg nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 977448b3c09Sdlg { 978448b3c09Sdlg struct nvme_ccb *ccb; 979448b3c09Sdlg struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 980448b3c09Sdlg u_int32_t head; 981448b3c09Sdlg u_int16_t flags; 982448b3c09Sdlg int rv = 0; 983448b3c09Sdlg 984448b3c09Sdlg if (!mtx_enter_try(&q->q_cq_mtx)) 985448b3c09Sdlg return (-1); 986448b3c09Sdlg 987448b3c09Sdlg head = q->q_cq_head; 9880ee935ccSdlg 9890ee935ccSdlg nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 990448b3c09Sdlg for (;;) { 991448b3c09Sdlg cqe = &ring[head]; 992448b3c09Sdlg flags = lemtoh16(&cqe->flags); 993448b3c09Sdlg if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 994448b3c09Sdlg break; 995448b3c09Sdlg 996448b3c09Sdlg ccb = &sc->sc_ccbs[cqe->cid]; 997448b3c09Sdlg ccb->ccb_done(sc, ccb, cqe); 998448b3c09Sdlg 999448b3c09Sdlg if (++head >= q->q_entries) { 1000448b3c09Sdlg head = 0; 1001448b3c09Sdlg q->q_cq_phase ^= NVME_CQE_PHASE; 1002448b3c09Sdlg } 1003448b3c09Sdlg 1004448b3c09Sdlg rv = 1; 1005448b3c09Sdlg } 10060ee935ccSdlg nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1007448b3c09Sdlg 1008448b3c09Sdlg if (rv) 100920386a6cSdlg nvme_write4(sc, q->q_cqhdbl, q->q_cq_head = head); 1010448b3c09Sdlg mtx_leave(&q->q_cq_mtx); 1011448b3c09Sdlg 1012448b3c09Sdlg return (rv); 1013448b3c09Sdlg } 1014448b3c09Sdlg 1015448b3c09Sdlg int 101688aa9192Sdlg nvme_identify(struct nvme_softc *sc, u_int mps) 1017448b3c09Sdlg { 10189f81ea3bSdlg char sn[41], mn[81], fr[17]; 10199f81ea3bSdlg struct nvm_identify_controller *identify; 1020448b3c09Sdlg struct nvme_dmamem *mem; 10219f81ea3bSdlg struct nvme_ccb *ccb; 102288aa9192Sdlg u_int mdts; 1023448b3c09Sdlg int rv = 1; 1024448b3c09Sdlg 1025448b3c09Sdlg ccb = nvme_ccb_get(sc); 1026448b3c09Sdlg if (ccb == NULL) 1027448b3c09Sdlg panic("nvme_identify: nvme_ccb_get returned NULL"); 1028448b3c09Sdlg 10299f81ea3bSdlg mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 1030448b3c09Sdlg if (mem == NULL) 1031448b3c09Sdlg return (1); 1032448b3c09Sdlg 10339f81ea3bSdlg ccb->ccb_done = nvme_empty_done; 1034448b3c09Sdlg ccb->ccb_cookie = mem; 1035448b3c09Sdlg 10360ee935ccSdlg nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 1037448b3c09Sdlg rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify); 10380ee935ccSdlg nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1039448b3c09Sdlg 1040b62ffe02Sdlg nvme_ccb_put(sc, ccb); 1041b62ffe02Sdlg 10429f81ea3bSdlg if (rv != 0) 10439f81ea3bSdlg goto done; 10449f81ea3bSdlg 104517756a2bSdlg identify = NVME_DMA_KVA(mem); 104617756a2bSdlg 10479f81ea3bSdlg scsi_strvis(sn, identify->sn, sizeof(identify->sn)); 10489f81ea3bSdlg scsi_strvis(mn, identify->mn, sizeof(identify->mn)); 10499f81ea3bSdlg scsi_strvis(fr, identify->fr, sizeof(identify->fr)); 10509f81ea3bSdlg 10519f81ea3bSdlg printf("%s: %s, firmware %s, serial %s\n", DEVNAME(sc), mn, fr, sn); 10529f81ea3bSdlg 105388aa9192Sdlg if (identify->mdts > 0) { 105488aa9192Sdlg mdts = (1 << identify->mdts) * (1 << mps); 105588aa9192Sdlg if (mdts < sc->sc_mdts) 105688aa9192Sdlg sc->sc_mdts = mdts; 105788aa9192Sdlg } 105888aa9192Sdlg 105917756a2bSdlg sc->sc_nn = lemtoh32(&identify->nn); 106017756a2bSdlg 1061e276f6b1Sjcs /* 1062e276f6b1Sjcs * At least one Apple NVMe device presents a second, bogus disk that is 1063e276f6b1Sjcs * inaccessible, so cap targets at 1. 1064e276f6b1Sjcs * 1065397f5692Skettenis * sd1 at scsibus1 targ 2 lun 0: <NVMe, APPLE SSD AP0512, 16.1> [..] 1066e276f6b1Sjcs * sd1: 0MB, 4096 bytes/sector, 2 sectors 1067e276f6b1Sjcs */ 1068e276f6b1Sjcs if (sc->sc_nn > 1 && 1069e276f6b1Sjcs mn[0] == 'A' && mn[1] == 'P' && mn[2] == 'P' && mn[3] == 'L' && 1070e276f6b1Sjcs mn[4] == 'E') 1071e276f6b1Sjcs sc->sc_nn = 1; 1072e276f6b1Sjcs 107317756a2bSdlg memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 107417756a2bSdlg 10759f81ea3bSdlg done: 1076448b3c09Sdlg nvme_dmamem_free(sc, mem); 1077448b3c09Sdlg 1078448b3c09Sdlg return (rv); 1079448b3c09Sdlg } 1080448b3c09Sdlg 1081cd15a86fSdlg int 1082cd15a86fSdlg nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1083cd15a86fSdlg { 1084cd15a86fSdlg struct nvme_sqe_q sqe; 1085cd15a86fSdlg struct nvme_ccb *ccb; 1086cd15a86fSdlg int rv; 1087cd15a86fSdlg 1088cd15a86fSdlg ccb = scsi_io_get(&sc->sc_iopool, 0); 1089cd15a86fSdlg KASSERT(ccb != NULL); 1090cd15a86fSdlg 1091cd15a86fSdlg ccb->ccb_done = nvme_empty_done; 1092cd15a86fSdlg ccb->ccb_cookie = &sqe; 1093cd15a86fSdlg 1094cd15a86fSdlg memset(&sqe, 0, sizeof(sqe)); 1095cd15a86fSdlg sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1096cd15a86fSdlg htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1097cd15a86fSdlg htolem16(&sqe.qsize, q->q_entries - 1); 1098cd15a86fSdlg htolem16(&sqe.qid, q->q_id); 1099cd15a86fSdlg sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1100cd15a86fSdlg 1101cd15a86fSdlg rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1102cd15a86fSdlg if (rv != 0) 1103cd15a86fSdlg goto fail; 1104cd15a86fSdlg 1105cd15a86fSdlg ccb->ccb_done = nvme_empty_done; 1106cd15a86fSdlg ccb->ccb_cookie = &sqe; 1107cd15a86fSdlg 1108cd15a86fSdlg memset(&sqe, 0, sizeof(sqe)); 1109cd15a86fSdlg sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1110cd15a86fSdlg htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1111cd15a86fSdlg htolem16(&sqe.qsize, q->q_entries - 1); 1112cd15a86fSdlg htolem16(&sqe.qid, q->q_id); 1113cd15a86fSdlg htolem16(&sqe.cqid, q->q_id); 1114cd15a86fSdlg sqe.qflags = NVM_SQE_Q_PC; 1115cd15a86fSdlg 1116cd15a86fSdlg rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1117cd15a86fSdlg if (rv != 0) 1118cd15a86fSdlg goto fail; 1119cd15a86fSdlg 1120cd15a86fSdlg fail: 1121cd15a86fSdlg scsi_io_put(&sc->sc_iopool, ccb); 1122cd15a86fSdlg return (rv); 1123cd15a86fSdlg } 1124cd15a86fSdlg 11255313ab17Sdlg int 11265313ab17Sdlg nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 11275313ab17Sdlg { 11285313ab17Sdlg struct nvme_sqe_q sqe; 11295313ab17Sdlg struct nvme_ccb *ccb; 11305313ab17Sdlg int rv; 11315313ab17Sdlg 11325313ab17Sdlg ccb = scsi_io_get(&sc->sc_iopool, 0); 11335313ab17Sdlg KASSERT(ccb != NULL); 11345313ab17Sdlg 11355313ab17Sdlg ccb->ccb_done = nvme_empty_done; 11365313ab17Sdlg ccb->ccb_cookie = &sqe; 11375313ab17Sdlg 11385313ab17Sdlg memset(&sqe, 0, sizeof(sqe)); 11395313ab17Sdlg sqe.opcode = NVM_ADMIN_DEL_IOSQ; 11405313ab17Sdlg htolem16(&sqe.qid, q->q_id); 11415313ab17Sdlg 11425313ab17Sdlg rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 11435313ab17Sdlg if (rv != 0) 11445313ab17Sdlg goto fail; 11455313ab17Sdlg 11465313ab17Sdlg ccb->ccb_done = nvme_empty_done; 11475313ab17Sdlg ccb->ccb_cookie = &sqe; 11485313ab17Sdlg 11495313ab17Sdlg memset(&sqe, 0, sizeof(sqe)); 11505313ab17Sdlg sqe.opcode = NVM_ADMIN_DEL_IOCQ; 11515313ab17Sdlg htolem16(&sqe.qid, q->q_id); 11525313ab17Sdlg 11535313ab17Sdlg rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 11545313ab17Sdlg if (rv != 0) 11555313ab17Sdlg goto fail; 11565313ab17Sdlg 1157aa5ddcf9Ssf nvme_q_free(sc, q); 1158aa5ddcf9Ssf 11595313ab17Sdlg fail: 11605313ab17Sdlg scsi_io_put(&sc->sc_iopool, ccb); 11615313ab17Sdlg return (rv); 11625313ab17Sdlg 11635313ab17Sdlg } 11645313ab17Sdlg 1165448b3c09Sdlg void 1166448b3c09Sdlg nvme_fill_identify(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 1167448b3c09Sdlg { 1168448b3c09Sdlg struct nvme_sqe *sqe = slot; 1169448b3c09Sdlg struct nvme_dmamem *mem = ccb->ccb_cookie; 1170448b3c09Sdlg 1171448b3c09Sdlg sqe->opcode = NVM_ADMIN_IDENTIFY; 1172448b3c09Sdlg htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1173448b3c09Sdlg htolem32(&sqe->cdw10, 1); 1174448b3c09Sdlg } 1175448b3c09Sdlg 1176448b3c09Sdlg int 1177448b3c09Sdlg nvme_ccbs_alloc(struct nvme_softc *sc, u_int nccbs) 1178448b3c09Sdlg { 1179448b3c09Sdlg struct nvme_ccb *ccb; 1180b1699ccdSdlg bus_addr_t off; 1181b1699ccdSdlg u_int64_t *prpl; 1182448b3c09Sdlg u_int i; 1183448b3c09Sdlg 11849f6fb5c7Sderaadt sc->sc_ccbs = mallocarray(nccbs, sizeof(*ccb), M_DEVBUF, 1185448b3c09Sdlg M_WAITOK | M_CANFAIL); 1186448b3c09Sdlg if (sc->sc_ccbs == NULL) 1187448b3c09Sdlg return (1); 1188448b3c09Sdlg 1189b1699ccdSdlg sc->sc_ccb_prpls = nvme_dmamem_alloc(sc, 1190b1699ccdSdlg sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1191b1699ccdSdlg 1192b1699ccdSdlg prpl = NVME_DMA_KVA(sc->sc_ccb_prpls); 1193b1699ccdSdlg off = 0; 1194b1699ccdSdlg 1195448b3c09Sdlg for (i = 0; i < nccbs; i++) { 1196448b3c09Sdlg ccb = &sc->sc_ccbs[i]; 1197448b3c09Sdlg 1198b1699ccdSdlg if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1199b1699ccdSdlg sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1200856ec1cfSdlg sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1201448b3c09Sdlg &ccb->ccb_dmamap) != 0) 1202448b3c09Sdlg goto free_maps; 1203448b3c09Sdlg 1204448b3c09Sdlg ccb->ccb_id = i; 1205b1699ccdSdlg ccb->ccb_prpl = prpl; 1206b1699ccdSdlg ccb->ccb_prpl_off = off; 1207b1699ccdSdlg ccb->ccb_prpl_dva = NVME_DMA_DVA(sc->sc_ccb_prpls) + off; 1208b1699ccdSdlg 1209448b3c09Sdlg SIMPLEQ_INSERT_TAIL(&sc->sc_ccb_list, ccb, ccb_entry); 1210b1699ccdSdlg 1211b1699ccdSdlg prpl += sc->sc_max_sgl; 1212b1699ccdSdlg off += sizeof(*prpl) * sc->sc_max_sgl; 1213448b3c09Sdlg } 1214448b3c09Sdlg 1215448b3c09Sdlg return (0); 1216448b3c09Sdlg 1217448b3c09Sdlg free_maps: 1218a1141c40Stedu nvme_ccbs_free(sc, nccbs); 1219448b3c09Sdlg return (1); 1220448b3c09Sdlg } 1221448b3c09Sdlg 1222448b3c09Sdlg void * 1223448b3c09Sdlg nvme_ccb_get(void *cookie) 1224448b3c09Sdlg { 1225448b3c09Sdlg struct nvme_softc *sc = cookie; 1226448b3c09Sdlg struct nvme_ccb *ccb; 1227448b3c09Sdlg 1228448b3c09Sdlg mtx_enter(&sc->sc_ccb_mtx); 1229448b3c09Sdlg ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list); 1230448b3c09Sdlg if (ccb != NULL) 1231448b3c09Sdlg SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1232448b3c09Sdlg mtx_leave(&sc->sc_ccb_mtx); 1233448b3c09Sdlg 1234448b3c09Sdlg return (ccb); 1235448b3c09Sdlg } 1236448b3c09Sdlg 1237448b3c09Sdlg void 1238448b3c09Sdlg nvme_ccb_put(void *cookie, void *io) 1239448b3c09Sdlg { 1240448b3c09Sdlg struct nvme_softc *sc = cookie; 1241448b3c09Sdlg struct nvme_ccb *ccb = io; 1242448b3c09Sdlg 1243448b3c09Sdlg mtx_enter(&sc->sc_ccb_mtx); 1244448b3c09Sdlg SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry); 1245448b3c09Sdlg mtx_leave(&sc->sc_ccb_mtx); 1246448b3c09Sdlg } 1247448b3c09Sdlg 1248448b3c09Sdlg void 1249a1141c40Stedu nvme_ccbs_free(struct nvme_softc *sc, unsigned int nccbs) 1250448b3c09Sdlg { 1251448b3c09Sdlg struct nvme_ccb *ccb; 1252448b3c09Sdlg 1253448b3c09Sdlg while ((ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list)) != NULL) { 1254448b3c09Sdlg SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1255448b3c09Sdlg bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1256448b3c09Sdlg } 1257448b3c09Sdlg 1258b1699ccdSdlg nvme_dmamem_free(sc, sc->sc_ccb_prpls); 1259a1141c40Stedu free(sc->sc_ccbs, M_DEVBUF, nccbs * sizeof(*ccb)); 1260448b3c09Sdlg } 1261448b3c09Sdlg 1262282c0692Sdlg struct nvme_queue * 126359968badSdlg nvme_q_alloc(struct nvme_softc *sc, u_int16_t id, u_int entries, u_int dstrd) 1264282c0692Sdlg { 1265282c0692Sdlg struct nvme_queue *q; 1266282c0692Sdlg 1267282c0692Sdlg q = malloc(sizeof(*q), M_DEVBUF, M_WAITOK | M_CANFAIL); 1268282c0692Sdlg if (q == NULL) 1269282c0692Sdlg return (NULL); 1270282c0692Sdlg 1271448b3c09Sdlg q->q_sq_dmamem = nvme_dmamem_alloc(sc, 127269136b8eSdlg sizeof(struct nvme_sqe) * entries); 1273282c0692Sdlg if (q->q_sq_dmamem == NULL) 1274282c0692Sdlg goto free; 1275282c0692Sdlg 1276448b3c09Sdlg q->q_cq_dmamem = nvme_dmamem_alloc(sc, 127769136b8eSdlg sizeof(struct nvme_cqe) * entries); 1278638d48daSdlg if (q->q_cq_dmamem == NULL) 1279282c0692Sdlg goto free_sq; 1280282c0692Sdlg 1281448b3c09Sdlg memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1282448b3c09Sdlg memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1283448b3c09Sdlg 1284448b3c09Sdlg mtx_init(&q->q_sq_mtx, IPL_BIO); 1285448b3c09Sdlg mtx_init(&q->q_cq_mtx, IPL_BIO); 128659968badSdlg q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 128759968badSdlg q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1288aa5ddcf9Ssf 12898f4e0057Sdlg q->q_id = id; 1290282c0692Sdlg q->q_entries = entries; 1291448b3c09Sdlg q->q_sq_tail = 0; 1292448b3c09Sdlg q->q_cq_head = 0; 1293448b3c09Sdlg q->q_cq_phase = NVME_CQE_PHASE; 1294448b3c09Sdlg 12950ee935ccSdlg nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 12960ee935ccSdlg nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1297282c0692Sdlg 1298282c0692Sdlg return (q); 1299282c0692Sdlg 1300282c0692Sdlg free_sq: 1301282c0692Sdlg nvme_dmamem_free(sc, q->q_sq_dmamem); 1302282c0692Sdlg free: 1303234dfda1Sderaadt free(q, M_DEVBUF, sizeof *q); 1304282c0692Sdlg 1305282c0692Sdlg return (NULL); 1306282c0692Sdlg } 1307282c0692Sdlg 1308aa5ddcf9Ssf int 1309aa5ddcf9Ssf nvme_q_reset(struct nvme_softc *sc, struct nvme_queue *q) 1310aa5ddcf9Ssf { 1311aa5ddcf9Ssf memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1312aa5ddcf9Ssf memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1313aa5ddcf9Ssf 1314aa5ddcf9Ssf q->q_sqtdbl = NVME_SQTDBL(q->q_id, sc->sc_dstrd); 1315aa5ddcf9Ssf q->q_cqhdbl = NVME_CQHDBL(q->q_id, sc->sc_dstrd); 1316aa5ddcf9Ssf 1317aa5ddcf9Ssf q->q_sq_tail = 0; 1318aa5ddcf9Ssf q->q_cq_head = 0; 1319aa5ddcf9Ssf q->q_cq_phase = NVME_CQE_PHASE; 1320aa5ddcf9Ssf 1321aa5ddcf9Ssf nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1322aa5ddcf9Ssf nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1323aa5ddcf9Ssf 1324aa5ddcf9Ssf return (0); 1325aa5ddcf9Ssf } 1326aa5ddcf9Ssf 1327282c0692Sdlg void 1328448b3c09Sdlg nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1329282c0692Sdlg { 13300ee935ccSdlg nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 13310ee935ccSdlg nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1332282c0692Sdlg nvme_dmamem_free(sc, q->q_cq_dmamem); 1333282c0692Sdlg nvme_dmamem_free(sc, q->q_sq_dmamem); 1334234dfda1Sderaadt free(q, M_DEVBUF, sizeof *q); 1335282c0692Sdlg } 1336282c0692Sdlg 1337282c0692Sdlg int 1338282c0692Sdlg nvme_intr(void *xsc) 1339282c0692Sdlg { 1340448b3c09Sdlg struct nvme_softc *sc = xsc; 134187cbc23fSdlg int rv = 0; 1342448b3c09Sdlg 134387cbc23fSdlg if (nvme_q_complete(sc, sc->sc_q)) 134487cbc23fSdlg rv = 1; 134587cbc23fSdlg if (nvme_q_complete(sc, sc->sc_admin_q)) 134687cbc23fSdlg rv = 1; 134787cbc23fSdlg 134887cbc23fSdlg return (rv); 1349282c0692Sdlg } 1350282c0692Sdlg 1351eb77e636Sdlg int 1352eb77e636Sdlg nvme_intr_intx(void *xsc) 1353eb77e636Sdlg { 1354eb77e636Sdlg struct nvme_softc *sc = xsc; 1355eb77e636Sdlg int rv; 1356eb77e636Sdlg 1357eb77e636Sdlg nvme_write4(sc, NVME_INTMS, 1); 1358eb77e636Sdlg rv = nvme_intr(sc); 1359eb77e636Sdlg nvme_write4(sc, NVME_INTMC, 1); 1360eb77e636Sdlg 1361eb77e636Sdlg return (rv); 1362eb77e636Sdlg } 1363eb77e636Sdlg 1364282c0692Sdlg struct nvme_dmamem * 1365282c0692Sdlg nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1366282c0692Sdlg { 1367282c0692Sdlg struct nvme_dmamem *ndm; 1368282c0692Sdlg int nsegs; 1369282c0692Sdlg 137072f0e87bSdlg ndm = malloc(sizeof(*ndm), M_DEVBUF, M_WAITOK | M_ZERO); 1371282c0692Sdlg if (ndm == NULL) 1372282c0692Sdlg return (NULL); 1373282c0692Sdlg 1374282c0692Sdlg ndm->ndm_size = size; 1375282c0692Sdlg 1376282c0692Sdlg if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 137772f0e87bSdlg BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1378282c0692Sdlg goto ndmfree; 1379282c0692Sdlg 1380448b3c09Sdlg if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 138172f0e87bSdlg 1, &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 1382282c0692Sdlg goto destroy; 1383282c0692Sdlg 1384282c0692Sdlg if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 138572f0e87bSdlg &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1386282c0692Sdlg goto free; 1387282c0692Sdlg 1388282c0692Sdlg if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 138972f0e87bSdlg NULL, BUS_DMA_WAITOK) != 0) 1390282c0692Sdlg goto unmap; 1391282c0692Sdlg 1392282c0692Sdlg return (ndm); 1393282c0692Sdlg 1394282c0692Sdlg unmap: 1395282c0692Sdlg bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1396282c0692Sdlg free: 1397282c0692Sdlg bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1398282c0692Sdlg destroy: 1399282c0692Sdlg bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1400282c0692Sdlg ndmfree: 1401234dfda1Sderaadt free(ndm, M_DEVBUF, sizeof *ndm); 1402282c0692Sdlg 1403282c0692Sdlg return (NULL); 1404282c0692Sdlg } 1405282c0692Sdlg 1406282c0692Sdlg void 14070ee935ccSdlg nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 14080ee935ccSdlg { 14090ee935ccSdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 14100ee935ccSdlg 0, NVME_DMA_LEN(mem), ops); 14110ee935ccSdlg } 14120ee935ccSdlg 14130ee935ccSdlg void 1414282c0692Sdlg nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1415282c0692Sdlg { 1416282c0692Sdlg bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1417282c0692Sdlg bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1418282c0692Sdlg bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1419282c0692Sdlg bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1420234dfda1Sderaadt free(ndm, M_DEVBUF, sizeof *ndm); 1421282c0692Sdlg } 1422282c0692Sdlg 142303d86467Sjmatthew #ifdef HIBERNATE 142403d86467Sjmatthew 142503d86467Sjmatthew int 142603d86467Sjmatthew nvme_hibernate_admin_cmd(struct nvme_softc *sc, struct nvme_sqe *sqe, 142703d86467Sjmatthew struct nvme_cqe *cqe, int cid) 142803d86467Sjmatthew { 142903d86467Sjmatthew struct nvme_sqe *asqe = NVME_DMA_KVA(sc->sc_admin_q->q_sq_dmamem); 143003d86467Sjmatthew struct nvme_cqe *acqe = NVME_DMA_KVA(sc->sc_admin_q->q_cq_dmamem); 143103d86467Sjmatthew struct nvme_queue *q = sc->sc_admin_q; 143203d86467Sjmatthew int tail; 143303d86467Sjmatthew u_int16_t flags; 143403d86467Sjmatthew 143503d86467Sjmatthew /* submit command */ 143603d86467Sjmatthew tail = q->q_sq_tail; 143703d86467Sjmatthew if (++q->q_sq_tail >= q->q_entries) 143803d86467Sjmatthew q->q_sq_tail = 0; 143903d86467Sjmatthew 144003d86467Sjmatthew asqe += tail; 144103d86467Sjmatthew bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 144203d86467Sjmatthew sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 144303d86467Sjmatthew *asqe = *sqe; 144403d86467Sjmatthew asqe->cid = cid; 144503d86467Sjmatthew bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 144603d86467Sjmatthew sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 144703d86467Sjmatthew 144803d86467Sjmatthew nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 144903d86467Sjmatthew 145003d86467Sjmatthew /* wait for completion */ 145103d86467Sjmatthew acqe += q->q_cq_head; 145203d86467Sjmatthew for (;;) { 145303d86467Sjmatthew nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 145403d86467Sjmatthew flags = lemtoh16(&acqe->flags); 145503d86467Sjmatthew if ((flags & NVME_CQE_PHASE) == q->q_cq_phase) 145603d86467Sjmatthew break; 145703d86467Sjmatthew 145803d86467Sjmatthew delay(10); 145903d86467Sjmatthew } 146003d86467Sjmatthew 146103d86467Sjmatthew if (++q->q_cq_head >= q->q_entries) { 146203d86467Sjmatthew q->q_cq_head = 0; 146303d86467Sjmatthew q->q_cq_phase ^= NVME_CQE_PHASE; 146403d86467Sjmatthew } 146503d86467Sjmatthew nvme_write4(sc, q->q_cqhdbl, q->q_cq_head); 146603d86467Sjmatthew if ((NVME_CQE_SC(flags) != NVME_CQE_SC_SUCCESS) || (acqe->cid != cid)) 146703d86467Sjmatthew return (EIO); 146803d86467Sjmatthew 146903d86467Sjmatthew return (0); 147003d86467Sjmatthew } 147103d86467Sjmatthew 147203d86467Sjmatthew int 147303d86467Sjmatthew nvme_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, 147403d86467Sjmatthew int op, void *page) 147503d86467Sjmatthew { 147603d86467Sjmatthew struct nvme_hibernate_page { 147703d86467Sjmatthew u_int64_t prpl[MAXPHYS / PAGE_SIZE]; 147803d86467Sjmatthew 147903d86467Sjmatthew struct nvme_softc *sc; 148003d86467Sjmatthew int nsid; 148103d86467Sjmatthew int sq_tail; 148203d86467Sjmatthew int cq_head; 148303d86467Sjmatthew int cqe_phase; 148403d86467Sjmatthew 148503d86467Sjmatthew daddr_t poffset; 148603d86467Sjmatthew size_t psize; 148703d86467Sjmatthew } *my = page; 148803d86467Sjmatthew struct nvme_sqe_io *isqe; 148903d86467Sjmatthew struct nvme_cqe *icqe; 149003d86467Sjmatthew paddr_t data_phys, page_phys; 149103d86467Sjmatthew u_int64_t data_bus_phys, page_bus_phys; 149203d86467Sjmatthew u_int16_t flags; 149303d86467Sjmatthew int i; 149403d86467Sjmatthew 149503d86467Sjmatthew if (op == HIB_INIT) { 149603d86467Sjmatthew struct device *disk; 149703d86467Sjmatthew struct device *scsibus; 149803d86467Sjmatthew extern struct cfdriver sd_cd; 149903d86467Sjmatthew struct scsi_link *link; 150003d86467Sjmatthew struct scsibus_softc *bus_sc; 150103d86467Sjmatthew struct nvme_sqe_q qsqe; 150203d86467Sjmatthew struct nvme_cqe qcqe; 150303d86467Sjmatthew 150403d86467Sjmatthew /* find nvme softc */ 150503d86467Sjmatthew disk = disk_lookup(&sd_cd, DISKUNIT(dev)); 150603d86467Sjmatthew scsibus = disk->dv_parent; 150703d86467Sjmatthew my->sc = (struct nvme_softc *)disk->dv_parent->dv_parent; 150803d86467Sjmatthew 150903d86467Sjmatthew /* find scsi_link, which tells us the target */ 151003d86467Sjmatthew my->nsid = 0; 151103d86467Sjmatthew bus_sc = (struct scsibus_softc *)scsibus; 151203d86467Sjmatthew SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) { 151303d86467Sjmatthew if (link->device_softc == disk) { 1514397f5692Skettenis my->nsid = link->target; 151503d86467Sjmatthew break; 151603d86467Sjmatthew } 151703d86467Sjmatthew } 151803d86467Sjmatthew if (my->nsid == 0) 151903d86467Sjmatthew return (EIO); 152003d86467Sjmatthew 152103d86467Sjmatthew my->poffset = blkno; 152203d86467Sjmatthew my->psize = size; 152303d86467Sjmatthew 152403d86467Sjmatthew memset(NVME_DMA_KVA(my->sc->sc_hib_q->q_cq_dmamem), 0, 152503d86467Sjmatthew my->sc->sc_hib_q->q_entries * sizeof(struct nvme_cqe)); 152603d86467Sjmatthew memset(NVME_DMA_KVA(my->sc->sc_hib_q->q_sq_dmamem), 0, 152703d86467Sjmatthew my->sc->sc_hib_q->q_entries * sizeof(struct nvme_sqe)); 152803d86467Sjmatthew 152903d86467Sjmatthew my->sq_tail = 0; 153003d86467Sjmatthew my->cq_head = 0; 153103d86467Sjmatthew my->cqe_phase = NVME_CQE_PHASE; 153203d86467Sjmatthew 153303d86467Sjmatthew pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 153403d86467Sjmatthew 153503d86467Sjmatthew memset(&qsqe, 0, sizeof(qsqe)); 153603d86467Sjmatthew qsqe.opcode = NVM_ADMIN_ADD_IOCQ; 153703d86467Sjmatthew htolem64(&qsqe.prp1, 153803d86467Sjmatthew NVME_DMA_DVA(my->sc->sc_hib_q->q_cq_dmamem)); 153903d86467Sjmatthew htolem16(&qsqe.qsize, my->sc->sc_hib_q->q_entries - 1); 154003d86467Sjmatthew htolem16(&qsqe.qid, my->sc->sc_hib_q->q_id); 154103d86467Sjmatthew qsqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 154203d86467Sjmatthew if (nvme_hibernate_admin_cmd(my->sc, (struct nvme_sqe *)&qsqe, 154303d86467Sjmatthew &qcqe, 1) != 0) 154403d86467Sjmatthew return (EIO); 154503d86467Sjmatthew 154603d86467Sjmatthew memset(&qsqe, 0, sizeof(qsqe)); 154703d86467Sjmatthew qsqe.opcode = NVM_ADMIN_ADD_IOSQ; 154803d86467Sjmatthew htolem64(&qsqe.prp1, 154903d86467Sjmatthew NVME_DMA_DVA(my->sc->sc_hib_q->q_sq_dmamem)); 155003d86467Sjmatthew htolem16(&qsqe.qsize, my->sc->sc_hib_q->q_entries - 1); 155103d86467Sjmatthew htolem16(&qsqe.qid, my->sc->sc_hib_q->q_id); 155203d86467Sjmatthew htolem16(&qsqe.cqid, my->sc->sc_hib_q->q_id); 155303d86467Sjmatthew qsqe.qflags = NVM_SQE_Q_PC; 155403d86467Sjmatthew if (nvme_hibernate_admin_cmd(my->sc, (struct nvme_sqe *)&qsqe, 155503d86467Sjmatthew &qcqe, 2) != 0) 155603d86467Sjmatthew return (EIO); 155703d86467Sjmatthew 155803d86467Sjmatthew return (0); 155903d86467Sjmatthew } 156003d86467Sjmatthew 156103d86467Sjmatthew if (op != HIB_W) 156203d86467Sjmatthew return (0); 156303d86467Sjmatthew 156403d86467Sjmatthew isqe = NVME_DMA_KVA(my->sc->sc_hib_q->q_sq_dmamem); 156503d86467Sjmatthew isqe += my->sq_tail; 156603d86467Sjmatthew if (++my->sq_tail == my->sc->sc_hib_q->q_entries) 156703d86467Sjmatthew my->sq_tail = 0; 156803d86467Sjmatthew 156903d86467Sjmatthew memset(isqe, 0, sizeof(*isqe)); 157003d86467Sjmatthew isqe->opcode = NVM_CMD_WRITE; 157103d86467Sjmatthew htolem32(&isqe->nsid, my->nsid); 157203d86467Sjmatthew 157303d86467Sjmatthew pmap_extract(pmap_kernel(), addr, &data_phys); 157403d86467Sjmatthew data_bus_phys = data_phys; 157503d86467Sjmatthew htolem64(&isqe->entry.prp[0], data_bus_phys); 157603d86467Sjmatthew if ((size > my->sc->sc_mps) && (size <= my->sc->sc_mps * 2)) { 157703d86467Sjmatthew htolem64(&isqe->entry.prp[1], data_bus_phys + my->sc->sc_mps); 157803d86467Sjmatthew } else if (size > my->sc->sc_mps * 2) { 157903d86467Sjmatthew pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys); 158003d86467Sjmatthew page_bus_phys = page_phys; 158103d86467Sjmatthew htolem64(&isqe->entry.prp[1], page_bus_phys + 158203d86467Sjmatthew offsetof(struct nvme_hibernate_page, prpl)); 158303d86467Sjmatthew for (i = 1; i < (size / my->sc->sc_mps); i++) { 158403d86467Sjmatthew htolem64(&my->prpl[i - 1], data_bus_phys + 158503d86467Sjmatthew (i * my->sc->sc_mps)); 158603d86467Sjmatthew } 158703d86467Sjmatthew } 158803d86467Sjmatthew 158903d86467Sjmatthew isqe->slba = blkno + my->poffset; 159003d86467Sjmatthew isqe->nlb = (size / DEV_BSIZE) - 1; 159103d86467Sjmatthew isqe->cid = blkno % 0xffff; 159203d86467Sjmatthew 159303d86467Sjmatthew nvme_write4(my->sc, NVME_SQTDBL(NVME_HIB_Q, my->sc->sc_dstrd), 159403d86467Sjmatthew my->sq_tail); 159503d86467Sjmatthew 159603d86467Sjmatthew icqe = NVME_DMA_KVA(my->sc->sc_hib_q->q_cq_dmamem); 159703d86467Sjmatthew icqe += my->cq_head; 159803d86467Sjmatthew for (;;) { 159903d86467Sjmatthew flags = lemtoh16(&icqe->flags); 160003d86467Sjmatthew if ((flags & NVME_CQE_PHASE) == my->cqe_phase) 160103d86467Sjmatthew break; 160203d86467Sjmatthew 160303d86467Sjmatthew delay(10); 160403d86467Sjmatthew } 160503d86467Sjmatthew 160603d86467Sjmatthew if (++my->cq_head == my->sc->sc_hib_q->q_entries) { 160703d86467Sjmatthew my->cq_head = 0; 160803d86467Sjmatthew my->cqe_phase ^= NVME_CQE_PHASE; 160903d86467Sjmatthew } 161003d86467Sjmatthew nvme_write4(my->sc, NVME_CQHDBL(NVME_HIB_Q, my->sc->sc_dstrd), 161103d86467Sjmatthew my->cq_head); 161203d86467Sjmatthew if ((NVME_CQE_SC(flags) != NVME_CQE_SC_SUCCESS) || 161303d86467Sjmatthew (icqe->cid != blkno % 0xffff)) 161403d86467Sjmatthew return (EIO); 161503d86467Sjmatthew 161603d86467Sjmatthew return (0); 161703d86467Sjmatthew } 161803d86467Sjmatthew 161903d86467Sjmatthew #endif 1620