xref: /freebsd/sys/powerpc/ps3/ps3disk.c (revision e17f5b1d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2011 glevand (geoffrey.levand@mail.ru)
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  *
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/sysctl.h>
35 #include <sys/disk.h>
36 #include <sys/bio.h>
37 #include <sys/bus.h>
38 #include <sys/conf.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/module.h>
44 #include <sys/mutex.h>
45 
46 #include <vm/vm.h>
47 #include <vm/pmap.h>
48 
49 #include <machine/pio.h>
50 #include <machine/bus.h>
51 #include <machine/platform.h>
52 #include <machine/resource.h>
53 #include <sys/bus.h>
54 #include <sys/rman.h>
55 
56 #include <geom/geom_disk.h>
57 
58 #include "ps3bus.h"
59 #include "ps3-hvcall.h"
60 
61 #define PS3DISK_LOCK_INIT(_sc)		\
62 	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), "ps3disk", MTX_DEF)
63 #define PS3DISK_LOCK_DESTROY(_sc)	mtx_destroy(&_sc->sc_mtx);
64 #define PS3DISK_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
65 #define	PS3DISK_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
66 #define PS3DISK_ASSERT_LOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_OWNED);
67 #define PS3DISK_ASSERT_UNLOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);
68 
69 #define LV1_STORAGE_ATA_HDDOUT 		0x23
70 
71 static SYSCTL_NODE(_hw, OID_AUTO, ps3disk, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
72     "PS3 Disk driver parameters");
73 
74 #ifdef PS3DISK_DEBUG
75 static int ps3disk_debug = 0;
76 SYSCTL_INT(_hw_ps3disk, OID_AUTO, debug, CTLFLAG_RW, &ps3disk_debug,
77 	0, "control debugging printfs");
78 TUNABLE_INT("hw.ps3disk.debug", &ps3disk_debug);
79 enum {
80 	PS3DISK_DEBUG_INTR	= 0x00000001,
81 	PS3DISK_DEBUG_TASK	= 0x00000002,
82 	PS3DISK_DEBUG_READ	= 0x00000004,
83 	PS3DISK_DEBUG_WRITE	= 0x00000008,
84 	PS3DISK_DEBUG_FLUSH	= 0x00000010,
85 	PS3DISK_DEBUG_ANY	= 0xffffffff
86 };
87 #define	DPRINTF(sc, m, fmt, ...)				\
88 do {								\
89 	if (sc->sc_debug & (m))					\
90 		printf(fmt, __VA_ARGS__);			\
91 } while (0)
92 #else
93 #define	DPRINTF(sc, m, fmt, ...)
94 #endif
95 
96 struct ps3disk_region {
97 	uint64_t r_id;
98 	uint64_t r_start;
99 	uint64_t r_size;
100 	uint64_t r_flags;
101 };
102 
103 struct ps3disk_softc {
104 	device_t sc_dev;
105 
106 	struct mtx sc_mtx;
107 
108 	uint64_t sc_blksize;
109 	uint64_t sc_nblocks;
110 
111 	uint64_t sc_nregs;
112 	struct ps3disk_region *sc_reg;
113 
114 	int sc_irqid;
115 	struct resource	*sc_irq;
116 	void *sc_irqctx;
117 
118 	struct disk **sc_disk;
119 
120 	struct bio_queue_head sc_bioq;
121 	struct bio_queue_head sc_deferredq;
122 	struct proc *sc_task;
123 
124 	bus_dma_tag_t sc_dmatag;
125 
126 	int sc_running;
127 	int sc_debug;
128 };
129 
130 static int ps3disk_open(struct disk *dp);
131 static int ps3disk_close(struct disk *dp);
132 static void ps3disk_strategy(struct bio *bp);
133 
134 static void ps3disk_task(void *arg);
135 static void ps3disk_intr(void *arg);
136 static int ps3disk_get_disk_geometry(struct ps3disk_softc *sc);
137 static int ps3disk_enum_regions(struct ps3disk_softc *sc);
138 static void ps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs,
139     int error);
140 
141 static void ps3disk_sysctlattach(struct ps3disk_softc *sc);
142 
143 static MALLOC_DEFINE(M_PS3DISK, "ps3disk", "PS3 Disk");
144 
145 static int
146 ps3disk_probe(device_t dev)
147 {
148 	if (ps3bus_get_bustype(dev) != PS3_BUSTYPE_STORAGE ||
149 	    ps3bus_get_devtype(dev) != PS3_DEVTYPE_DISK)
150 		return (ENXIO);
151 
152 	device_set_desc(dev, "Playstation 3 Disk");
153 
154 	return (BUS_PROBE_SPECIFIC);
155 }
156 
157 static int
158 ps3disk_attach(device_t dev)
159 {
160 	struct ps3disk_softc *sc;
161 	struct disk *d;
162 	intmax_t mb;
163 	uint64_t junk;
164 	char unit;
165 	int i, err;
166 
167 	sc = device_get_softc(dev);
168 	sc->sc_dev = dev;
169 
170 	PS3DISK_LOCK_INIT(sc);
171 
172 	err = ps3disk_get_disk_geometry(sc);
173 	if (err) {
174 		device_printf(dev, "Could not get disk geometry\n");
175 		err = ENXIO;
176 		goto fail_destroy_lock;
177 	}
178 
179 	device_printf(dev, "block size %lu total blocks %lu\n",
180 	    sc->sc_blksize, sc->sc_nblocks);
181 
182 	err = ps3disk_enum_regions(sc);
183 	if (err) {
184 		device_printf(dev, "Could not enumerate disk regions\n");
185 		err = ENXIO;
186 		goto fail_destroy_lock;
187 	}
188 
189 	device_printf(dev, "Found %lu regions\n", sc->sc_nregs);
190 
191 	if (!sc->sc_nregs) {
192 		err = ENXIO;
193 		goto fail_destroy_lock;
194 	}
195 
196 	/* Setup interrupt handler */
197 	sc->sc_irqid = 0;
198 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irqid,
199 	    RF_ACTIVE);
200 	if (!sc->sc_irq) {
201 		device_printf(dev, "Could not allocate IRQ\n");
202 		err = ENXIO;
203 		goto fail_free_regions;
204 	}
205 
206 	err = bus_setup_intr(dev, sc->sc_irq,
207 	    INTR_TYPE_BIO | INTR_MPSAFE | INTR_ENTROPY,
208 	    NULL, ps3disk_intr, sc, &sc->sc_irqctx);
209 	if (err) {
210 		device_printf(dev, "Could not setup IRQ\n");
211 		err = ENXIO;
212 		goto fail_release_intr;
213 	}
214 
215 	/* Setup DMA */
216 	err = bus_dma_tag_create(bus_get_dma_tag(dev), 4096, 0,
217 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
218 	    BUS_SPACE_UNRESTRICTED, 1, PAGE_SIZE, 0,
219 	    busdma_lock_mutex, &sc->sc_mtx, &sc->sc_dmatag);
220 	if (err) {
221 		device_printf(dev, "Could not create DMA tag\n");
222 		err = ENXIO;
223 		goto fail_teardown_intr;
224 	}
225 
226 	/* Setup disks */
227 
228 	sc->sc_disk = malloc(sc->sc_nregs * sizeof(struct disk *),
229 	    M_PS3DISK, M_ZERO | M_WAITOK);
230 	if (!sc->sc_disk) {
231 		device_printf(dev, "Could not allocate disk(s)\n");
232 		err = ENOMEM;
233 		goto fail_teardown_intr;
234 	}
235 
236 	for (i = 0; i < sc->sc_nregs; i++) {
237 		struct ps3disk_region *rp = &sc->sc_reg[i];
238 
239 		d = sc->sc_disk[i] = disk_alloc();
240 		d->d_open = ps3disk_open;
241 		d->d_close = ps3disk_close;
242 		d->d_strategy = ps3disk_strategy;
243 		d->d_name = "ps3disk";
244 		d->d_drv1 = sc;
245 		d->d_maxsize = PAGE_SIZE;
246 		d->d_sectorsize = sc->sc_blksize;
247 		d->d_unit = i;
248 		d->d_mediasize = sc->sc_reg[i].r_size * sc->sc_blksize;
249 		d->d_flags |= DISKFLAG_CANFLUSHCACHE;
250 
251 		mb = d->d_mediasize >> 20;
252 		unit = 'M';
253 		if (mb >= 10240) {
254 			unit = 'G';
255 			mb /= 1024;
256 		}
257 
258 		/* Test to see if we can read this region */
259 		err = lv1_storage_read(ps3bus_get_device(dev), d->d_unit,
260 		    0, 0, rp->r_flags, 0, &junk);
261 		device_printf(dev, "region %d %ju%cB%s\n", i, mb, unit,
262 		    (err == LV1_DENIED_BY_POLICY) ?  " (hypervisor protected)"
263 		    : "");
264 
265 		if (err != LV1_DENIED_BY_POLICY)
266 			disk_create(d, DISK_VERSION);
267 	}
268 	err = 0;
269 
270 	bioq_init(&sc->sc_bioq);
271 	bioq_init(&sc->sc_deferredq);
272 	kproc_create(&ps3disk_task, sc, &sc->sc_task, 0, 0, "ps3disk");
273 
274 	ps3disk_sysctlattach(sc);
275 	sc->sc_running = 1;
276 	return (0);
277 
278 fail_teardown_intr:
279 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
280 fail_release_intr:
281 	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
282 fail_free_regions:
283 	free(sc->sc_reg, M_PS3DISK);
284 fail_destroy_lock:
285 	PS3DISK_LOCK_DESTROY(sc);
286 	return (err);
287 }
288 
289 static int
290 ps3disk_detach(device_t dev)
291 {
292 	struct ps3disk_softc *sc = device_get_softc(dev);
293 	int i;
294 
295 	for (i = 0; i < sc->sc_nregs; i++)
296 		disk_destroy(sc->sc_disk[i]);
297 
298 	bus_dma_tag_destroy(sc->sc_dmatag);
299 
300 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
301 	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
302 
303 	free(sc->sc_disk, M_PS3DISK);
304 	free(sc->sc_reg, M_PS3DISK);
305 
306 	PS3DISK_LOCK_DESTROY(sc);
307 
308 	return (0);
309 }
310 
311 static int
312 ps3disk_open(struct disk *dp)
313 {
314 	return (0);
315 }
316 
317 static int
318 ps3disk_close(struct disk *dp)
319 {
320 	return (0);
321 }
322 
323 /* Process deferred blocks */
324 static void
325 ps3disk_task(void *arg)
326 {
327 	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
328 	struct bio *bp;
329 
330 
331 	while (1) {
332 		kproc_suspend_check(sc->sc_task);
333 		tsleep(&sc->sc_deferredq, PRIBIO, "ps3disk", 10);
334 
335 		PS3DISK_LOCK(sc);
336 		bp = bioq_takefirst(&sc->sc_deferredq);
337 		PS3DISK_UNLOCK(sc);
338 
339 		if (bp == NULL)
340 			continue;
341 
342 		if (bp->bio_driver1 != NULL) {
343 			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
344 			    bp->bio_driver1);
345 			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
346 			    bp->bio_driver1);
347 		}
348 
349 		ps3disk_strategy(bp);
350 	}
351 
352 	kproc_exit(0);
353 }
354 
355 static void
356 ps3disk_strategy(struct bio *bp)
357 {
358 	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
359 	int err;
360 
361 	if (sc == NULL) {
362 		bp->bio_flags |= BIO_ERROR;
363 		bp->bio_error = EINVAL;
364 		biodone(bp);
365 		return;
366 	}
367 
368 	PS3DISK_LOCK(sc);
369 	bp->bio_resid = bp->bio_bcount;
370 	bioq_insert_tail(&sc->sc_bioq, bp);
371 
372 	DPRINTF(sc, PS3DISK_DEBUG_TASK, "%s: bio_cmd 0x%02x\n",
373 	    __func__, bp->bio_cmd);
374 
375 	err = 0;
376 	if (bp->bio_cmd == BIO_FLUSH) {
377 		bp->bio_driver1 = 0;
378 		err = lv1_storage_send_device_command(
379 		    ps3bus_get_device(sc->sc_dev), LV1_STORAGE_ATA_HDDOUT,
380 		    0, 0, 0, 0, (uint64_t *)&bp->bio_driver2);
381 		if (err == LV1_BUSY)
382 			err = EAGAIN;
383 	} else if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
384 		if (bp->bio_bcount % sc->sc_blksize != 0) {
385 			err = EINVAL;
386 		} else {
387 			bus_dmamap_create(sc->sc_dmatag, BUS_DMA_COHERENT,
388 			    (bus_dmamap_t *)(&bp->bio_driver1));
389 			err = bus_dmamap_load(sc->sc_dmatag,
390 			    (bus_dmamap_t)(bp->bio_driver1), bp->bio_data,
391 			    bp->bio_bcount, ps3disk_transfer, bp, 0);
392 			if (err == EINPROGRESS)
393 				err = 0;
394 		}
395 	} else {
396 		err = EINVAL;
397 	}
398 
399 	if (err == EAGAIN) {
400 		bioq_remove(&sc->sc_bioq, bp);
401 		bioq_insert_tail(&sc->sc_deferredq, bp);
402 	} else if (err != 0) {
403 		bp->bio_error = err;
404 		bp->bio_flags |= BIO_ERROR;
405 		bioq_remove(&sc->sc_bioq, bp);
406 		disk_err(bp, "hard error", -1, 1);
407 		biodone(bp);
408 	}
409 
410 	PS3DISK_UNLOCK(sc);
411 }
412 
413 static void
414 ps3disk_intr(void *arg)
415 {
416 	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
417 	device_t dev = sc->sc_dev;
418 	uint64_t devid = ps3bus_get_device(dev);
419 	struct bio *bp;
420 	uint64_t tag, status;
421 
422 	if (lv1_storage_get_async_status(devid, &tag, &status) != 0)
423 		return;
424 
425 	PS3DISK_LOCK(sc);
426 
427 	DPRINTF(sc, PS3DISK_DEBUG_INTR, "%s: tag 0x%016lx "
428 	    "status 0x%016lx\n", __func__, tag, status);
429 
430 	/* Locate the matching request */
431 	TAILQ_FOREACH(bp, &sc->sc_bioq.queue, bio_queue) {
432 		if ((uint64_t)bp->bio_driver2 != tag)
433 			continue;
434 
435 		if (status != 0) {
436 			device_printf(sc->sc_dev, "%s error (%#lx)\n",
437 			    (bp->bio_cmd == BIO_READ) ? "Read" : "Write",
438 			    status);
439 			bp->bio_error = EIO;
440 			bp->bio_flags |= BIO_ERROR;
441 		} else {
442 			bp->bio_error = 0;
443 			bp->bio_resid = 0;
444 			bp->bio_flags |= BIO_DONE;
445 		}
446 
447 		if (bp->bio_driver1 != NULL) {
448 			if (bp->bio_cmd == BIO_READ)
449 				bus_dmamap_sync(sc->sc_dmatag, (bus_dmamap_t)
450 				    bp->bio_driver1, BUS_DMASYNC_POSTREAD);
451 			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
452 			    bp->bio_driver1);
453 			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
454 			    bp->bio_driver1);
455 		}
456 
457 		bioq_remove(&sc->sc_bioq, bp);
458 		biodone(bp);
459 		break;
460 	}
461 
462 	if (bioq_first(&sc->sc_deferredq) != NULL)
463 		wakeup(&sc->sc_deferredq);
464 
465 	PS3DISK_UNLOCK(sc);
466 }
467 
468 static int
469 ps3disk_get_disk_geometry(struct ps3disk_softc *sc)
470 {
471 	device_t dev = sc->sc_dev;
472 	uint64_t bus_index = ps3bus_get_busidx(dev);
473 	uint64_t dev_index = ps3bus_get_devidx(dev);
474 	uint64_t junk;
475 	int err;
476 
477 	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
478 	    (lv1_repository_string("bus") >> 32) | bus_index,
479 	    lv1_repository_string("dev") | dev_index,
480 	    lv1_repository_string("blk_size"), 0, &sc->sc_blksize, &junk);
481 	if (err) {
482 		device_printf(dev, "Could not get block size (0x%08x)\n", err);
483 		return (ENXIO);
484 	}
485 
486 	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
487 	    (lv1_repository_string("bus") >> 32) | bus_index,
488 	    lv1_repository_string("dev") | dev_index,
489 	    lv1_repository_string("n_blocks"), 0, &sc->sc_nblocks, &junk);
490 	if (err) {
491 		device_printf(dev, "Could not get total number of blocks "
492 		    "(0x%08x)\n", err);
493 		err = ENXIO;
494 	}
495 
496 	return (err);
497 }
498 
499 static int
500 ps3disk_enum_regions(struct ps3disk_softc *sc)
501 {
502 	device_t dev = sc->sc_dev;
503 	uint64_t bus_index = ps3bus_get_busidx(dev);
504 	uint64_t dev_index = ps3bus_get_devidx(dev);
505 	uint64_t junk;
506 	int i, err;
507 
508 	/* Read number of regions */
509 
510 	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
511 	    (lv1_repository_string("bus") >> 32) | bus_index,
512 	    lv1_repository_string("dev") | dev_index,
513 	    lv1_repository_string("n_regs"), 0, &sc->sc_nregs, &junk);
514 	if (err) {
515 		device_printf(dev, "Could not get number of regions (0x%08x)\n",
516 		    err);
517 		err = ENXIO;
518 		goto fail;
519 	}
520 
521 	if (!sc->sc_nregs)
522 		return 0;
523 
524 	sc->sc_reg = malloc(sc->sc_nregs * sizeof(struct ps3disk_region),
525 	    M_PS3DISK, M_ZERO | M_WAITOK);
526 	if (!sc->sc_reg) {
527 		err = ENOMEM;
528 		goto fail;
529 	}
530 
531 	/* Setup regions */
532 
533 	for (i = 0; i < sc->sc_nregs; i++) {
534 		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
535 		    (lv1_repository_string("bus") >> 32) | bus_index,
536 		    lv1_repository_string("dev") | dev_index,
537 		    lv1_repository_string("region") | i,
538 		    lv1_repository_string("id"), &sc->sc_reg[i].r_id, &junk);
539 		if (err) {
540 			device_printf(dev, "Could not get region id (0x%08x)\n",
541 			    err);
542 			err = ENXIO;
543 			goto fail;
544 		}
545 
546 		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
547 		    (lv1_repository_string("bus") >> 32) | bus_index,
548 		    lv1_repository_string("dev") | dev_index,
549 		    lv1_repository_string("region") | i,
550 		    lv1_repository_string("start"), &sc->sc_reg[i].r_start,
551 		    &junk);
552 		if (err) {
553 			device_printf(dev, "Could not get region start "
554 			    "(0x%08x)\n", err);
555 			err = ENXIO;
556 			goto fail;
557 		}
558 
559 		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
560 		    (lv1_repository_string("bus") >> 32) | bus_index,
561 		    lv1_repository_string("dev") | dev_index,
562 		    lv1_repository_string("region") | i,
563 		    lv1_repository_string("size"), &sc->sc_reg[i].r_size,
564 		    &junk);
565 		if (err) {
566 			device_printf(dev, "Could not get region size "
567 			    "(0x%08x)\n", err);
568 			err = ENXIO;
569 			goto fail;
570 		}
571 
572 		if (i == 0)
573 			sc->sc_reg[i].r_flags = 0x2;
574 		else
575 			sc->sc_reg[i].r_flags = 0;
576 	}
577 
578 	return (0);
579 
580 fail:
581 
582 	sc->sc_nregs = 0;
583 	if (sc->sc_reg)
584 		free(sc->sc_reg, M_PS3DISK);
585 
586 	return (err);
587 }
588 
589 static void
590 ps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
591 {
592 	struct bio *bp = (struct bio *)(arg);
593 	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
594 	struct ps3disk_region *rp = &sc->sc_reg[bp->bio_disk->d_unit];
595 	uint64_t devid = ps3bus_get_device(sc->sc_dev);
596 	uint64_t block;
597 	int i, err;
598 
599 	/* Locks already held by busdma */
600 	PS3DISK_ASSERT_LOCKED(sc);
601 
602 	if (error) {
603 		bp->bio_error = error;
604 		bp->bio_flags |= BIO_ERROR;
605 		bioq_remove(&sc->sc_bioq, bp);
606 		biodone(bp);
607 		return;
608 	}
609 
610 	block = bp->bio_pblkno;
611 	for (i = 0; i < nsegs; i++) {
612 		KASSERT((segs[i].ds_len % sc->sc_blksize) == 0,
613 		    ("DMA fragments not blocksize multiples"));
614 
615 		if (bp->bio_cmd == BIO_READ) {
616 			err = lv1_storage_read(devid, rp->r_id,
617 			    block, segs[i].ds_len/sc->sc_blksize,
618 			    rp->r_flags, segs[i].ds_addr,
619 			    (uint64_t *)&bp->bio_driver2);
620 		} else {
621 			bus_dmamap_sync(sc->sc_dmatag,
622 			    (bus_dmamap_t)bp->bio_driver1,
623 			    BUS_DMASYNC_PREWRITE);
624 			err = lv1_storage_write(devid, rp->r_id,
625 			    block, segs[i].ds_len/sc->sc_blksize,
626 			    rp->r_flags, segs[i].ds_addr,
627 			    (uint64_t *)&bp->bio_driver2);
628 		}
629 
630 		if (err) {
631 			if (err == LV1_BUSY) {
632 				bioq_remove(&sc->sc_bioq, bp);
633 				bioq_insert_tail(&sc->sc_deferredq, bp);
634 			} else {
635 				bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
636 				    bp->bio_driver1);
637 				bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
638 				    bp->bio_driver1);
639 				device_printf(sc->sc_dev, "Could not read "
640 				    "sectors (0x%08x)\n", err);
641 				bp->bio_error = EINVAL;
642 				bp->bio_flags |= BIO_ERROR;
643 				bioq_remove(&sc->sc_bioq, bp);
644 				biodone(bp);
645 			}
646 
647 			break;
648 		}
649 
650 		DPRINTF(sc, PS3DISK_DEBUG_READ, "%s: tag 0x%016lx\n",
651 		    __func__, sc->sc_bounce_tag);
652 	}
653 }
654 
655 #ifdef PS3DISK_DEBUG
656 static int
657 ps3disk_sysctl_debug(SYSCTL_HANDLER_ARGS)
658 {
659 	struct ps3disk_softc *sc = arg1;
660 	int debug, error;
661 
662 	debug = sc->sc_debug;
663 
664 	error = sysctl_handle_int(oidp, &debug, 0, req);
665 	if (error || !req->newptr)
666 		return error;
667 
668 	sc->sc_debug = debug;
669 
670 	return 0;
671 }
672 #endif
673 
674 static void
675 ps3disk_sysctlattach(struct ps3disk_softc *sc)
676 {
677 #ifdef PS3DISK_DEBUG
678 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
679 	struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
680 
681 	sc->sc_debug = ps3disk_debug;
682 
683 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
684 	    "debug", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
685 	    ps3disk_sysctl_debug, "I", "control debugging printfs");
686 #endif
687 }
688 
689 static device_method_t ps3disk_methods[] = {
690 	DEVMETHOD(device_probe,		ps3disk_probe),
691 	DEVMETHOD(device_attach,	ps3disk_attach),
692 	DEVMETHOD(device_detach,	ps3disk_detach),
693 	{0, 0},
694 };
695 
696 static driver_t ps3disk_driver = {
697 	"ps3disk",
698 	ps3disk_methods,
699 	sizeof(struct ps3disk_softc),
700 };
701 
702 static devclass_t ps3disk_devclass;
703 
704 DRIVER_MODULE(ps3disk, ps3bus, ps3disk_driver, ps3disk_devclass, 0, 0);
705