xref: /dragonfly/sys/dev/raid/mfi/mfi.c (revision 10cbe914)
1 /*-
2  * Copyright (c) 2006 IronPort Systems
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 /*-
27  * Copyright (c) 2007 LSI Corp.
28  * Copyright (c) 2007 Rajesh Prabhakaran.
29  * All rights reserved.
30  *
31  * Redistribution and use in source and binary forms, with or without
32  * modification, are permitted provided that the following conditions
33  * are met:
34  * 1. Redistributions of source code must retain the above copyright
35  *    notice, this list of conditions and the following disclaimer.
36  * 2. Redistributions in binary form must reproduce the above copyright
37  *    notice, this list of conditions and the following disclaimer in the
38  *    documentation and/or other materials provided with the distribution.
39  *
40  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
41  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
42  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
43  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
44  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
45  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
46  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
48  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
49  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
50  * SUCH DAMAGE.
51  *
52  * $FreeBSD: src/sys/dev/mfi/mfi.c,v 1.54 2009/12/07 21:24:07 jkim Exp $
53  */
54 
55 #include "opt_mfi.h"
56 
57 #include <sys/param.h>
58 #include <sys/systm.h>
59 #include <sys/sysctl.h>
60 #include <sys/malloc.h>
61 #include <sys/kernel.h>
62 #include <sys/bus.h>
63 #include <sys/eventhandler.h>
64 #include <sys/rman.h>
65 #include <sys/bus_dma.h>
66 #include <sys/buf2.h>
67 #include <sys/ioccom.h>
68 #include <sys/uio.h>
69 #include <sys/proc.h>
70 #include <sys/signalvar.h>
71 #include <sys/device.h>
72 #include <sys/mplock2.h>
73 
74 #include <dev/raid/mfi/mfireg.h>
75 #include <dev/raid/mfi/mfi_ioctl.h>
76 #include <dev/raid/mfi/mfivar.h>
77 
78 static int	mfi_alloc_commands(struct mfi_softc *);
79 static int	mfi_comms_init(struct mfi_softc *);
80 static int	mfi_wait_command(struct mfi_softc *, struct mfi_command *);
81 static int	mfi_get_controller_info(struct mfi_softc *);
82 static int	mfi_get_log_state(struct mfi_softc *,
83 		    struct mfi_evt_log_state **);
84 static int	mfi_parse_entries(struct mfi_softc *, int, int);
85 static int	mfi_dcmd_command(struct mfi_softc *, struct mfi_command **,
86 		    uint32_t, void **, size_t);
87 static void	mfi_data_cb(void *, bus_dma_segment_t *, int, int);
88 static void	mfi_startup(void *arg);
89 static void	mfi_intr(void *arg);
90 static void	mfi_ldprobe(struct mfi_softc *sc);
91 static int	mfi_aen_register(struct mfi_softc *sc, int seq, int locale);
92 static void	mfi_aen_complete(struct mfi_command *);
93 static int	mfi_aen_setup(struct mfi_softc *, uint32_t);
94 static int	mfi_add_ld(struct mfi_softc *sc, int);
95 static void	mfi_add_ld_complete(struct mfi_command *);
96 static struct mfi_command * mfi_bio_command(struct mfi_softc *);
97 static void	mfi_bio_complete(struct mfi_command *);
98 static int	mfi_mapcmd(struct mfi_softc *, struct mfi_command *);
99 static int	mfi_send_frame(struct mfi_softc *, struct mfi_command *);
100 static void	mfi_complete(struct mfi_softc *, struct mfi_command *);
101 static int	mfi_abort(struct mfi_softc *, struct mfi_command *);
102 static int	mfi_linux_ioctl_int(struct cdev *, u_long, caddr_t, int);
103 static void	mfi_timeout(void *);
104 static int	mfi_user_command(struct mfi_softc *,
105 		    struct mfi_ioc_passthru *);
106 static void 	mfi_enable_intr_xscale(struct mfi_softc *sc);
107 static void 	mfi_enable_intr_ppc(struct mfi_softc *sc);
108 static int32_t 	mfi_read_fw_status_xscale(struct mfi_softc *sc);
109 static int32_t 	mfi_read_fw_status_ppc(struct mfi_softc *sc);
110 static int 	mfi_check_clear_intr_xscale(struct mfi_softc *sc);
111 static int 	mfi_check_clear_intr_ppc(struct mfi_softc *sc);
112 static void 	mfi_issue_cmd_xscale(struct mfi_softc *sc,uint32_t bus_add,uint32_t frame_cnt);
113 static void 	mfi_issue_cmd_ppc(struct mfi_softc *sc,uint32_t bus_add,uint32_t frame_cnt);
114 static void	mfi_filter_detach(struct knote *);
115 static int	mfi_filter_read(struct knote *, long);
116 static int	mfi_filter_write(struct knote *, long);
117 
118 SYSCTL_NODE(_hw, OID_AUTO, mfi, CTLFLAG_RD, 0, "MFI driver parameters");
119 static int	mfi_event_locale = MFI_EVT_LOCALE_ALL;
120 TUNABLE_INT("hw.mfi.event_locale", &mfi_event_locale);
121 SYSCTL_INT(_hw_mfi, OID_AUTO, event_locale, CTLFLAG_RW, &mfi_event_locale,
122             0, "event message locale");
123 
124 static int	mfi_event_class = MFI_EVT_CLASS_INFO;
125 TUNABLE_INT("hw.mfi.event_class", &mfi_event_class);
126 SYSCTL_INT(_hw_mfi, OID_AUTO, event_class, CTLFLAG_RW, &mfi_event_class,
127           0, "event message class");
128 
129 static int	mfi_max_cmds = 128;
130 TUNABLE_INT("hw.mfi.max_cmds", &mfi_max_cmds);
131 SYSCTL_INT(_hw_mfi, OID_AUTO, max_cmds, CTLFLAG_RD, &mfi_max_cmds,
132 	   0, "Max commands");
133 
134 /* Management interface */
135 static d_open_t		mfi_open;
136 static d_close_t	mfi_close;
137 static d_ioctl_t	mfi_ioctl;
138 static d_kqfilter_t	mfi_kqfilter;
139 
140 static struct dev_ops mfi_ops = {
141 	{ "mfi", 0, 0 },
142 	.d_open = 	mfi_open,
143 	.d_close =	mfi_close,
144 	.d_ioctl =	mfi_ioctl,
145 	.d_kqfilter =	mfi_kqfilter,
146 };
147 
148 static struct filterops mfi_read_filterops =
149 	{ FILTEROP_ISFD, NULL, mfi_filter_detach, mfi_filter_read };
150 static struct filterops mfi_write_filterops =
151 	{ FILTEROP_ISFD, NULL, mfi_filter_detach, mfi_filter_write };
152 
153 MALLOC_DEFINE(M_MFIBUF, "mfibuf", "Buffers for the MFI driver");
154 
155 #define MFI_INQ_LENGTH SHORT_INQUIRY_LENGTH
156 
157 static void
158 mfi_enable_intr_xscale(struct mfi_softc *sc)
159 {
160 	MFI_WRITE4(sc, MFI_OMSK, 0x01);
161 }
162 
163 static void
164 mfi_enable_intr_ppc(struct mfi_softc *sc)
165 {
166 	MFI_WRITE4(sc, MFI_ODCR0, 0xFFFFFFFF);
167 	if (sc->mfi_flags & MFI_FLAGS_1078) {
168 		MFI_WRITE4(sc, MFI_OMSK, ~MFI_1078_EIM);
169 	} else if (sc->mfi_flags & MFI_FLAGS_GEN2) {
170 		MFI_WRITE4(sc, MFI_OMSK, ~MFI_GEN2_EIM);
171 	}
172 }
173 
174 static int32_t
175 mfi_read_fw_status_xscale(struct mfi_softc *sc)
176 {
177 	return MFI_READ4(sc, MFI_OMSG0);
178 }
179 
180 static int32_t
181 mfi_read_fw_status_ppc(struct mfi_softc *sc)
182 {
183 	return MFI_READ4(sc, MFI_OSP0);
184 }
185 
186 static int
187 mfi_check_clear_intr_xscale(struct mfi_softc *sc)
188 {
189 	int32_t status;
190 
191 	status = MFI_READ4(sc, MFI_OSTS);
192 	if ((status & MFI_OSTS_INTR_VALID) == 0)
193 		return 1;
194 
195 	MFI_WRITE4(sc, MFI_OSTS, status);
196 	return 0;
197 }
198 
199 static int
200 mfi_check_clear_intr_ppc(struct mfi_softc *sc)
201 {
202 	int32_t status;
203 
204 	status = MFI_READ4(sc, MFI_OSTS);
205 	if (sc->mfi_flags & MFI_FLAGS_1078) {
206 		if (!(status & MFI_1078_RM)) {
207 			return 1;
208 		}
209 	} else if (sc->mfi_flags & MFI_FLAGS_GEN2) {
210 		if (!(status & MFI_GEN2_RM)) {
211 			return 1;
212 		}
213 	}
214 
215 	MFI_WRITE4(sc, MFI_ODCR0, status);
216 	return 0;
217 }
218 
219 static void
220 mfi_issue_cmd_xscale(struct mfi_softc *sc,uint32_t bus_add,uint32_t frame_cnt)
221 {
222 	MFI_WRITE4(sc, MFI_IQP,(bus_add >>3)|frame_cnt);
223 }
224 
225 static void
226 mfi_issue_cmd_ppc(struct mfi_softc *sc,uint32_t bus_add,uint32_t frame_cnt)
227 {
228 	MFI_WRITE4(sc, MFI_IQP, (bus_add |frame_cnt <<1)|1 );
229 }
230 
231 static int
232 mfi_transition_firmware(struct mfi_softc *sc)
233 {
234 	uint32_t fw_state, cur_state;
235 	int max_wait, i;
236 
237 	fw_state = sc->mfi_read_fw_status(sc)& MFI_FWSTATE_MASK;
238 	while (fw_state != MFI_FWSTATE_READY) {
239 		if (bootverbose)
240 			device_printf(sc->mfi_dev, "Waiting for firmware to "
241 			"become ready\n");
242 		cur_state = fw_state;
243 		switch (fw_state) {
244 		case MFI_FWSTATE_FAULT:
245 			device_printf(sc->mfi_dev, "Firmware fault\n");
246 			return (ENXIO);
247 		case MFI_FWSTATE_WAIT_HANDSHAKE:
248 			MFI_WRITE4(sc, MFI_IDB, MFI_FWINIT_CLEAR_HANDSHAKE);
249 			max_wait = 2;
250 			break;
251 		case MFI_FWSTATE_OPERATIONAL:
252 			MFI_WRITE4(sc, MFI_IDB, MFI_FWINIT_READY);
253 			max_wait = 10;
254 			break;
255 		case MFI_FWSTATE_UNDEFINED:
256 		case MFI_FWSTATE_BB_INIT:
257 			max_wait = 2;
258 			break;
259 		case MFI_FWSTATE_FW_INIT:
260 		case MFI_FWSTATE_DEVICE_SCAN:
261 		case MFI_FWSTATE_FLUSH_CACHE:
262 			max_wait = 20;
263 			break;
264 		default:
265 			device_printf(sc->mfi_dev,"Unknown firmware state %d\n",
266 			    fw_state);
267 			return (ENXIO);
268 		}
269 		for (i = 0; i < (max_wait * 10); i++) {
270 			fw_state = sc->mfi_read_fw_status(sc) & MFI_FWSTATE_MASK;
271 			if (fw_state == cur_state)
272 				DELAY(100000);
273 			else
274 				break;
275 		}
276 		if (fw_state == cur_state) {
277 			device_printf(sc->mfi_dev, "firmware stuck in state "
278 			    "%#x\n", fw_state);
279 			return (ENXIO);
280 		}
281 	}
282 	return (0);
283 }
284 
285 static void
286 mfi_addr32_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
287 {
288 	uint32_t *addr;
289 
290 	addr = arg;
291 	*addr = segs[0].ds_addr;
292 }
293 
294 int
295 mfi_attach(struct mfi_softc *sc)
296 {
297 	uint32_t status;
298 	int error, commsz, framessz, sensesz;
299 	int frames, unit, max_fw_sge;
300 
301 	device_printf(sc->mfi_dev, "Megaraid SAS driver Ver 3.00 \n");
302 
303 	lockinit(&sc->mfi_io_lock, "MFI I/O lock", 0, LK_CANRECURSE);
304 	lockinit(&sc->mfi_config_lock, "MFI config", 0, LK_CANRECURSE);
305 	TAILQ_INIT(&sc->mfi_ld_tqh);
306 	TAILQ_INIT(&sc->mfi_aen_pids);
307 	TAILQ_INIT(&sc->mfi_cam_ccbq);
308 
309 	mfi_initq_free(sc);
310 	mfi_initq_ready(sc);
311 	mfi_initq_busy(sc);
312 	mfi_initq_bio(sc);
313 
314 	if (sc->mfi_flags & MFI_FLAGS_1064R) {
315 		sc->mfi_enable_intr = mfi_enable_intr_xscale;
316 		sc->mfi_read_fw_status = mfi_read_fw_status_xscale;
317 		sc->mfi_check_clear_intr = mfi_check_clear_intr_xscale;
318 		sc->mfi_issue_cmd = mfi_issue_cmd_xscale;
319 	}
320 	else {
321 		sc->mfi_enable_intr =  mfi_enable_intr_ppc;
322 		sc->mfi_read_fw_status = mfi_read_fw_status_ppc;
323 		sc->mfi_check_clear_intr = mfi_check_clear_intr_ppc;
324 		sc->mfi_issue_cmd = mfi_issue_cmd_ppc;
325 	}
326 
327 
328 	/* Before we get too far, see if the firmware is working */
329 	if ((error = mfi_transition_firmware(sc)) != 0) {
330 		device_printf(sc->mfi_dev, "Firmware not in READY state, "
331 		    "error %d\n", error);
332 		return (ENXIO);
333 	}
334 
335 	/*
336 	 * Get information needed for sizing the contiguous memory for the
337 	 * frame pool.  Size down the sgl parameter since we know that
338 	 * we will never need more than what's required for MAXPHYS.
339 	 * It would be nice if these constants were available at runtime
340 	 * instead of compile time.
341 	 */
342 	status = sc->mfi_read_fw_status(sc);
343 	sc->mfi_max_fw_cmds = status & MFI_FWSTATE_MAXCMD_MASK;
344 	max_fw_sge = (status & MFI_FWSTATE_MAXSGL_MASK) >> 16;
345 	sc->mfi_max_sge = min(max_fw_sge, ((MFI_MAXPHYS / PAGE_SIZE) + 1));
346 
347 	/*
348 	 * Create the dma tag for data buffers.  Used both for block I/O
349 	 * and for various internal data queries.
350 	 */
351 	if (bus_dma_tag_create( sc->mfi_parent_dmat,	/* parent */
352 				1, 0,			/* algnmnt, boundary */
353 				BUS_SPACE_MAXADDR,	/* lowaddr */
354 				BUS_SPACE_MAXADDR,	/* highaddr */
355 				NULL, NULL,		/* filter, filterarg */
356 				BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
357 				sc->mfi_max_sge,	/* nsegments */
358 				BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
359 				BUS_DMA_ALLOCNOW,	/* flags */
360 				&sc->mfi_buffer_dmat)) {
361 		device_printf(sc->mfi_dev, "Cannot allocate buffer DMA tag\n");
362 		return (ENOMEM);
363 	}
364 
365 	/*
366 	 * Allocate DMA memory for the comms queues.  Keep it under 4GB for
367 	 * efficiency.  The mfi_hwcomms struct includes space for 1 reply queue
368 	 * entry, so the calculated size here will be will be 1 more than
369 	 * mfi_max_fw_cmds.  This is apparently a requirement of the hardware.
370 	 */
371 	commsz = (sizeof(uint32_t) * sc->mfi_max_fw_cmds) +
372 	    sizeof(struct mfi_hwcomms);
373 	if (bus_dma_tag_create( sc->mfi_parent_dmat,	/* parent */
374 				1, 0,			/* algnmnt, boundary */
375 				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
376 				BUS_SPACE_MAXADDR,	/* highaddr */
377 				NULL, NULL,		/* filter, filterarg */
378 				commsz,			/* maxsize */
379 				1,			/* msegments */
380 				commsz,			/* maxsegsize */
381 				0,			/* flags */
382 				&sc->mfi_comms_dmat)) {
383 		device_printf(sc->mfi_dev, "Cannot allocate comms DMA tag\n");
384 		return (ENOMEM);
385 	}
386 	if (bus_dmamem_alloc(sc->mfi_comms_dmat, (void **)&sc->mfi_comms,
387 	    BUS_DMA_NOWAIT, &sc->mfi_comms_dmamap)) {
388 		device_printf(sc->mfi_dev, "Cannot allocate comms memory\n");
389 		return (ENOMEM);
390 	}
391 	bzero(sc->mfi_comms, commsz);
392 	bus_dmamap_load(sc->mfi_comms_dmat, sc->mfi_comms_dmamap,
393 	    sc->mfi_comms, commsz, mfi_addr32_cb, &sc->mfi_comms_busaddr, 0);
394 
395 	/*
396 	 * Allocate DMA memory for the command frames.  Keep them in the
397 	 * lower 4GB for efficiency.  Calculate the size of the commands at
398 	 * the same time; each command is one 64 byte frame plus a set of
399          * additional frames for holding sg lists or other data.
400 	 * The assumption here is that the SG list will start at the second
401 	 * frame and not use the unused bytes in the first frame.  While this
402 	 * isn't technically correct, it simplifies the calculation and allows
403 	 * for command frames that might be larger than an mfi_io_frame.
404 	 */
405 	if (sizeof(bus_addr_t) == 8) {
406 		sc->mfi_sge_size = sizeof(struct mfi_sg64);
407 		sc->mfi_flags |= MFI_FLAGS_SG64;
408 	} else {
409 		sc->mfi_sge_size = sizeof(struct mfi_sg32);
410 	}
411 	frames = (sc->mfi_sge_size * sc->mfi_max_sge - 1) / MFI_FRAME_SIZE + 2;
412 	sc->mfi_cmd_size = frames * MFI_FRAME_SIZE;
413 	framessz = sc->mfi_cmd_size * sc->mfi_max_fw_cmds;
414 	if (bus_dma_tag_create( sc->mfi_parent_dmat,	/* parent */
415 				64, 0,			/* algnmnt, boundary */
416 				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
417 				BUS_SPACE_MAXADDR,	/* highaddr */
418 				NULL, NULL,		/* filter, filterarg */
419 				framessz,		/* maxsize */
420 				1,			/* nsegments */
421 				framessz,		/* maxsegsize */
422 				0,			/* flags */
423 				&sc->mfi_frames_dmat)) {
424 		device_printf(sc->mfi_dev, "Cannot allocate frame DMA tag\n");
425 		return (ENOMEM);
426 	}
427 	if (bus_dmamem_alloc(sc->mfi_frames_dmat, (void **)&sc->mfi_frames,
428 	    BUS_DMA_NOWAIT, &sc->mfi_frames_dmamap)) {
429 		device_printf(sc->mfi_dev, "Cannot allocate frames memory\n");
430 		return (ENOMEM);
431 	}
432 	bzero(sc->mfi_frames, framessz);
433 	bus_dmamap_load(sc->mfi_frames_dmat, sc->mfi_frames_dmamap,
434 	    sc->mfi_frames, framessz, mfi_addr32_cb, &sc->mfi_frames_busaddr,0);
435 
436 	/*
437 	 * Allocate DMA memory for the frame sense data.  Keep them in the
438 	 * lower 4GB for efficiency
439 	 */
440 	sensesz = sc->mfi_max_fw_cmds * MFI_SENSE_LEN;
441 	if (bus_dma_tag_create( sc->mfi_parent_dmat,	/* parent */
442 				4, 0,			/* algnmnt, boundary */
443 				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
444 				BUS_SPACE_MAXADDR,	/* highaddr */
445 				NULL, NULL,		/* filter, filterarg */
446 				sensesz,		/* maxsize */
447 				1,			/* nsegments */
448 				sensesz,		/* maxsegsize */
449 				0,			/* flags */
450 				&sc->mfi_sense_dmat)) {
451 		device_printf(sc->mfi_dev, "Cannot allocate sense DMA tag\n");
452 		return (ENOMEM);
453 	}
454 	if (bus_dmamem_alloc(sc->mfi_sense_dmat, (void **)&sc->mfi_sense,
455 	    BUS_DMA_NOWAIT, &sc->mfi_sense_dmamap)) {
456 		device_printf(sc->mfi_dev, "Cannot allocate sense memory\n");
457 		return (ENOMEM);
458 	}
459 	bus_dmamap_load(sc->mfi_sense_dmat, sc->mfi_sense_dmamap,
460 	    sc->mfi_sense, sensesz, mfi_addr32_cb, &sc->mfi_sense_busaddr, 0);
461 
462 	if ((error = mfi_alloc_commands(sc)) != 0)
463 		return (error);
464 
465 	if ((error = mfi_comms_init(sc)) != 0)
466 		return (error);
467 
468 	if ((error = mfi_get_controller_info(sc)) != 0)
469 		return (error);
470 
471 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
472 	if ((error = mfi_aen_setup(sc, 0), 0) != 0) {
473 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
474 		return (error);
475 	}
476 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
477 
478 	/*
479 	 * Set up the interrupt handler.  XXX This should happen in
480 	 * mfi_pci.c
481 	 */
482 	sc->mfi_irq_rid = 0;
483 	if ((sc->mfi_irq = bus_alloc_resource_any(sc->mfi_dev, SYS_RES_IRQ,
484 	    &sc->mfi_irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
485 		device_printf(sc->mfi_dev, "Cannot allocate interrupt\n");
486 		return (EINVAL);
487 	}
488 	if (bus_setup_intr(sc->mfi_dev, sc->mfi_irq, 0,
489 	    mfi_intr, sc, &sc->mfi_intr, NULL)) {
490 		device_printf(sc->mfi_dev, "Cannot set up interrupt\n");
491 		return (EINVAL);
492 	}
493 
494 	/* Register a config hook to probe the bus for arrays */
495 	sc->mfi_ich.ich_func = mfi_startup;
496 	sc->mfi_ich.ich_arg = sc;
497 	if (config_intrhook_establish(&sc->mfi_ich) != 0) {
498 		device_printf(sc->mfi_dev, "Cannot establish configuration "
499 		    "hook\n");
500 		return (EINVAL);
501 	}
502 
503 	/*
504 	 * Register a shutdown handler.
505 	 */
506 	if ((sc->mfi_eh = EVENTHANDLER_REGISTER(shutdown_final, mfi_shutdown,
507 	    sc, SHUTDOWN_PRI_DEFAULT)) == NULL) {
508 		device_printf(sc->mfi_dev, "Warning: shutdown event "
509 		    "registration failed\n");
510 	}
511 
512 	/*
513 	 * Create the control device for doing management
514 	 */
515 	unit = device_get_unit(sc->mfi_dev);
516 	sc->mfi_cdev = make_dev(&mfi_ops, unit, UID_ROOT, GID_OPERATOR,
517 	    0640, "mfi%d", unit);
518 	if (unit == 0)
519 		make_dev_alias(sc->mfi_cdev, "megaraid_sas_ioctl_node");
520 	if (sc->mfi_cdev != NULL)
521 		sc->mfi_cdev->si_drv1 = sc;
522 	sysctl_ctx_init(&sc->mfi_sysctl_ctx);
523 	sc->mfi_sysctl_tree = SYSCTL_ADD_NODE(&sc->mfi_sysctl_ctx,
524 	    SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
525 	    device_get_nameunit(sc->mfi_dev), CTLFLAG_RD, 0, "");
526 	if (sc->mfi_sysctl_tree == NULL) {
527 		device_printf(sc->mfi_dev, "can't add sysctl node\n");
528 		return (EINVAL);
529 	}
530 	SYSCTL_ADD_INT(&sc->mfi_sysctl_ctx,
531 	    SYSCTL_CHILDREN(sc->mfi_sysctl_tree),
532 	    OID_AUTO, "delete_busy_volumes", CTLFLAG_RW,
533 	    &sc->mfi_delete_busy_volumes, 0, "Allow removal of busy volumes");
534 	SYSCTL_ADD_INT(&sc->mfi_sysctl_ctx,
535 	    SYSCTL_CHILDREN(sc->mfi_sysctl_tree),
536 	    OID_AUTO, "keep_deleted_volumes", CTLFLAG_RW,
537 	    &sc->mfi_keep_deleted_volumes, 0,
538 	    "Don't detach the mfid device for a busy volume that is deleted");
539 
540 	device_add_child(sc->mfi_dev, "mfip", -1);
541 	bus_generic_attach(sc->mfi_dev);
542 
543 	/* Start the timeout watchdog */
544 	callout_init(&sc->mfi_watchdog_callout);
545 	callout_reset(&sc->mfi_watchdog_callout, MFI_CMD_TIMEOUT * hz,
546 	    mfi_timeout, sc);
547 
548 	return (0);
549 }
550 
551 static int
552 mfi_alloc_commands(struct mfi_softc *sc)
553 {
554 	struct mfi_command *cm;
555 	int i, ncmds;
556 
557 	/*
558 	 * XXX Should we allocate all the commands up front, or allocate on
559 	 * demand later like 'aac' does?
560 	 */
561 	ncmds = MIN(mfi_max_cmds, sc->mfi_max_fw_cmds);
562 	if (bootverbose)
563 		device_printf(sc->mfi_dev, "Max fw cmds= %d, sizing driver "
564 		   "pool to %d\n", sc->mfi_max_fw_cmds, ncmds);
565 
566 	sc->mfi_commands = kmalloc(sizeof(struct mfi_command) * ncmds, M_MFIBUF,
567 	    M_WAITOK | M_ZERO);
568 
569 	for (i = 0; i < ncmds; i++) {
570 		cm = &sc->mfi_commands[i];
571 		cm->cm_frame = (union mfi_frame *)((uintptr_t)sc->mfi_frames +
572 		    sc->mfi_cmd_size * i);
573 		cm->cm_frame_busaddr = sc->mfi_frames_busaddr +
574 		    sc->mfi_cmd_size * i;
575 		cm->cm_frame->header.context = i;
576 		cm->cm_sense = &sc->mfi_sense[i];
577 		cm->cm_sense_busaddr= sc->mfi_sense_busaddr + MFI_SENSE_LEN * i;
578 		cm->cm_sc = sc;
579 		cm->cm_index = i;
580 		if (bus_dmamap_create(sc->mfi_buffer_dmat, 0,
581 		    &cm->cm_dmamap) == 0)
582 			mfi_release_command(cm);
583 		else
584 			break;
585 		sc->mfi_total_cmds++;
586 	}
587 
588 	return (0);
589 }
590 
591 void
592 mfi_release_command(struct mfi_command *cm)
593 {
594 	struct mfi_frame_header *hdr;
595 	uint32_t *hdr_data;
596 
597 	/*
598 	 * Zero out the important fields of the frame, but make sure the
599 	 * context field is preserved.  For efficiency, handle the fields
600 	 * as 32 bit words.  Clear out the first S/G entry too for safety.
601 	 */
602 	hdr = &cm->cm_frame->header;
603 	if (cm->cm_data != NULL && hdr->sg_count) {
604 		cm->cm_sg->sg32[0].len = 0;
605 		cm->cm_sg->sg32[0].addr = 0;
606 	}
607 
608 	hdr_data = (uint32_t *)cm->cm_frame;
609 	hdr_data[0] = 0;	/* cmd, sense_len, cmd_status, scsi_status */
610 	hdr_data[1] = 0;	/* target_id, lun_id, cdb_len, sg_count */
611 	hdr_data[4] = 0;	/* flags, timeout */
612 	hdr_data[5] = 0;	/* data_len */
613 
614 	cm->cm_extra_frames = 0;
615 	cm->cm_flags = 0;
616 	cm->cm_complete = NULL;
617 	cm->cm_private = NULL;
618 	cm->cm_data = NULL;
619 	cm->cm_sg = 0;
620 	cm->cm_total_frame_size = 0;
621 
622 	mfi_enqueue_free(cm);
623 }
624 
625 static int
626 mfi_dcmd_command(struct mfi_softc *sc, struct mfi_command **cmp, uint32_t opcode,
627     void **bufp, size_t bufsize)
628 {
629 	struct mfi_command *cm;
630 	struct mfi_dcmd_frame *dcmd;
631 	void *buf = NULL;
632 
633 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
634 
635 	cm = mfi_dequeue_free(sc);
636 	if (cm == NULL)
637 		return (EBUSY);
638 
639 	if ((bufsize > 0) && (bufp != NULL)) {
640 		if (*bufp == NULL) {
641 			buf = kmalloc(bufsize, M_MFIBUF, M_NOWAIT|M_ZERO);
642 			if (buf == NULL) {
643 				mfi_release_command(cm);
644 				return (ENOMEM);
645 			}
646 			*bufp = buf;
647 		} else {
648 			buf = *bufp;
649 		}
650 	}
651 
652 	dcmd =  &cm->cm_frame->dcmd;
653 	bzero(dcmd->mbox, MFI_MBOX_SIZE);
654 	dcmd->header.cmd = MFI_CMD_DCMD;
655 	dcmd->header.timeout = 0;
656 	dcmd->header.flags = 0;
657 	dcmd->header.data_len = bufsize;
658 	dcmd->opcode = opcode;
659 	cm->cm_sg = &dcmd->sgl;
660 	cm->cm_total_frame_size = MFI_DCMD_FRAME_SIZE;
661 	cm->cm_flags = 0;
662 	cm->cm_data = buf;
663 	cm->cm_private = buf;
664 	cm->cm_len = bufsize;
665 
666 	*cmp = cm;
667 	if ((bufp != NULL) && (*bufp == NULL) && (buf != NULL))
668 		*bufp = buf;
669 	return (0);
670 }
671 
672 static int
673 mfi_comms_init(struct mfi_softc *sc)
674 {
675 	struct mfi_command *cm;
676 	struct mfi_init_frame *init;
677 	struct mfi_init_qinfo *qinfo;
678 	int error;
679 
680 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
681 	if ((cm = mfi_dequeue_free(sc)) == NULL)
682 		return (EBUSY);
683 
684 	/*
685 	 * Abuse the SG list area of the frame to hold the init_qinfo
686 	 * object;
687 	 */
688 	init = &cm->cm_frame->init;
689 	qinfo = (struct mfi_init_qinfo *)((uintptr_t)init + MFI_FRAME_SIZE);
690 
691 	bzero(qinfo, sizeof(struct mfi_init_qinfo));
692 	qinfo->rq_entries = sc->mfi_max_fw_cmds + 1;
693 	qinfo->rq_addr_lo = sc->mfi_comms_busaddr +
694 	    offsetof(struct mfi_hwcomms, hw_reply_q);
695 	qinfo->pi_addr_lo = sc->mfi_comms_busaddr +
696 	    offsetof(struct mfi_hwcomms, hw_pi);
697 	qinfo->ci_addr_lo = sc->mfi_comms_busaddr +
698 	    offsetof(struct mfi_hwcomms, hw_ci);
699 
700 	init->header.cmd = MFI_CMD_INIT;
701 	init->header.data_len = sizeof(struct mfi_init_qinfo);
702 	init->qinfo_new_addr_lo = cm->cm_frame_busaddr + MFI_FRAME_SIZE;
703 	cm->cm_data = NULL;
704 	cm->cm_flags = MFI_CMD_POLLED;
705 
706 	if ((error = mfi_mapcmd(sc, cm)) != 0) {
707 		device_printf(sc->mfi_dev, "failed to send init command\n");
708 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
709 		return (error);
710 	}
711 	mfi_release_command(cm);
712 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
713 
714 	return (0);
715 }
716 
717 static int
718 mfi_get_controller_info(struct mfi_softc *sc)
719 {
720 	struct mfi_command *cm = NULL;
721 	struct mfi_ctrl_info *ci = NULL;
722 	uint32_t max_sectors_1, max_sectors_2;
723 	int error;
724 
725 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
726 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_GETINFO,
727 	    (void **)&ci, sizeof(*ci));
728 	if (error)
729 		goto out;
730 	cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
731 
732 	if ((error = mfi_mapcmd(sc, cm)) != 0) {
733 		device_printf(sc->mfi_dev, "Failed to get controller info\n");
734 		sc->mfi_max_io = (sc->mfi_max_sge - 1) * PAGE_SIZE /
735 		    MFI_SECTOR_LEN;
736 		error = 0;
737 		goto out;
738 	}
739 
740 	bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
741 	    BUS_DMASYNC_POSTREAD);
742 	bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
743 
744 	max_sectors_1 = (1 << ci->stripe_sz_ops.min) * ci->max_strips_per_io;
745 	max_sectors_2 = ci->max_request_size;
746 	sc->mfi_max_io = min(max_sectors_1, max_sectors_2);
747 
748 out:
749 	if (ci)
750 		kfree(ci, M_MFIBUF);
751 	if (cm)
752 		mfi_release_command(cm);
753 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
754 	return (error);
755 }
756 
757 static int
758 mfi_get_log_state(struct mfi_softc *sc, struct mfi_evt_log_state **log_state)
759 {
760 	struct mfi_command *cm = NULL;
761 	int error;
762 
763 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_EVENT_GETINFO,
764 	    (void **)log_state, sizeof(**log_state));
765 	if (error)
766 		goto out;
767 	cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
768 
769 	if ((error = mfi_mapcmd(sc, cm)) != 0) {
770 		device_printf(sc->mfi_dev, "Failed to get log state\n");
771 		goto out;
772 	}
773 
774 	bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
775 	    BUS_DMASYNC_POSTREAD);
776 	bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
777 
778 out:
779 	if (cm)
780 		mfi_release_command(cm);
781 
782 	return (error);
783 }
784 
785 static int
786 mfi_aen_setup(struct mfi_softc *sc, uint32_t seq_start)
787 {
788 	struct mfi_evt_log_state *log_state = NULL;
789 	union mfi_evt class_locale;
790 	int error = 0;
791 	uint32_t seq;
792 
793 	class_locale.members.reserved = 0;
794 	class_locale.members.locale = mfi_event_locale;
795 	class_locale.members.class  = mfi_event_class;
796 
797 	if (seq_start == 0) {
798 		error = mfi_get_log_state(sc, &log_state);
799 		if (error) {
800 			if (log_state)
801 				kfree(log_state, M_MFIBUF);
802 			return (error);
803 		}
804 
805 		/*
806 		 * Walk through any events that fired since the last
807 		 * shutdown.
808 		 */
809 		mfi_parse_entries(sc, log_state->shutdown_seq_num,
810 		    log_state->newest_seq_num);
811 		seq = log_state->newest_seq_num;
812 	} else
813 		seq = seq_start;
814 	mfi_aen_register(sc, seq, class_locale.word);
815 	if (log_state != NULL)
816 		kfree(log_state, M_MFIBUF);
817 
818 	return 0;
819 }
820 
821 static int
822 mfi_wait_command(struct mfi_softc *sc, struct mfi_command *cm)
823 {
824 
825 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
826 	cm->cm_complete = NULL;
827 
828 
829 	/*
830 	 * MegaCli can issue a DCMD of 0.  In this case do nothing
831 	 * and return 0 to it as status
832 	 */
833 	if (cm->cm_frame->dcmd.opcode == 0) {
834 		cm->cm_frame->header.cmd_status = MFI_STAT_OK;
835 		cm->cm_error = 0;
836 		return (cm->cm_error);
837 	}
838 	mfi_enqueue_ready(cm);
839 	mfi_startio(sc);
840 	if ((cm->cm_flags & MFI_CMD_COMPLETED) == 0)
841 		lksleep(cm, &sc->mfi_io_lock, 0, "mfiwait", 0);
842 	return (cm->cm_error);
843 }
844 
845 void
846 mfi_free(struct mfi_softc *sc)
847 {
848 	struct mfi_command *cm;
849 	int i;
850 
851 #if 0 /* XXX swildner */
852 	callout_drain(&sc->mfi_watchdog_callout);
853 #endif
854 
855 	if (sc->mfi_cdev != NULL)
856 		destroy_dev(sc->mfi_cdev);
857 	dev_ops_remove_minor(&mfi_ops, device_get_unit(sc->mfi_dev));
858 
859 	if (sc->mfi_total_cmds != 0) {
860 		for (i = 0; i < sc->mfi_total_cmds; i++) {
861 			cm = &sc->mfi_commands[i];
862 			bus_dmamap_destroy(sc->mfi_buffer_dmat, cm->cm_dmamap);
863 		}
864 		kfree(sc->mfi_commands, M_MFIBUF);
865 	}
866 
867 	if (sc->mfi_intr)
868 		bus_teardown_intr(sc->mfi_dev, sc->mfi_irq, sc->mfi_intr);
869 	if (sc->mfi_irq != NULL)
870 		bus_release_resource(sc->mfi_dev, SYS_RES_IRQ, sc->mfi_irq_rid,
871 		    sc->mfi_irq);
872 
873 	if (sc->mfi_sense_busaddr != 0)
874 		bus_dmamap_unload(sc->mfi_sense_dmat, sc->mfi_sense_dmamap);
875 	if (sc->mfi_sense != NULL)
876 		bus_dmamem_free(sc->mfi_sense_dmat, sc->mfi_sense,
877 		    sc->mfi_sense_dmamap);
878 	if (sc->mfi_sense_dmat != NULL)
879 		bus_dma_tag_destroy(sc->mfi_sense_dmat);
880 
881 	if (sc->mfi_frames_busaddr != 0)
882 		bus_dmamap_unload(sc->mfi_frames_dmat, sc->mfi_frames_dmamap);
883 	if (sc->mfi_frames != NULL)
884 		bus_dmamem_free(sc->mfi_frames_dmat, sc->mfi_frames,
885 		    sc->mfi_frames_dmamap);
886 	if (sc->mfi_frames_dmat != NULL)
887 		bus_dma_tag_destroy(sc->mfi_frames_dmat);
888 
889 	if (sc->mfi_comms_busaddr != 0)
890 		bus_dmamap_unload(sc->mfi_comms_dmat, sc->mfi_comms_dmamap);
891 	if (sc->mfi_comms != NULL)
892 		bus_dmamem_free(sc->mfi_comms_dmat, sc->mfi_comms,
893 		    sc->mfi_comms_dmamap);
894 	if (sc->mfi_comms_dmat != NULL)
895 		bus_dma_tag_destroy(sc->mfi_comms_dmat);
896 
897 	if (sc->mfi_buffer_dmat != NULL)
898 		bus_dma_tag_destroy(sc->mfi_buffer_dmat);
899 	if (sc->mfi_parent_dmat != NULL)
900 		bus_dma_tag_destroy(sc->mfi_parent_dmat);
901 
902 	if (sc->mfi_sysctl_tree != NULL)
903 		sysctl_ctx_free(&sc->mfi_sysctl_ctx);
904 
905 #if 0 /* XXX swildner: not sure if we need something like mtx_initialized() */
906 
907 	if (mtx_initialized(&sc->mfi_io_lock)) {
908 		lockuninit(&sc->mfi_io_lock);
909 		sx_destroy(&sc->mfi_config_lock);
910 	}
911 #endif
912 
913 	lockuninit(&sc->mfi_io_lock);
914 	lockuninit(&sc->mfi_config_lock);
915 
916 	return;
917 }
918 
919 static void
920 mfi_startup(void *arg)
921 {
922 	struct mfi_softc *sc;
923 
924 	sc = (struct mfi_softc *)arg;
925 
926 	config_intrhook_disestablish(&sc->mfi_ich);
927 
928 	sc->mfi_enable_intr(sc);
929 	lockmgr(&sc->mfi_config_lock, LK_EXCLUSIVE);
930 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
931 	mfi_ldprobe(sc);
932 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
933 	lockmgr(&sc->mfi_config_lock, LK_RELEASE);
934 }
935 
936 static void
937 mfi_intr(void *arg)
938 {
939 	struct mfi_softc *sc;
940 	struct mfi_command *cm;
941 	uint32_t pi, ci, context;
942 
943 	sc = (struct mfi_softc *)arg;
944 
945 	if (sc->mfi_check_clear_intr(sc))
946 		return;
947 
948 	pi = sc->mfi_comms->hw_pi;
949 	ci = sc->mfi_comms->hw_ci;
950 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
951 	while (ci != pi) {
952 		context = sc->mfi_comms->hw_reply_q[ci];
953 		if (context < sc->mfi_max_fw_cmds) {
954 			cm = &sc->mfi_commands[context];
955 			mfi_remove_busy(cm);
956 			cm->cm_error = 0;
957 			mfi_complete(sc, cm);
958 		}
959 		if (++ci == (sc->mfi_max_fw_cmds + 1)) {
960 			ci = 0;
961 		}
962 	}
963 
964 	sc->mfi_comms->hw_ci = ci;
965 
966 	/* Give defered I/O a chance to run */
967 	if (sc->mfi_flags & MFI_FLAGS_QFRZN)
968 		sc->mfi_flags &= ~MFI_FLAGS_QFRZN;
969 	mfi_startio(sc);
970 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
971 
972 	return;
973 }
974 
975 int
976 mfi_shutdown(struct mfi_softc *sc)
977 {
978 	struct mfi_dcmd_frame *dcmd;
979 	struct mfi_command *cm;
980 	int error;
981 
982 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
983 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_SHUTDOWN, NULL, 0);
984 	if (error) {
985 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
986 		return (error);
987 	}
988 
989 	if (sc->mfi_aen_cm != NULL)
990 		mfi_abort(sc, sc->mfi_aen_cm);
991 
992 	dcmd = &cm->cm_frame->dcmd;
993 	dcmd->header.flags = MFI_FRAME_DIR_NONE;
994 	cm->cm_flags = MFI_CMD_POLLED;
995 	cm->cm_data = NULL;
996 
997 	if ((error = mfi_mapcmd(sc, cm)) != 0) {
998 		device_printf(sc->mfi_dev, "Failed to shutdown controller\n");
999 	}
1000 
1001 	mfi_release_command(cm);
1002 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1003 	return (error);
1004 }
1005 
1006 static void
1007 mfi_ldprobe(struct mfi_softc *sc)
1008 {
1009 	struct mfi_frame_header *hdr;
1010 	struct mfi_command *cm = NULL;
1011 	struct mfi_ld_list *list = NULL;
1012 	struct mfi_disk *ld;
1013 	int error, i;
1014 
1015 	KKASSERT(lockstatus(&sc->mfi_config_lock, curthread) != 0);
1016 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
1017 
1018 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_LD_GET_LIST,
1019 	    (void **)&list, sizeof(*list));
1020 	if (error)
1021 		goto out;
1022 
1023 	cm->cm_flags = MFI_CMD_DATAIN;
1024 	if (mfi_wait_command(sc, cm) != 0) {
1025 		device_printf(sc->mfi_dev, "Failed to get device listing\n");
1026 		goto out;
1027 	}
1028 
1029 	hdr = &cm->cm_frame->header;
1030 	if (hdr->cmd_status != MFI_STAT_OK) {
1031 		device_printf(sc->mfi_dev, "MFI_DCMD_LD_GET_LIST failed %x\n",
1032 		    hdr->cmd_status);
1033 		goto out;
1034 	}
1035 
1036 	for (i = 0; i < list->ld_count; i++) {
1037 		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
1038 			if (ld->ld_id == list->ld_list[i].ld.v.target_id)
1039 				goto skip_add;
1040 		}
1041 		mfi_add_ld(sc, list->ld_list[i].ld.v.target_id);
1042 	skip_add:;
1043 	}
1044 out:
1045 	if (list)
1046 		kfree(list, M_MFIBUF);
1047 	if (cm)
1048 		mfi_release_command(cm);
1049 
1050 	return;
1051 }
1052 
1053 /*
1054  * The timestamp is the number of seconds since 00:00 Jan 1, 2000.  If
1055  * the bits in 24-31 are all set, then it is the number of seconds since
1056  * boot.
1057  */
1058 static const char *
1059 format_timestamp(uint32_t timestamp)
1060 {
1061 	static char buffer[32];
1062 
1063 	if ((timestamp & 0xff000000) == 0xff000000)
1064 		ksnprintf(buffer, sizeof(buffer), "boot + %us", timestamp &
1065 		    0x00ffffff);
1066 	else
1067 		ksnprintf(buffer, sizeof(buffer), "%us", timestamp);
1068 	return (buffer);
1069 }
1070 
1071 static const char *
1072 format_class(int8_t class)
1073 {
1074 	static char buffer[6];
1075 
1076 	switch (class) {
1077 	case MFI_EVT_CLASS_DEBUG:
1078 		return ("debug");
1079 	case MFI_EVT_CLASS_PROGRESS:
1080 		return ("progress");
1081 	case MFI_EVT_CLASS_INFO:
1082 		return ("info");
1083 	case MFI_EVT_CLASS_WARNING:
1084 		return ("WARN");
1085 	case MFI_EVT_CLASS_CRITICAL:
1086 		return ("CRIT");
1087 	case MFI_EVT_CLASS_FATAL:
1088 		return ("FATAL");
1089 	case MFI_EVT_CLASS_DEAD:
1090 		return ("DEAD");
1091 	default:
1092 		ksnprintf(buffer, sizeof(buffer), "%d", class);
1093 		return (buffer);
1094 	}
1095 }
1096 
1097 static void
1098 mfi_decode_evt(struct mfi_softc *sc, struct mfi_evt_detail *detail)
1099 {
1100 
1101 	device_printf(sc->mfi_dev, "%d (%s/0x%04x/%s) - %s\n", detail->seq,
1102 	    format_timestamp(detail->time), detail->class.members.locale,
1103 	    format_class(detail->class.members.class), detail->description);
1104 }
1105 
1106 static int
1107 mfi_aen_register(struct mfi_softc *sc, int seq, int locale)
1108 {
1109 	struct mfi_command *cm;
1110 	struct mfi_dcmd_frame *dcmd;
1111 	union mfi_evt current_aen, prior_aen;
1112 	struct mfi_evt_detail *ed = NULL;
1113 	int error = 0;
1114 
1115 	current_aen.word = locale;
1116 	if (sc->mfi_aen_cm != NULL) {
1117 		prior_aen.word =
1118 		    ((uint32_t *)&sc->mfi_aen_cm->cm_frame->dcmd.mbox)[1];
1119 		if (prior_aen.members.class <= current_aen.members.class &&
1120 		    !((prior_aen.members.locale & current_aen.members.locale)
1121 		    ^current_aen.members.locale)) {
1122 			return (0);
1123 		} else {
1124 			prior_aen.members.locale |= current_aen.members.locale;
1125 			if (prior_aen.members.class
1126 			    < current_aen.members.class)
1127 				current_aen.members.class =
1128 				    prior_aen.members.class;
1129 			mfi_abort(sc, sc->mfi_aen_cm);
1130 		}
1131 	}
1132 
1133 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_EVENT_WAIT,
1134 	    (void **)&ed, sizeof(*ed));
1135 	if (error) {
1136 		goto out;
1137 	}
1138 
1139 	dcmd = &cm->cm_frame->dcmd;
1140 	((uint32_t *)&dcmd->mbox)[0] = seq;
1141 	((uint32_t *)&dcmd->mbox)[1] = locale;
1142 	cm->cm_flags = MFI_CMD_DATAIN;
1143 	cm->cm_complete = mfi_aen_complete;
1144 
1145 	sc->mfi_aen_cm = cm;
1146 
1147 	mfi_enqueue_ready(cm);
1148 	mfi_startio(sc);
1149 
1150 out:
1151 	return (error);
1152 }
1153 
1154 static void
1155 mfi_aen_complete(struct mfi_command *cm)
1156 {
1157 	struct mfi_frame_header *hdr;
1158 	struct mfi_softc *sc;
1159 	struct mfi_evt_detail *detail;
1160 	struct mfi_aen *mfi_aen_entry, *tmp;
1161 	int seq = 0, aborted = 0;
1162 
1163 	sc = cm->cm_sc;
1164 	hdr = &cm->cm_frame->header;
1165 
1166 	if (sc->mfi_aen_cm == NULL)
1167 		return;
1168 
1169 	if (sc->mfi_aen_cm->cm_aen_abort || hdr->cmd_status == 0xff) {
1170 		sc->mfi_aen_cm->cm_aen_abort = 0;
1171 		aborted = 1;
1172 	} else {
1173 		sc->mfi_aen_triggered = 1;
1174 		if (sc->mfi_poll_waiting) {
1175 			sc->mfi_poll_waiting = 0;
1176 			KNOTE(&sc->mfi_kq.ki_note, 0);
1177 		}
1178 		detail = cm->cm_data;
1179 		/*
1180 		 * XXX If this function is too expensive or is recursive, then
1181 		 * events should be put onto a queue and processed later.
1182 		 */
1183 		mfi_decode_evt(sc, detail);
1184 		seq = detail->seq + 1;
1185 		TAILQ_FOREACH_MUTABLE(mfi_aen_entry, &sc->mfi_aen_pids, aen_link, tmp) {
1186 			TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
1187 			    aen_link);
1188 			lwkt_gettoken(&proc_token);
1189 			ksignal(mfi_aen_entry->p, SIGIO);
1190 			lwkt_reltoken(&proc_token);
1191 			kfree(mfi_aen_entry, M_MFIBUF);
1192 		}
1193 	}
1194 
1195 	kfree(cm->cm_data, M_MFIBUF);
1196 	sc->mfi_aen_cm = NULL;
1197 	wakeup(&sc->mfi_aen_cm);
1198 	mfi_release_command(cm);
1199 
1200 	/* set it up again so the driver can catch more events */
1201 	if (!aborted) {
1202 		mfi_aen_setup(sc, seq);
1203 	}
1204 }
1205 
1206 #define MAX_EVENTS 15
1207 
1208 static int
1209 mfi_parse_entries(struct mfi_softc *sc, int start_seq, int stop_seq)
1210 {
1211 	struct mfi_command *cm;
1212 	struct mfi_dcmd_frame *dcmd;
1213 	struct mfi_evt_list *el;
1214 	union mfi_evt class_locale;
1215 	int error, i, seq, size;
1216 
1217 	class_locale.members.reserved = 0;
1218 	class_locale.members.locale = mfi_event_locale;
1219 	class_locale.members.class  = mfi_event_class;
1220 
1221 	size = sizeof(struct mfi_evt_list) + sizeof(struct mfi_evt_detail)
1222 		* (MAX_EVENTS - 1);
1223 	el = kmalloc(size, M_MFIBUF, M_NOWAIT | M_ZERO);
1224 	if (el == NULL)
1225 		return (ENOMEM);
1226 
1227 	for (seq = start_seq;;) {
1228 		if ((cm = mfi_dequeue_free(sc)) == NULL) {
1229 			kfree(el, M_MFIBUF);
1230 			return (EBUSY);
1231 		}
1232 
1233 		dcmd = &cm->cm_frame->dcmd;
1234 		bzero(dcmd->mbox, MFI_MBOX_SIZE);
1235 		dcmd->header.cmd = MFI_CMD_DCMD;
1236 		dcmd->header.timeout = 0;
1237 		dcmd->header.data_len = size;
1238 		dcmd->opcode = MFI_DCMD_CTRL_EVENT_GET;
1239 		((uint32_t *)&dcmd->mbox)[0] = seq;
1240 		((uint32_t *)&dcmd->mbox)[1] = class_locale.word;
1241 		cm->cm_sg = &dcmd->sgl;
1242 		cm->cm_total_frame_size = MFI_DCMD_FRAME_SIZE;
1243 		cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_POLLED;
1244 		cm->cm_data = el;
1245 		cm->cm_len = size;
1246 
1247 		if ((error = mfi_mapcmd(sc, cm)) != 0) {
1248 			device_printf(sc->mfi_dev,
1249 			    "Failed to get controller entries\n");
1250 			mfi_release_command(cm);
1251 			break;
1252 		}
1253 
1254 		bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
1255 		    BUS_DMASYNC_POSTREAD);
1256 		bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
1257 
1258 		if (dcmd->header.cmd_status == MFI_STAT_NOT_FOUND) {
1259 			mfi_release_command(cm);
1260 			break;
1261 		}
1262 		if (dcmd->header.cmd_status != MFI_STAT_OK) {
1263 			device_printf(sc->mfi_dev,
1264 			    "Error %d fetching controller entries\n",
1265 			    dcmd->header.cmd_status);
1266 			mfi_release_command(cm);
1267 			break;
1268 		}
1269 		mfi_release_command(cm);
1270 
1271 		for (i = 0; i < el->count; i++) {
1272 			/*
1273 			 * If this event is newer than 'stop_seq' then
1274 			 * break out of the loop.  Note that the log
1275 			 * is a circular buffer so we have to handle
1276 			 * the case that our stop point is earlier in
1277 			 * the buffer than our start point.
1278 			 */
1279 			if (el->event[i].seq >= stop_seq) {
1280 				if (start_seq <= stop_seq)
1281 					break;
1282 				else if (el->event[i].seq < start_seq)
1283 					break;
1284 			}
1285 			mfi_decode_evt(sc, &el->event[i]);
1286 		}
1287 		seq = el->event[el->count - 1].seq + 1;
1288 	}
1289 
1290 	kfree(el, M_MFIBUF);
1291 	return (0);
1292 }
1293 
1294 static int
1295 mfi_add_ld(struct mfi_softc *sc, int id)
1296 {
1297 	struct mfi_command *cm;
1298 	struct mfi_dcmd_frame *dcmd = NULL;
1299 	struct mfi_ld_info *ld_info = NULL;
1300 	int error;
1301 
1302 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
1303 
1304 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_LD_GET_INFO,
1305 	    (void **)&ld_info, sizeof(*ld_info));
1306 	if (error) {
1307 		device_printf(sc->mfi_dev,
1308 		    "Failed to allocate for MFI_DCMD_LD_GET_INFO %d\n", error);
1309 		if (ld_info)
1310 			kfree(ld_info, M_MFIBUF);
1311 		return (error);
1312 	}
1313 	cm->cm_flags = MFI_CMD_DATAIN;
1314 	dcmd = &cm->cm_frame->dcmd;
1315 	dcmd->mbox[0] = id;
1316 	if (mfi_wait_command(sc, cm) != 0) {
1317 		device_printf(sc->mfi_dev,
1318 		    "Failed to get logical drive: %d\n", id);
1319 		kfree(ld_info, M_MFIBUF);
1320 		return (0);
1321 	}
1322 
1323 	mfi_add_ld_complete(cm);
1324 	return (0);
1325 }
1326 
1327 static void
1328 mfi_add_ld_complete(struct mfi_command *cm)
1329 {
1330 	struct mfi_frame_header *hdr;
1331 	struct mfi_ld_info *ld_info;
1332 	struct mfi_softc *sc;
1333 	device_t child;
1334 
1335 	sc = cm->cm_sc;
1336 	hdr = &cm->cm_frame->header;
1337 	ld_info = cm->cm_private;
1338 
1339 	if (hdr->cmd_status != MFI_STAT_OK) {
1340 		kfree(ld_info, M_MFIBUF);
1341 		mfi_release_command(cm);
1342 		return;
1343 	}
1344 	mfi_release_command(cm);
1345 
1346 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1347 	get_mplock();
1348 	if ((child = device_add_child(sc->mfi_dev, "mfid", -1)) == NULL) {
1349 		device_printf(sc->mfi_dev, "Failed to add logical disk\n");
1350 		kfree(ld_info, M_MFIBUF);
1351 		rel_mplock();
1352 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1353 		return;
1354 	}
1355 
1356 	device_set_ivars(child, ld_info);
1357 	device_set_desc(child, "MFI Logical Disk");
1358 	bus_generic_attach(sc->mfi_dev);
1359 	rel_mplock();
1360 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1361 }
1362 
1363 static struct mfi_command *
1364 mfi_bio_command(struct mfi_softc *sc)
1365 {
1366 	struct mfi_io_frame *io;
1367 	struct mfi_command *cm;
1368 	struct bio *bio;
1369 	struct buf *bp;
1370 	struct mfi_disk *disk;
1371 	int flags, blkcount;
1372 
1373 	if ((cm = mfi_dequeue_free(sc)) == NULL)
1374 		return (NULL);
1375 
1376 	if ((bio = mfi_dequeue_bio(sc)) == NULL) {
1377 		mfi_release_command(cm);
1378 		return (NULL);
1379 	}
1380 
1381 	bp = bio->bio_buf;
1382 	io = &cm->cm_frame->io;
1383 	switch (bp->b_cmd & 0x03) {
1384 	case BUF_CMD_READ:
1385 		io->header.cmd = MFI_CMD_LD_READ;
1386 		flags = MFI_CMD_DATAIN;
1387 		break;
1388 	case BUF_CMD_WRITE:
1389 		io->header.cmd = MFI_CMD_LD_WRITE;
1390 		flags = MFI_CMD_DATAOUT;
1391 		break;
1392 	default:
1393 		panic("Invalid bio command");
1394 	}
1395 
1396 	/* Cheat with the sector length to avoid a non-constant division */
1397 	blkcount = (bp->b_bcount + MFI_SECTOR_LEN - 1) / MFI_SECTOR_LEN;
1398 	disk = bio->bio_driver_info;
1399 	io->header.target_id = disk->ld_id;
1400 	io->header.timeout = 0;
1401 	io->header.flags = 0;
1402 	io->header.sense_len = MFI_SENSE_LEN;
1403 	io->header.data_len = blkcount;
1404 	io->sense_addr_lo = cm->cm_sense_busaddr;
1405 	io->sense_addr_hi = 0;
1406 	io->lba_hi = ((bio->bio_offset / MFI_SECTOR_LEN) & 0xffffffff00000000) >> 32;
1407 	io->lba_lo = (bio->bio_offset / MFI_SECTOR_LEN) & 0xffffffff;
1408 	cm->cm_complete = mfi_bio_complete;
1409 	cm->cm_private = bio;
1410 	cm->cm_data = bp->b_data;
1411 	cm->cm_len = bp->b_bcount;
1412 	cm->cm_sg = &io->sgl;
1413 	cm->cm_total_frame_size = MFI_IO_FRAME_SIZE;
1414 	cm->cm_flags = flags;
1415 	return (cm);
1416 }
1417 
1418 static void
1419 mfi_bio_complete(struct mfi_command *cm)
1420 {
1421 	struct bio *bio;
1422 	struct buf *bp;
1423 	struct mfi_frame_header *hdr;
1424 	struct mfi_softc *sc;
1425 
1426 	bio = cm->cm_private;
1427 	bp = bio->bio_buf;
1428 	hdr = &cm->cm_frame->header;
1429 	sc = cm->cm_sc;
1430 
1431 	if ((hdr->cmd_status != 0) || (hdr->scsi_status != 0)) {
1432 		bp->b_flags |= B_ERROR;
1433 		bp->b_error = EIO;
1434 		device_printf(sc->mfi_dev, "I/O error, status= %d "
1435 		    "scsi_status= %d\n", hdr->cmd_status, hdr->scsi_status);
1436 		mfi_print_sense(cm->cm_sc, cm->cm_sense);
1437 	} else if (cm->cm_error != 0) {
1438 		bp->b_flags |= B_ERROR;
1439 	}
1440 
1441 	mfi_release_command(cm);
1442 	mfi_disk_complete(bio);
1443 }
1444 
1445 void
1446 mfi_startio(struct mfi_softc *sc)
1447 {
1448 	struct mfi_command *cm;
1449 	struct ccb_hdr *ccbh;
1450 
1451 	for (;;) {
1452 		/* Don't bother if we're short on resources */
1453 		if (sc->mfi_flags & MFI_FLAGS_QFRZN)
1454 			break;
1455 
1456 		/* Try a command that has already been prepared */
1457 		cm = mfi_dequeue_ready(sc);
1458 
1459 		if (cm == NULL) {
1460 			if ((ccbh = TAILQ_FIRST(&sc->mfi_cam_ccbq)) != NULL)
1461 				cm = sc->mfi_cam_start(ccbh);
1462 		}
1463 
1464 		/* Nope, so look for work on the bioq */
1465 		if (cm == NULL)
1466 			cm = mfi_bio_command(sc);
1467 
1468 		/* No work available, so exit */
1469 		if (cm == NULL)
1470 			break;
1471 
1472 		/* Send the command to the controller */
1473 		if (mfi_mapcmd(sc, cm) != 0) {
1474 			mfi_requeue_ready(cm);
1475 			break;
1476 		}
1477 	}
1478 }
1479 
1480 static int
1481 mfi_mapcmd(struct mfi_softc *sc, struct mfi_command *cm)
1482 {
1483 	int error, polled;
1484 
1485 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
1486 
1487 	if (cm->cm_data != NULL) {
1488 		polled = (cm->cm_flags & MFI_CMD_POLLED) ? BUS_DMA_NOWAIT : 0;
1489 		error = bus_dmamap_load(sc->mfi_buffer_dmat, cm->cm_dmamap,
1490 		    cm->cm_data, cm->cm_len, mfi_data_cb, cm, polled);
1491 		if (error == EINPROGRESS) {
1492 			sc->mfi_flags |= MFI_FLAGS_QFRZN;
1493 			return (0);
1494 		}
1495 	} else {
1496 		error = mfi_send_frame(sc, cm);
1497 	}
1498 
1499 	return (error);
1500 }
1501 
1502 static void
1503 mfi_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1504 {
1505 	struct mfi_frame_header *hdr;
1506 	struct mfi_command *cm;
1507 	union mfi_sgl *sgl;
1508 	struct mfi_softc *sc;
1509 	int i, dir;
1510 
1511 	cm = (struct mfi_command *)arg;
1512 	sc = cm->cm_sc;
1513 	hdr = &cm->cm_frame->header;
1514 	sgl = cm->cm_sg;
1515 
1516 	if (error) {
1517 		kprintf("error %d in callback\n", error);
1518 		cm->cm_error = error;
1519 		mfi_complete(sc, cm);
1520 		return;
1521 	}
1522 
1523 	if ((sc->mfi_flags & MFI_FLAGS_SG64) == 0) {
1524 		for (i = 0; i < nsegs; i++) {
1525 			sgl->sg32[i].addr = segs[i].ds_addr;
1526 			sgl->sg32[i].len = segs[i].ds_len;
1527 		}
1528 	} else {
1529 		for (i = 0; i < nsegs; i++) {
1530 			sgl->sg64[i].addr = segs[i].ds_addr;
1531 			sgl->sg64[i].len = segs[i].ds_len;
1532 		}
1533 		hdr->flags |= MFI_FRAME_SGL64;
1534 	}
1535 	hdr->sg_count = nsegs;
1536 
1537 	dir = 0;
1538 	if (cm->cm_flags & MFI_CMD_DATAIN) {
1539 		dir |= BUS_DMASYNC_PREREAD;
1540 		hdr->flags |= MFI_FRAME_DIR_READ;
1541 	}
1542 	if (cm->cm_flags & MFI_CMD_DATAOUT) {
1543 		dir |= BUS_DMASYNC_PREWRITE;
1544 		hdr->flags |= MFI_FRAME_DIR_WRITE;
1545 	}
1546 	bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap, dir);
1547 	cm->cm_flags |= MFI_CMD_MAPPED;
1548 
1549 	/*
1550 	 * Instead of calculating the total number of frames in the
1551 	 * compound frame, it's already assumed that there will be at
1552 	 * least 1 frame, so don't compensate for the modulo of the
1553 	 * following division.
1554 	 */
1555 	cm->cm_total_frame_size += (sc->mfi_sge_size * nsegs);
1556 	cm->cm_extra_frames = (cm->cm_total_frame_size - 1) / MFI_FRAME_SIZE;
1557 
1558 	mfi_send_frame(sc, cm);
1559 
1560 	return;
1561 }
1562 
1563 static int
1564 mfi_send_frame(struct mfi_softc *sc, struct mfi_command *cm)
1565 {
1566 	struct mfi_frame_header *hdr;
1567 	int tm = MFI_POLL_TIMEOUT_SECS * 1000;
1568 
1569 	hdr = &cm->cm_frame->header;
1570 
1571 	if ((cm->cm_flags & MFI_CMD_POLLED) == 0) {
1572 		cm->cm_timestamp = time_second;
1573 		mfi_enqueue_busy(cm);
1574 	} else {
1575 		hdr->cmd_status = 0xff;
1576 		hdr->flags |= MFI_FRAME_DONT_POST_IN_REPLY_QUEUE;
1577 	}
1578 
1579 	/*
1580 	 * The bus address of the command is aligned on a 64 byte boundary,
1581 	 * leaving the least 6 bits as zero.  For whatever reason, the
1582 	 * hardware wants the address shifted right by three, leaving just
1583 	 * 3 zero bits.  These three bits are then used as a prefetching
1584 	 * hint for the hardware to predict how many frames need to be
1585 	 * fetched across the bus.  If a command has more than 8 frames
1586 	 * then the 3 bits are set to 0x7 and the firmware uses other
1587 	 * information in the command to determine the total amount to fetch.
1588 	 * However, FreeBSD doesn't support I/O larger than 128K, so 8 frames
1589 	 * is enough for both 32bit and 64bit systems.
1590 	 */
1591 	if (cm->cm_extra_frames > 7)
1592 		cm->cm_extra_frames = 7;
1593 
1594 	sc->mfi_issue_cmd(sc,cm->cm_frame_busaddr,cm->cm_extra_frames);
1595 
1596 	if ((cm->cm_flags & MFI_CMD_POLLED) == 0)
1597 		return (0);
1598 
1599 	/* This is a polled command, so busy-wait for it to complete. */
1600 	while (hdr->cmd_status == 0xff) {
1601 		DELAY(1000);
1602 		tm -= 1;
1603 		if (tm <= 0)
1604 			break;
1605 	}
1606 
1607 	if (hdr->cmd_status == 0xff) {
1608 		device_printf(sc->mfi_dev, "Frame %p timed out "
1609 			      "command 0x%X\n", hdr, cm->cm_frame->dcmd.opcode);
1610 		return (ETIMEDOUT);
1611 	}
1612 
1613 	return (0);
1614 }
1615 
1616 static void
1617 mfi_complete(struct mfi_softc *sc, struct mfi_command *cm)
1618 {
1619 	int dir;
1620 
1621 	if ((cm->cm_flags & MFI_CMD_MAPPED) != 0) {
1622 		dir = 0;
1623 		if (cm->cm_flags & MFI_CMD_DATAIN)
1624 			dir |= BUS_DMASYNC_POSTREAD;
1625 		if (cm->cm_flags & MFI_CMD_DATAOUT)
1626 			dir |= BUS_DMASYNC_POSTWRITE;
1627 
1628 		bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap, dir);
1629 		bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
1630 		cm->cm_flags &= ~MFI_CMD_MAPPED;
1631 	}
1632 
1633 	cm->cm_flags |= MFI_CMD_COMPLETED;
1634 
1635 	if (cm->cm_complete != NULL)
1636 		cm->cm_complete(cm);
1637 	else
1638 		wakeup(cm);
1639 }
1640 
1641 static int
1642 mfi_abort(struct mfi_softc *sc, struct mfi_command *cm_abort)
1643 {
1644 	struct mfi_command *cm;
1645 	struct mfi_abort_frame *abort;
1646 	int i = 0;
1647 
1648 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
1649 
1650 	if ((cm = mfi_dequeue_free(sc)) == NULL) {
1651 		return (EBUSY);
1652 	}
1653 
1654 	abort = &cm->cm_frame->abort;
1655 	abort->header.cmd = MFI_CMD_ABORT;
1656 	abort->header.flags = 0;
1657 	abort->abort_context = cm_abort->cm_frame->header.context;
1658 	abort->abort_mfi_addr_lo = cm_abort->cm_frame_busaddr;
1659 	abort->abort_mfi_addr_hi = 0;
1660 	cm->cm_data = NULL;
1661 	cm->cm_flags = MFI_CMD_POLLED;
1662 
1663 	sc->mfi_aen_cm->cm_aen_abort = 1;
1664 	mfi_mapcmd(sc, cm);
1665 	mfi_release_command(cm);
1666 
1667 	while (i < 5 && sc->mfi_aen_cm != NULL) {
1668 		lksleep(&sc->mfi_aen_cm, &sc->mfi_io_lock, 0, "mfiabort", 5 * hz);
1669 		i++;
1670 	}
1671 
1672 	return (0);
1673 }
1674 
1675 int
1676 mfi_dump_blocks(struct mfi_softc *sc, int id, uint64_t lba, void *virt, int len)
1677 {
1678 	struct mfi_command *cm;
1679 	struct mfi_io_frame *io;
1680 	int error;
1681 
1682 	if ((cm = mfi_dequeue_free(sc)) == NULL)
1683 		return (EBUSY);
1684 
1685 	io = &cm->cm_frame->io;
1686 	io->header.cmd = MFI_CMD_LD_WRITE;
1687 	io->header.target_id = id;
1688 	io->header.timeout = 0;
1689 	io->header.flags = 0;
1690 	io->header.sense_len = MFI_SENSE_LEN;
1691 	io->header.data_len = (len + MFI_SECTOR_LEN - 1) / MFI_SECTOR_LEN;
1692 	io->sense_addr_lo = cm->cm_sense_busaddr;
1693 	io->sense_addr_hi = 0;
1694 	io->lba_hi = (lba & 0xffffffff00000000) >> 32;
1695 	io->lba_lo = lba & 0xffffffff;
1696 	cm->cm_data = virt;
1697 	cm->cm_len = len;
1698 	cm->cm_sg = &io->sgl;
1699 	cm->cm_total_frame_size = MFI_IO_FRAME_SIZE;
1700 	cm->cm_flags = MFI_CMD_POLLED | MFI_CMD_DATAOUT;
1701 
1702 	error = mfi_mapcmd(sc, cm);
1703 	bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
1704 	    BUS_DMASYNC_POSTWRITE);
1705 	bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
1706 	mfi_release_command(cm);
1707 
1708 	return (error);
1709 }
1710 
1711 static int
1712 mfi_open(struct dev_open_args *ap)
1713 {
1714 	cdev_t dev = ap->a_head.a_dev;
1715 	struct mfi_softc *sc;
1716 	int error;
1717 
1718 	sc = dev->si_drv1;
1719 
1720 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1721 	if (sc->mfi_detaching)
1722 		error = ENXIO;
1723 	else {
1724 		sc->mfi_flags |= MFI_FLAGS_OPEN;
1725 		error = 0;
1726 	}
1727 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1728 
1729 	return (error);
1730 }
1731 
1732 static int
1733 mfi_close(struct dev_close_args *ap)
1734 {
1735 	cdev_t dev = ap->a_head.a_dev;
1736 	struct mfi_softc *sc;
1737 	struct mfi_aen *mfi_aen_entry, *tmp;
1738 
1739 	sc = dev->si_drv1;
1740 
1741 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1742 	sc->mfi_flags &= ~MFI_FLAGS_OPEN;
1743 
1744 	TAILQ_FOREACH_MUTABLE(mfi_aen_entry, &sc->mfi_aen_pids, aen_link, tmp) {
1745 		if (mfi_aen_entry->p == curproc) {
1746 			TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
1747 			    aen_link);
1748 			kfree(mfi_aen_entry, M_MFIBUF);
1749 		}
1750 	}
1751 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1752 	return (0);
1753 }
1754 
1755 static int
1756 mfi_config_lock(struct mfi_softc *sc, uint32_t opcode)
1757 {
1758 
1759 	switch (opcode) {
1760 	case MFI_DCMD_LD_DELETE:
1761 	case MFI_DCMD_CFG_ADD:
1762 	case MFI_DCMD_CFG_CLEAR:
1763 		lockmgr(&sc->mfi_config_lock, LK_EXCLUSIVE);
1764 		return (1);
1765 	default:
1766 		return (0);
1767 	}
1768 }
1769 
1770 static void
1771 mfi_config_unlock(struct mfi_softc *sc, int locked)
1772 {
1773 
1774 	if (locked)
1775 		lockmgr(&sc->mfi_config_lock, LK_RELEASE);
1776 }
1777 
1778 /* Perform pre-issue checks on commands from userland and possibly veto them. */
1779 static int
1780 mfi_check_command_pre(struct mfi_softc *sc, struct mfi_command *cm)
1781 {
1782 	struct mfi_disk *ld, *ld2;
1783 	int error;
1784 
1785 	KKASSERT(lockstatus(&sc->mfi_io_lock, curthread) != 0);
1786 	error = 0;
1787 	switch (cm->cm_frame->dcmd.opcode) {
1788 	case MFI_DCMD_LD_DELETE:
1789 		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
1790 			if (ld->ld_id == cm->cm_frame->dcmd.mbox[0])
1791 				break;
1792 		}
1793 		if (ld == NULL)
1794 			error = ENOENT;
1795 		else
1796 			error = mfi_disk_disable(ld);
1797 		break;
1798 	case MFI_DCMD_CFG_CLEAR:
1799 		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
1800 			error = mfi_disk_disable(ld);
1801 			if (error)
1802 				break;
1803 		}
1804 		if (error) {
1805 			TAILQ_FOREACH(ld2, &sc->mfi_ld_tqh, ld_link) {
1806 				if (ld2 == ld)
1807 					break;
1808 				mfi_disk_enable(ld2);
1809 			}
1810 		}
1811 		break;
1812 	default:
1813 		break;
1814 	}
1815 	return (error);
1816 }
1817 
1818 /* Perform post-issue checks on commands from userland. */
1819 static void
1820 mfi_check_command_post(struct mfi_softc *sc, struct mfi_command *cm)
1821 {
1822 	struct mfi_disk *ld, *ldn;
1823 
1824 	switch (cm->cm_frame->dcmd.opcode) {
1825 	case MFI_DCMD_LD_DELETE:
1826 		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
1827 			if (ld->ld_id == cm->cm_frame->dcmd.mbox[0])
1828 				break;
1829 		}
1830 		KASSERT(ld != NULL, ("volume dissappeared"));
1831 		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
1832 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1833 			get_mplock();
1834 			device_delete_child(sc->mfi_dev, ld->ld_dev);
1835 			rel_mplock();
1836 			lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1837 		} else
1838 			mfi_disk_enable(ld);
1839 		break;
1840 	case MFI_DCMD_CFG_CLEAR:
1841 		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
1842 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1843 			get_mplock();
1844 			TAILQ_FOREACH_MUTABLE(ld, &sc->mfi_ld_tqh, ld_link, ldn) {
1845 				device_delete_child(sc->mfi_dev, ld->ld_dev);
1846 			}
1847 			rel_mplock();
1848 			lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1849 		} else {
1850 			TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link)
1851 				mfi_disk_enable(ld);
1852 		}
1853 		break;
1854 	case MFI_DCMD_CFG_ADD:
1855 		mfi_ldprobe(sc);
1856 		break;
1857 	case MFI_DCMD_CFG_FOREIGN_IMPORT:
1858 		mfi_ldprobe(sc);
1859 		break;
1860 	}
1861 }
1862 
1863 static int
1864 mfi_user_command(struct mfi_softc *sc, struct mfi_ioc_passthru *ioc)
1865 {
1866 	struct mfi_command *cm;
1867 	struct mfi_dcmd_frame *dcmd;
1868 	void *ioc_buf = NULL;
1869 	uint32_t context;
1870 	int error = 0, locked;
1871 
1872 
1873 	if (ioc->buf_size > 0) {
1874 		ioc_buf = kmalloc(ioc->buf_size, M_MFIBUF, M_WAITOK);
1875 		if (ioc_buf == NULL) {
1876 			return (ENOMEM);
1877 		}
1878 		error = copyin(ioc->buf, ioc_buf, ioc->buf_size);
1879 		if (error) {
1880 			device_printf(sc->mfi_dev, "failed to copyin\n");
1881 			kfree(ioc_buf, M_MFIBUF);
1882 			return (error);
1883 		}
1884 	}
1885 
1886 	locked = mfi_config_lock(sc, ioc->ioc_frame.opcode);
1887 
1888 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1889 	while ((cm = mfi_dequeue_free(sc)) == NULL)
1890 		lksleep(mfi_user_command, &sc->mfi_io_lock, 0, "mfiioc", hz);
1891 
1892 	/* Save context for later */
1893 	context = cm->cm_frame->header.context;
1894 
1895 	dcmd = &cm->cm_frame->dcmd;
1896 	bcopy(&ioc->ioc_frame, dcmd, sizeof(struct mfi_dcmd_frame));
1897 
1898 	cm->cm_sg = &dcmd->sgl;
1899 	cm->cm_total_frame_size = MFI_DCMD_FRAME_SIZE;
1900 	cm->cm_data = ioc_buf;
1901 	cm->cm_len = ioc->buf_size;
1902 
1903 	/* restore context */
1904 	cm->cm_frame->header.context = context;
1905 
1906 	/* Cheat since we don't know if we're writing or reading */
1907 	cm->cm_flags = MFI_CMD_DATAIN | MFI_CMD_DATAOUT;
1908 
1909 	error = mfi_check_command_pre(sc, cm);
1910 	if (error)
1911 		goto out;
1912 
1913 	error = mfi_wait_command(sc, cm);
1914 	if (error) {
1915 		device_printf(sc->mfi_dev, "ioctl failed %d\n", error);
1916 		goto out;
1917 	}
1918 	bcopy(dcmd, &ioc->ioc_frame, sizeof(struct mfi_dcmd_frame));
1919 	mfi_check_command_post(sc, cm);
1920 out:
1921 	mfi_release_command(cm);
1922 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1923 	mfi_config_unlock(sc, locked);
1924 	if (ioc->buf_size > 0)
1925 		error = copyout(ioc_buf, ioc->buf, ioc->buf_size);
1926 	if (ioc_buf)
1927 		kfree(ioc_buf, M_MFIBUF);
1928 	return (error);
1929 }
1930 
1931 #ifdef __x86_64__
1932 #define	PTRIN(p)		((void *)(uintptr_t)(p))
1933 #else
1934 #define	PTRIN(p)		(p)
1935 #endif
1936 
1937 static int
1938 mfi_ioctl(struct dev_ioctl_args *ap)
1939 {
1940 	cdev_t dev = ap->a_head.a_dev;
1941 	u_long cmd = ap->a_cmd;
1942 	int flag = ap->a_fflag;
1943 	caddr_t arg = ap->a_data;
1944 	struct mfi_softc *sc;
1945 	union mfi_statrequest *ms;
1946 	struct mfi_ioc_packet *ioc;
1947 #ifdef __x86_64__
1948 	struct mfi_ioc_packet32 *ioc32;
1949 #endif
1950 	struct mfi_ioc_aen *aen;
1951 	struct mfi_command *cm = NULL;
1952 	uint32_t context;
1953 	union mfi_sense_ptr sense_ptr;
1954 	uint8_t *data = NULL, *temp;
1955 	int i;
1956 	struct mfi_ioc_passthru *iop = (struct mfi_ioc_passthru *)arg;
1957 #ifdef __x86_64__
1958 	struct mfi_ioc_passthru32 *iop32 = (struct mfi_ioc_passthru32 *)arg;
1959 	struct mfi_ioc_passthru iop_swab;
1960 #endif
1961 	int error, locked;
1962 
1963 	sc = dev->si_drv1;
1964 	error = 0;
1965 
1966 	switch (cmd) {
1967 	case MFIIO_STATS:
1968 		ms = (union mfi_statrequest *)arg;
1969 		switch (ms->ms_item) {
1970 		case MFIQ_FREE:
1971 		case MFIQ_BIO:
1972 		case MFIQ_READY:
1973 		case MFIQ_BUSY:
1974 			bcopy(&sc->mfi_qstat[ms->ms_item], &ms->ms_qstat,
1975 			    sizeof(struct mfi_qstat));
1976 			break;
1977 		default:
1978 			error = ENOIOCTL;
1979 			break;
1980 		}
1981 		break;
1982 	case MFIIO_QUERY_DISK:
1983 	{
1984 		struct mfi_query_disk *qd;
1985 		struct mfi_disk *ld;
1986 
1987 		qd = (struct mfi_query_disk *)arg;
1988 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
1989 		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
1990 			if (ld->ld_id == qd->array_id)
1991 				break;
1992 		}
1993 		if (ld == NULL) {
1994 			qd->present = 0;
1995 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
1996 			return (0);
1997 		}
1998 		qd->present = 1;
1999 		if (ld->ld_flags & MFI_DISK_FLAGS_OPEN)
2000 			qd->open = 1;
2001 		bzero(qd->devname, SPECNAMELEN + 1);
2002 		ksnprintf(qd->devname, SPECNAMELEN, "mfid%d", ld->ld_unit);
2003 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2004 		break;
2005 	}
2006 	case MFI_CMD:
2007 #ifdef __x86_64__
2008 	case MFI_CMD32:
2009 #endif
2010 		{
2011 		devclass_t devclass;
2012 		ioc = (struct mfi_ioc_packet *)arg;
2013 		int adapter;
2014 
2015 		adapter = ioc->mfi_adapter_no;
2016 		if (device_get_unit(sc->mfi_dev) == 0 && adapter != 0) {
2017 			devclass = devclass_find("mfi");
2018 			sc = devclass_get_softc(devclass, adapter);
2019 		}
2020 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2021 		if ((cm = mfi_dequeue_free(sc)) == NULL) {
2022 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2023 			return (EBUSY);
2024 		}
2025 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2026 		locked = 0;
2027 
2028 		/*
2029 		 * save off original context since copying from user
2030 		 * will clobber some data
2031 		 */
2032 		context = cm->cm_frame->header.context;
2033 
2034 		bcopy(ioc->mfi_frame.raw, cm->cm_frame,
2035 		    2 * MFI_DCMD_FRAME_SIZE);  /* this isn't quite right */
2036 		cm->cm_total_frame_size = (sizeof(union mfi_sgl)
2037 		    * ioc->mfi_sge_count) + ioc->mfi_sgl_off;
2038 		if (ioc->mfi_sge_count) {
2039 			cm->cm_sg =
2040 			    (union mfi_sgl *)&cm->cm_frame->bytes[ioc->mfi_sgl_off];
2041 		}
2042 		cm->cm_flags = 0;
2043 		if (cm->cm_frame->header.flags & MFI_FRAME_DATAIN)
2044 			cm->cm_flags |= MFI_CMD_DATAIN;
2045 		if (cm->cm_frame->header.flags & MFI_FRAME_DATAOUT)
2046 			cm->cm_flags |= MFI_CMD_DATAOUT;
2047 		/* Legacy app shim */
2048 		if (cm->cm_flags == 0)
2049 			cm->cm_flags |= MFI_CMD_DATAIN | MFI_CMD_DATAOUT;
2050 		cm->cm_len = cm->cm_frame->header.data_len;
2051 		if (cm->cm_len &&
2052 		    (cm->cm_flags & (MFI_CMD_DATAIN | MFI_CMD_DATAOUT))) {
2053 			cm->cm_data = data = kmalloc(cm->cm_len, M_MFIBUF,
2054 			    M_WAITOK | M_ZERO);
2055 			if (cm->cm_data == NULL) {
2056 				device_printf(sc->mfi_dev, "Malloc failed\n");
2057 				goto out;
2058 			}
2059 		} else {
2060 			cm->cm_data = 0;
2061 		}
2062 
2063 		/* restore header context */
2064 		cm->cm_frame->header.context = context;
2065 
2066 		temp = data;
2067 		if (cm->cm_flags & MFI_CMD_DATAOUT) {
2068 			for (i = 0; i < ioc->mfi_sge_count; i++) {
2069 #ifdef __x86_64__
2070 				if (cmd == MFI_CMD) {
2071 					/* Native */
2072 					error = copyin(ioc->mfi_sgl[i].iov_base,
2073 					       temp,
2074 					       ioc->mfi_sgl[i].iov_len);
2075 				} else {
2076 					void *temp_convert;
2077 					/* 32bit */
2078 					ioc32 = (struct mfi_ioc_packet32 *)ioc;
2079 					temp_convert =
2080 					    PTRIN(ioc32->mfi_sgl[i].iov_base);
2081 					error = copyin(temp_convert,
2082 					       temp,
2083 					       ioc32->mfi_sgl[i].iov_len);
2084 				}
2085 #else
2086 				error = copyin(ioc->mfi_sgl[i].iov_base,
2087 				       temp,
2088 				       ioc->mfi_sgl[i].iov_len);
2089 #endif
2090 				if (error != 0) {
2091 					device_printf(sc->mfi_dev,
2092 					    "Copy in failed\n");
2093 					goto out;
2094 				}
2095 				temp = &temp[ioc->mfi_sgl[i].iov_len];
2096 			}
2097 		}
2098 
2099 		if (cm->cm_frame->header.cmd == MFI_CMD_DCMD)
2100 			locked = mfi_config_lock(sc, cm->cm_frame->dcmd.opcode);
2101 
2102 		if (cm->cm_frame->header.cmd == MFI_CMD_PD_SCSI_IO) {
2103 			cm->cm_frame->pass.sense_addr_lo = cm->cm_sense_busaddr;
2104 			cm->cm_frame->pass.sense_addr_hi = 0;
2105 		}
2106 
2107 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2108 		error = mfi_check_command_pre(sc, cm);
2109 		if (error) {
2110 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2111 			goto out;
2112 		}
2113 
2114 		if ((error = mfi_wait_command(sc, cm)) != 0) {
2115 			device_printf(sc->mfi_dev,
2116 			    "Controller polled failed\n");
2117 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2118 			goto out;
2119 		}
2120 
2121 		mfi_check_command_post(sc, cm);
2122 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2123 
2124 		temp = data;
2125 		if (cm->cm_flags & MFI_CMD_DATAIN) {
2126 			for (i = 0; i < ioc->mfi_sge_count; i++) {
2127 #ifdef __x86_64__
2128 				if (cmd == MFI_CMD) {
2129 					/* Native */
2130 					error = copyout(temp,
2131 						ioc->mfi_sgl[i].iov_base,
2132 						ioc->mfi_sgl[i].iov_len);
2133 				} else {
2134 					void *temp_convert;
2135 					/* 32bit */
2136 					ioc32 = (struct mfi_ioc_packet32 *)ioc;
2137 					temp_convert =
2138 					    PTRIN(ioc32->mfi_sgl[i].iov_base);
2139 					error = copyout(temp,
2140 						temp_convert,
2141 						ioc32->mfi_sgl[i].iov_len);
2142 				}
2143 #else
2144 				error = copyout(temp,
2145 					ioc->mfi_sgl[i].iov_base,
2146 					ioc->mfi_sgl[i].iov_len);
2147 #endif
2148 				if (error != 0) {
2149 					device_printf(sc->mfi_dev,
2150 					    "Copy out failed\n");
2151 					goto out;
2152 				}
2153 				temp = &temp[ioc->mfi_sgl[i].iov_len];
2154 			}
2155 		}
2156 
2157 		if (ioc->mfi_sense_len) {
2158 			/* get user-space sense ptr then copy out sense */
2159 			bcopy(&((struct mfi_ioc_packet*)arg)
2160 			    ->mfi_frame.raw[ioc->mfi_sense_off],
2161 			    &sense_ptr.sense_ptr_data[0],
2162 			    sizeof(sense_ptr.sense_ptr_data));
2163 #ifdef __x86_64__
2164 			if (cmd != MFI_CMD) {
2165 				/*
2166 				 * not 64bit native so zero out any address
2167 				 * over 32bit */
2168 				sense_ptr.addr.high = 0;
2169 			}
2170 #endif
2171 			error = copyout(cm->cm_sense, sense_ptr.user_space,
2172 			    ioc->mfi_sense_len);
2173 			if (error != 0) {
2174 				device_printf(sc->mfi_dev,
2175 				    "Copy out failed\n");
2176 				goto out;
2177 			}
2178 		}
2179 
2180 		ioc->mfi_frame.hdr.cmd_status = cm->cm_frame->header.cmd_status;
2181 out:
2182 		mfi_config_unlock(sc, locked);
2183 		if (data)
2184 			kfree(data, M_MFIBUF);
2185 		if (cm) {
2186 			lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2187 			mfi_release_command(cm);
2188 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2189 		}
2190 
2191 		break;
2192 		}
2193 	case MFI_SET_AEN:
2194 		aen = (struct mfi_ioc_aen *)arg;
2195 		error = mfi_aen_register(sc, aen->aen_seq_num,
2196 		    aen->aen_class_locale);
2197 
2198 		break;
2199 	case MFI_LINUX_CMD_2: /* Firmware Linux ioctl shim */
2200 		{
2201 			devclass_t devclass;
2202 			struct mfi_linux_ioc_packet l_ioc;
2203 			int adapter;
2204 
2205 			devclass = devclass_find("mfi");
2206 			if (devclass == NULL)
2207 				return (ENOENT);
2208 
2209 			error = copyin(arg, &l_ioc, sizeof(l_ioc));
2210 			if (error)
2211 				return (error);
2212 			adapter = l_ioc.lioc_adapter_no;
2213 			sc = devclass_get_softc(devclass, adapter);
2214 			if (sc == NULL)
2215 				return (ENOENT);
2216 			return (mfi_linux_ioctl_int(sc->mfi_cdev,
2217 			    cmd, arg, flag));
2218 			break;
2219 		}
2220 	case MFI_LINUX_SET_AEN_2: /* AEN Linux ioctl shim */
2221 		{
2222 			devclass_t devclass;
2223 			struct mfi_linux_ioc_aen l_aen;
2224 			int adapter;
2225 
2226 			devclass = devclass_find("mfi");
2227 			if (devclass == NULL)
2228 				return (ENOENT);
2229 
2230 			error = copyin(arg, &l_aen, sizeof(l_aen));
2231 			if (error)
2232 				return (error);
2233 			adapter = l_aen.laen_adapter_no;
2234 			sc = devclass_get_softc(devclass, adapter);
2235 			if (sc == NULL)
2236 				return (ENOENT);
2237 			return (mfi_linux_ioctl_int(sc->mfi_cdev,
2238 			    cmd, arg, flag));
2239 			break;
2240 		}
2241 #ifdef __x86_64__
2242 	case MFIIO_PASSTHRU32:
2243 		iop_swab.ioc_frame	= iop32->ioc_frame;
2244 		iop_swab.buf_size	= iop32->buf_size;
2245 		iop_swab.buf		= PTRIN(iop32->buf);
2246 		iop			= &iop_swab;
2247 		/* FALLTHROUGH */
2248 #endif
2249 	case MFIIO_PASSTHRU:
2250 		error = mfi_user_command(sc, iop);
2251 #ifdef __x86_64__
2252 		if (cmd == MFIIO_PASSTHRU32)
2253 			iop32->ioc_frame = iop_swab.ioc_frame;
2254 #endif
2255 		break;
2256 	default:
2257 		device_printf(sc->mfi_dev, "IOCTL 0x%lx not handled\n", cmd);
2258 		error = ENOENT;
2259 		break;
2260 	}
2261 
2262 	return (error);
2263 }
2264 
2265 static int
2266 mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag)
2267 {
2268 	struct mfi_softc *sc;
2269 	struct mfi_linux_ioc_packet l_ioc;
2270 	struct mfi_linux_ioc_aen l_aen;
2271 	struct mfi_command *cm = NULL;
2272 	struct mfi_aen *mfi_aen_entry;
2273 	union mfi_sense_ptr sense_ptr;
2274 	uint32_t context;
2275 	uint8_t *data = NULL, *temp;
2276 	int i;
2277 	int error, locked;
2278 
2279 	sc = dev->si_drv1;
2280 	error = 0;
2281 	switch (cmd) {
2282 	case MFI_LINUX_CMD_2: /* Firmware Linux ioctl shim */
2283 		error = copyin(arg, &l_ioc, sizeof(l_ioc));
2284 		if (error != 0)
2285 			return (error);
2286 
2287 		if (l_ioc.lioc_sge_count > MAX_LINUX_IOCTL_SGE) {
2288 			return (EINVAL);
2289 		}
2290 
2291 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2292 		if ((cm = mfi_dequeue_free(sc)) == NULL) {
2293 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2294 			return (EBUSY);
2295 		}
2296 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2297 		locked = 0;
2298 
2299 		/*
2300 		 * save off original context since copying from user
2301 		 * will clobber some data
2302 		 */
2303 		context = cm->cm_frame->header.context;
2304 
2305 		bcopy(l_ioc.lioc_frame.raw, cm->cm_frame,
2306 		      2 * MFI_DCMD_FRAME_SIZE);	/* this isn't quite right */
2307 		cm->cm_total_frame_size = (sizeof(union mfi_sgl)
2308 		      * l_ioc.lioc_sge_count) + l_ioc.lioc_sgl_off;
2309 		if (l_ioc.lioc_sge_count)
2310 			cm->cm_sg =
2311 			    (union mfi_sgl *)&cm->cm_frame->bytes[l_ioc.lioc_sgl_off];
2312 		cm->cm_flags = 0;
2313 		if (cm->cm_frame->header.flags & MFI_FRAME_DATAIN)
2314 			cm->cm_flags |= MFI_CMD_DATAIN;
2315 		if (cm->cm_frame->header.flags & MFI_FRAME_DATAOUT)
2316 			cm->cm_flags |= MFI_CMD_DATAOUT;
2317 		cm->cm_len = cm->cm_frame->header.data_len;
2318 		if (cm->cm_len &&
2319 		      (cm->cm_flags & (MFI_CMD_DATAIN | MFI_CMD_DATAOUT))) {
2320 			cm->cm_data = data = kmalloc(cm->cm_len, M_MFIBUF,
2321 			    M_WAITOK | M_ZERO);
2322 			if (cm->cm_data == NULL) {
2323 				device_printf(sc->mfi_dev, "Malloc failed\n");
2324 				goto out;
2325 			}
2326 		} else {
2327 			cm->cm_data = 0;
2328 		}
2329 
2330 		/* restore header context */
2331 		cm->cm_frame->header.context = context;
2332 
2333 		temp = data;
2334 		if (cm->cm_flags & MFI_CMD_DATAOUT) {
2335 			for (i = 0; i < l_ioc.lioc_sge_count; i++) {
2336 				error = copyin(PTRIN(l_ioc.lioc_sgl[i].iov_base),
2337 				       temp,
2338 				       l_ioc.lioc_sgl[i].iov_len);
2339 				if (error != 0) {
2340 					device_printf(sc->mfi_dev,
2341 					    "Copy in failed\n");
2342 					goto out;
2343 				}
2344 				temp = &temp[l_ioc.lioc_sgl[i].iov_len];
2345 			}
2346 		}
2347 
2348 		if (cm->cm_frame->header.cmd == MFI_CMD_DCMD)
2349 			locked = mfi_config_lock(sc, cm->cm_frame->dcmd.opcode);
2350 
2351 		if (cm->cm_frame->header.cmd == MFI_CMD_PD_SCSI_IO) {
2352 			cm->cm_frame->pass.sense_addr_lo = cm->cm_sense_busaddr;
2353 			cm->cm_frame->pass.sense_addr_hi = 0;
2354 		}
2355 
2356 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2357 		error = mfi_check_command_pre(sc, cm);
2358 		if (error) {
2359 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2360 			goto out;
2361 		}
2362 
2363 		if ((error = mfi_wait_command(sc, cm)) != 0) {
2364 			device_printf(sc->mfi_dev,
2365 			    "Controller polled failed\n");
2366 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2367 			goto out;
2368 		}
2369 
2370 		mfi_check_command_post(sc, cm);
2371 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2372 
2373 		temp = data;
2374 		if (cm->cm_flags & MFI_CMD_DATAIN) {
2375 			for (i = 0; i < l_ioc.lioc_sge_count; i++) {
2376 				error = copyout(temp,
2377 					PTRIN(l_ioc.lioc_sgl[i].iov_base),
2378 					l_ioc.lioc_sgl[i].iov_len);
2379 				if (error != 0) {
2380 					device_printf(sc->mfi_dev,
2381 					    "Copy out failed\n");
2382 					goto out;
2383 				}
2384 				temp = &temp[l_ioc.lioc_sgl[i].iov_len];
2385 			}
2386 		}
2387 
2388 		if (l_ioc.lioc_sense_len) {
2389 			/* get user-space sense ptr then copy out sense */
2390 			bcopy(&((struct mfi_linux_ioc_packet*)arg)
2391                             ->lioc_frame.raw[l_ioc.lioc_sense_off],
2392 			    &sense_ptr.sense_ptr_data[0],
2393 			    sizeof(sense_ptr.sense_ptr_data));
2394 #ifdef __x86_64__
2395 			/*
2396 			 * only 32bit Linux support so zero out any
2397 			 * address over 32bit
2398 			 */
2399 			sense_ptr.addr.high = 0;
2400 #endif
2401 			error = copyout(cm->cm_sense, sense_ptr.user_space,
2402 			    l_ioc.lioc_sense_len);
2403 			if (error != 0) {
2404 				device_printf(sc->mfi_dev,
2405 				    "Copy out failed\n");
2406 				goto out;
2407 			}
2408 		}
2409 
2410 		error = copyout(&cm->cm_frame->header.cmd_status,
2411 			&((struct mfi_linux_ioc_packet*)arg)
2412 			->lioc_frame.hdr.cmd_status,
2413 			1);
2414 		if (error != 0) {
2415 			device_printf(sc->mfi_dev,
2416 				      "Copy out failed\n");
2417 			goto out;
2418 		}
2419 
2420 out:
2421 		mfi_config_unlock(sc, locked);
2422 		if (data)
2423 			kfree(data, M_MFIBUF);
2424 		if (cm) {
2425 			lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2426 			mfi_release_command(cm);
2427 			lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2428 		}
2429 
2430 		return (error);
2431 	case MFI_LINUX_SET_AEN_2: /* AEN Linux ioctl shim */
2432 		error = copyin(arg, &l_aen, sizeof(l_aen));
2433 		if (error != 0)
2434 			return (error);
2435 		kprintf("AEN IMPLEMENTED for pid %d\n", curproc->p_pid);
2436 		mfi_aen_entry = kmalloc(sizeof(struct mfi_aen), M_MFIBUF,
2437 		    M_WAITOK);
2438 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2439 		if (mfi_aen_entry != NULL) {
2440 			mfi_aen_entry->p = curproc;
2441 			TAILQ_INSERT_TAIL(&sc->mfi_aen_pids, mfi_aen_entry,
2442 			    aen_link);
2443 		}
2444 		error = mfi_aen_register(sc, l_aen.laen_seq_num,
2445 		    l_aen.laen_class_locale);
2446 
2447 		if (error != 0) {
2448 			TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
2449 			    aen_link);
2450 			kfree(mfi_aen_entry, M_MFIBUF);
2451 		}
2452 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2453 
2454 		return (error);
2455 	default:
2456 		device_printf(sc->mfi_dev, "IOCTL 0x%lx not handled\n", cmd);
2457 		error = ENOENT;
2458 		break;
2459 	}
2460 
2461 	return (error);
2462 }
2463 
2464 static int
2465 mfi_kqfilter(struct dev_kqfilter_args *ap)
2466 {
2467 	cdev_t dev = ap->a_head.a_dev;
2468 	struct knote *kn = ap->a_kn;
2469 	struct mfi_softc *sc;
2470 	struct klist *klist;
2471 
2472 	ap->a_result = 0;
2473 	sc = dev->si_drv1;
2474 
2475 	switch (kn->kn_filter) {
2476 	case EVFILT_READ:
2477 		kn->kn_fop = &mfi_read_filterops;
2478 		kn->kn_hook = (caddr_t)sc;
2479 		break;
2480 	case EVFILT_WRITE:
2481 		kn->kn_fop = &mfi_write_filterops;
2482 		kn->kn_hook = (caddr_t)sc;
2483 		break;
2484 	default:
2485 		ap->a_result = EOPNOTSUPP;
2486 		return (0);
2487 	}
2488 
2489 	klist = &sc->mfi_kq.ki_note;
2490 	knote_insert(klist, kn);
2491 
2492 	return(0);
2493 }
2494 
2495 static void
2496 mfi_filter_detach(struct knote *kn)
2497 {
2498 	struct mfi_softc *sc = (struct mfi_softc *)kn->kn_hook;
2499 	struct klist *klist = &sc->mfi_kq.ki_note;
2500 
2501 	knote_remove(klist, kn);
2502 }
2503 
2504 static int
2505 mfi_filter_read(struct knote *kn, long hint)
2506 {
2507 	struct mfi_softc *sc = (struct mfi_softc *)kn->kn_hook;
2508 	int ready = 0;
2509 
2510 	if (sc->mfi_aen_triggered != 0) {
2511 		ready = 1;
2512 		sc->mfi_aen_triggered = 0;
2513 	}
2514 	if (sc->mfi_aen_triggered == 0 && sc->mfi_aen_cm == NULL)
2515 		kn->kn_flags |= EV_ERROR;
2516 
2517 	if (ready == 0)
2518 		sc->mfi_poll_waiting = 1;
2519 
2520 	return (ready);
2521 }
2522 
2523 static int
2524 mfi_filter_write(struct knote *kn, long hint)
2525 {
2526 	return (0);
2527 }
2528 
2529 static void
2530 mfi_dump_all(void)
2531 {
2532 	struct mfi_softc *sc;
2533 	struct mfi_command *cm;
2534 	devclass_t dc;
2535 	time_t deadline;
2536 	int timedout;
2537 	int i;
2538 
2539 	dc = devclass_find("mfi");
2540 	if (dc == NULL) {
2541 		kprintf("No mfi dev class\n");
2542 		return;
2543 	}
2544 
2545 	for (i = 0; ; i++) {
2546 		sc = devclass_get_softc(dc, i);
2547 		if (sc == NULL)
2548 			break;
2549 		device_printf(sc->mfi_dev, "Dumping\n\n");
2550 		timedout = 0;
2551 		deadline = time_second - MFI_CMD_TIMEOUT;
2552 		lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2553 		TAILQ_FOREACH(cm, &sc->mfi_busy, cm_link) {
2554 			if (cm->cm_timestamp < deadline) {
2555 				device_printf(sc->mfi_dev,
2556 				    "COMMAND %p TIMEOUT AFTER %d SECONDS\n", cm,
2557 				    (int)(time_second - cm->cm_timestamp));
2558 				MFI_PRINT_CMD(cm);
2559 				timedout++;
2560 			}
2561 		}
2562 
2563 #if 0
2564 		if (timedout)
2565 			MFI_DUMP_CMDS(SC);
2566 #endif
2567 
2568 		lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2569 	}
2570 
2571 	return;
2572 }
2573 
2574 static void
2575 mfi_timeout(void *data)
2576 {
2577 	struct mfi_softc *sc = (struct mfi_softc *)data;
2578 	struct mfi_command *cm;
2579 	time_t deadline;
2580 	int timedout = 0;
2581 
2582 	deadline = time_second - MFI_CMD_TIMEOUT;
2583 	lockmgr(&sc->mfi_io_lock, LK_EXCLUSIVE);
2584 	TAILQ_FOREACH(cm, &sc->mfi_busy, cm_link) {
2585 		if (sc->mfi_aen_cm == cm)
2586 			continue;
2587 		if ((sc->mfi_aen_cm != cm) && (cm->cm_timestamp < deadline)) {
2588 			device_printf(sc->mfi_dev,
2589 			    "COMMAND %p TIMEOUT AFTER %d SECONDS\n", cm,
2590 			    (int)(time_second - cm->cm_timestamp));
2591 			MFI_PRINT_CMD(cm);
2592 			MFI_VALIDATE_CMD(sc, cm);
2593 			timedout++;
2594 		}
2595 	}
2596 
2597 #if 0
2598 	if (timedout)
2599 		MFI_DUMP_CMDS(SC);
2600 #endif
2601 
2602 	lockmgr(&sc->mfi_io_lock, LK_RELEASE);
2603 
2604 	callout_reset(&sc->mfi_watchdog_callout, MFI_CMD_TIMEOUT * hz,
2605 	    mfi_timeout, sc);
2606 
2607 	if (0)
2608 		mfi_dump_all();
2609 	return;
2610 }
2611