xref: /dragonfly/sys/dev/disk/mpt/mpt_raid.c (revision bcb3e04d)
1 /*-
2  * Routines for handling the integrated RAID features LSI MPT Fusion adapters.
3  *
4  * Copyright (c) 2005, WHEEL Sp. z o.o.
5  * Copyright (c) 2005 Justin T. Gibbs.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions are
10  * met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
14  *    substantially similar to the "NO WARRANTY" disclaimer below
15  *    ("Disclaimer") and any redistribution must be conditioned upon including
16  *    a substantially similar Disclaimer requirement for further binary
17  *    redistribution.
18  * 3. Neither the names of the above listed copyright holders nor the names
19  *    of any contributors may be used to endorse or promote products derived
20  *    from this software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF THE COPYRIGHT
32  * OWNER OR CONTRIBUTOR IS ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 /*-
35  * Some Breakage and Bug Fixing added later.
36  * Copyright (c) 2006, by Matthew Jacob
37  * All Rights Reserved
38  *
39  * Support from LSI-Logic has also gone a great deal toward making this a
40  * workable subsystem and is gratefully acknowledged.
41  *
42  * $FreeBSD: src/sys/dev/mpt/mpt_raid.c,v 1.20 2009/05/21 12:36:40 jhb Exp $
43  */
44 
45 #include <dev/disk/mpt/mpt.h>
46 #include <dev/disk/mpt/mpt_raid.h>
47 
48 #include "dev/disk/mpt/mpilib/mpi_ioc.h" /* XXX Fix Event Handling!!! */
49 #include "dev/disk/mpt/mpilib/mpi_raid.h"
50 
51 #include <bus/cam/cam.h>
52 #include <bus/cam/cam_ccb.h>
53 #include <bus/cam/cam_sim.h>
54 #include <bus/cam/cam_xpt_periph.h>
55 #include <bus/cam/cam_xpt_sim.h>
56 
57 #if __FreeBSD_version < 500000
58 #include <sys/devicestat.h>
59 #define	GIANT_REQUIRED
60 #endif
61 #include <bus/cam/cam_periph.h>
62 
63 #include <sys/callout.h>
64 #include <sys/kthread.h>
65 #include <sys/sysctl.h>
66 
67 #include <machine/stdarg.h>
68 
69 struct mpt_raid_action_result
70 {
71 	union {
72 		MPI_RAID_VOL_INDICATOR	indicator_struct;
73 		uint32_t		new_settings;
74 		uint8_t			phys_disk_num;
75 	} action_data;
76 	uint16_t			action_status;
77 };
78 
79 #define REQ_TO_RAID_ACTION_RESULT(req) ((struct mpt_raid_action_result *) \
80 	(((MSG_RAID_ACTION_REQUEST *)(req->req_vbuf)) + 1))
81 
82 #define REQ_IOCSTATUS(req) ((req)->IOCStatus & MPI_IOCSTATUS_MASK)
83 
84 
85 static mpt_probe_handler_t	mpt_raid_probe;
86 static mpt_attach_handler_t	mpt_raid_attach;
87 static mpt_enable_handler_t	mpt_raid_enable;
88 static mpt_event_handler_t	mpt_raid_event;
89 static mpt_shutdown_handler_t	mpt_raid_shutdown;
90 static mpt_reset_handler_t	mpt_raid_ioc_reset;
91 static mpt_detach_handler_t	mpt_raid_detach;
92 
93 static struct mpt_personality mpt_raid_personality =
94 {
95 	.name		= "mpt_raid",
96 	.probe		= mpt_raid_probe,
97 	.attach		= mpt_raid_attach,
98 	.enable		= mpt_raid_enable,
99 	.event		= mpt_raid_event,
100 	.reset		= mpt_raid_ioc_reset,
101 	.shutdown	= mpt_raid_shutdown,
102 	.detach		= mpt_raid_detach,
103 };
104 
105 DECLARE_MPT_PERSONALITY(mpt_raid, SI_ORDER_THIRD);
106 MPT_PERSONALITY_DEPEND(mpt_raid, mpt_cam, 1, 1, 1);
107 
108 static mpt_reply_handler_t mpt_raid_reply_handler;
109 static int mpt_raid_reply_frame_handler(struct mpt_softc *mpt, request_t *req,
110 					MSG_DEFAULT_REPLY *reply_frame);
111 static int mpt_spawn_raid_thread(struct mpt_softc *mpt);
112 static void mpt_terminate_raid_thread(struct mpt_softc *mpt);
113 static void mpt_raid_thread(void *arg);
114 static timeout_t mpt_raid_timer;
115 #if 0
116 static void mpt_enable_vol(struct mpt_softc *mpt,
117 			   struct mpt_raid_volume *mpt_vol, int enable);
118 #endif
119 static void mpt_verify_mwce(struct mpt_softc *, struct mpt_raid_volume *);
120 static void mpt_adjust_queue_depth(struct mpt_softc *, struct mpt_raid_volume *,
121     struct cam_path *);
122 #if __FreeBSD_version < 500000
123 #define	mpt_raid_sysctl_attach(x)	do { } while (0)
124 #else
125 static void mpt_raid_sysctl_attach(struct mpt_softc *);
126 #endif
127 
128 static uint32_t raid_handler_id = MPT_HANDLER_ID_NONE;
129 
130 const char *
131 mpt_vol_type(struct mpt_raid_volume *vol)
132 {
133 	switch (vol->config_page->VolumeType) {
134 	case MPI_RAID_VOL_TYPE_IS:
135 		return ("RAID-0");
136 	case MPI_RAID_VOL_TYPE_IME:
137 		return ("RAID-1E");
138 	case MPI_RAID_VOL_TYPE_IM:
139 		return ("RAID-1");
140 	default:
141 		return ("Unknown");
142 	}
143 }
144 
145 const char *
146 mpt_vol_state(struct mpt_raid_volume *vol)
147 {
148 	switch (vol->config_page->VolumeStatus.State) {
149 	case MPI_RAIDVOL0_STATUS_STATE_OPTIMAL:
150 		return ("Optimal");
151 	case MPI_RAIDVOL0_STATUS_STATE_DEGRADED:
152 		return ("Degraded");
153 	case MPI_RAIDVOL0_STATUS_STATE_FAILED:
154 		return ("Failed");
155 	default:
156 		return ("Unknown");
157 	}
158 }
159 
160 const char *
161 mpt_disk_state(struct mpt_raid_disk *disk)
162 {
163 	switch (disk->config_page.PhysDiskStatus.State) {
164 	case MPI_PHYSDISK0_STATUS_ONLINE:
165 		return ("Online");
166 	case MPI_PHYSDISK0_STATUS_MISSING:
167 		return ("Missing");
168 	case MPI_PHYSDISK0_STATUS_NOT_COMPATIBLE:
169 		return ("Incompatible");
170 	case MPI_PHYSDISK0_STATUS_FAILED:
171 		return ("Failed");
172 	case MPI_PHYSDISK0_STATUS_INITIALIZING:
173 		return ("Initializing");
174 	case MPI_PHYSDISK0_STATUS_OFFLINE_REQUESTED:
175 		return ("Offline Requested");
176 	case MPI_PHYSDISK0_STATUS_FAILED_REQUESTED:
177 		return ("Failed per Host Request");
178 	case MPI_PHYSDISK0_STATUS_OTHER_OFFLINE:
179 		return ("Offline");
180 	default:
181 		return ("Unknown");
182 	}
183 }
184 
185 void
186 mpt_vol_prt(struct mpt_softc *mpt, struct mpt_raid_volume *vol,
187 	    const char *fmt, ...)
188 {
189 	__va_list ap;
190 
191 	kprintf("%s:vol%d(%s:%d:%d): ", device_get_nameunit(mpt->dev),
192 	       (u_int)(vol - mpt->raid_volumes), device_get_nameunit(mpt->dev),
193 	       vol->config_page->VolumeBus, vol->config_page->VolumeID);
194 	__va_start(ap, fmt);
195 	kvprintf(fmt, ap);
196 	__va_end(ap);
197 }
198 
199 void
200 mpt_disk_prt(struct mpt_softc *mpt, struct mpt_raid_disk *disk,
201 	     const char *fmt, ...)
202 {
203 	__va_list ap;
204 
205 	if (disk->volume != NULL) {
206 		kprintf("(%s:vol%d:%d): ",
207 		       device_get_nameunit(mpt->dev),
208 		       disk->volume->config_page->VolumeID,
209 		       disk->member_number);
210 	} else {
211 		kprintf("(%s:%d:%d): ", device_get_nameunit(mpt->dev),
212 		       disk->config_page.PhysDiskBus,
213 		       disk->config_page.PhysDiskID);
214 	}
215 	__va_start(ap, fmt);
216 	kvprintf(fmt, ap);
217 	__va_end(ap);
218 }
219 
220 static void
221 mpt_raid_async(void *callback_arg, u_int32_t code,
222 	       struct cam_path *path, void *arg)
223 {
224 	struct mpt_softc *mpt;
225 
226 	mpt = (struct mpt_softc*)callback_arg;
227 	switch (code) {
228 	case AC_FOUND_DEVICE:
229 	{
230 		struct ccb_getdev *cgd;
231 		struct mpt_raid_volume *mpt_vol;
232 
233 		cgd = (struct ccb_getdev *)arg;
234 		if (cgd == NULL) {
235 			break;
236 		}
237 
238 		mpt_lprt(mpt, MPT_PRT_DEBUG, "Callback for %d\n",
239 			 cgd->ccb_h.target_id);
240 
241 		RAID_VOL_FOREACH(mpt, mpt_vol) {
242 			if ((mpt_vol->flags & MPT_RVF_ACTIVE) == 0)
243 				continue;
244 
245 			if (mpt_vol->config_page->VolumeID
246 			 == cgd->ccb_h.target_id) {
247 				mpt_adjust_queue_depth(mpt, mpt_vol, path);
248 				break;
249 			}
250 		}
251 	}
252 	default:
253 		break;
254 	}
255 }
256 
257 int
258 mpt_raid_probe(struct mpt_softc *mpt)
259 {
260 	if (mpt->ioc_page2 == NULL || mpt->ioc_page2->MaxPhysDisks == 0) {
261 		return (ENODEV);
262 	}
263 	return (0);
264 }
265 
266 int
267 mpt_raid_attach(struct mpt_softc *mpt)
268 {
269 	struct ccb_setasync csa;
270 	mpt_handler_t	 handler;
271 	int		 error;
272 
273 	mpt_callout_init(&mpt->raid_timer);
274 
275 	error = mpt_spawn_raid_thread(mpt);
276 	if (error != 0) {
277 		mpt_prt(mpt, "Unable to spawn RAID thread!\n");
278 		goto cleanup;
279 	}
280 
281 	MPT_LOCK(mpt);
282 	handler.reply_handler = mpt_raid_reply_handler;
283 	error = mpt_register_handler(mpt, MPT_HANDLER_REPLY, handler,
284 				     &raid_handler_id);
285 	if (error != 0) {
286 		mpt_prt(mpt, "Unable to register RAID haandler!\n");
287 		goto cleanup;
288 	}
289 
290 	xpt_setup_ccb(&csa.ccb_h, mpt->path, 5);
291 	csa.ccb_h.func_code = XPT_SASYNC_CB;
292 	csa.event_enable = AC_FOUND_DEVICE;
293 	csa.callback = mpt_raid_async;
294 	csa.callback_arg = mpt;
295 	xpt_action((union ccb *)&csa);
296 	if (csa.ccb_h.status != CAM_REQ_CMP) {
297 		mpt_prt(mpt, "mpt_raid_attach: Unable to register "
298 			"CAM async handler.\n");
299 	}
300 	MPT_UNLOCK(mpt);
301 
302 	mpt_raid_sysctl_attach(mpt);
303 	return (0);
304 cleanup:
305 	MPT_UNLOCK(mpt);
306 	mpt_raid_detach(mpt);
307 	return (error);
308 }
309 
310 int
311 mpt_raid_enable(struct mpt_softc *mpt)
312 {
313 	return (0);
314 }
315 
316 void
317 mpt_raid_detach(struct mpt_softc *mpt)
318 {
319 	struct ccb_setasync csa;
320 	mpt_handler_t handler;
321 
322 	callout_stop(&mpt->raid_timer);
323 	MPT_LOCK(mpt);
324 	mpt_terminate_raid_thread(mpt);
325 
326 	handler.reply_handler = mpt_raid_reply_handler;
327 	mpt_deregister_handler(mpt, MPT_HANDLER_REPLY, handler,
328 			       raid_handler_id);
329 	xpt_setup_ccb(&csa.ccb_h, mpt->path, /*priority*/5);
330 	csa.ccb_h.func_code = XPT_SASYNC_CB;
331 	csa.event_enable = 0;
332 	csa.callback = mpt_raid_async;
333 	csa.callback_arg = mpt;
334 	xpt_action((union ccb *)&csa);
335 	MPT_UNLOCK(mpt);
336 }
337 
338 static void
339 mpt_raid_ioc_reset(struct mpt_softc *mpt, int type)
340 {
341 	/* Nothing to do yet. */
342 }
343 
344 static const char *raid_event_txt[] =
345 {
346 	"Volume Created",
347 	"Volume Deleted",
348 	"Volume Settings Changed",
349 	"Volume Status Changed",
350 	"Volume Physical Disk Membership Changed",
351 	"Physical Disk Created",
352 	"Physical Disk Deleted",
353 	"Physical Disk Settings Changed",
354 	"Physical Disk Status Changed",
355 	"Domain Validation Required",
356 	"SMART Data Received",
357 	"Replace Action Started",
358 };
359 
360 static int
361 mpt_raid_event(struct mpt_softc *mpt, request_t *req,
362 	       MSG_EVENT_NOTIFY_REPLY *msg)
363 {
364 	EVENT_DATA_RAID *raid_event;
365 	struct mpt_raid_volume *mpt_vol;
366 	struct mpt_raid_disk *mpt_disk;
367 	CONFIG_PAGE_RAID_VOL_0 *vol_pg;
368 	int i;
369 	int print_event;
370 
371 	if (msg->Event != MPI_EVENT_INTEGRATED_RAID) {
372 		return (0);
373 	}
374 
375 	raid_event = (EVENT_DATA_RAID *)&msg->Data;
376 
377 	mpt_vol = NULL;
378 	vol_pg = NULL;
379 	if (mpt->raid_volumes != NULL && mpt->ioc_page2 != NULL) {
380 		for (i = 0; i < mpt->ioc_page2->MaxVolumes; i++) {
381 			mpt_vol = &mpt->raid_volumes[i];
382 			vol_pg = mpt_vol->config_page;
383 
384 			if ((mpt_vol->flags & MPT_RVF_ACTIVE) == 0)
385 				continue;
386 
387 			if (vol_pg->VolumeID == raid_event->VolumeID
388 			 && vol_pg->VolumeBus == raid_event->VolumeBus)
389 				break;
390 		}
391 		if (i >= mpt->ioc_page2->MaxVolumes) {
392 			mpt_vol = NULL;
393 			vol_pg = NULL;
394 		}
395 	}
396 
397 	mpt_disk = NULL;
398 	if (raid_event->PhysDiskNum != 0xFF && mpt->raid_disks != NULL) {
399 		mpt_disk = mpt->raid_disks + raid_event->PhysDiskNum;
400 		if ((mpt_disk->flags & MPT_RDF_ACTIVE) == 0) {
401 			mpt_disk = NULL;
402 		}
403 	}
404 
405 	print_event = 1;
406 	switch(raid_event->ReasonCode) {
407 	case MPI_EVENT_RAID_RC_VOLUME_CREATED:
408 	case MPI_EVENT_RAID_RC_VOLUME_DELETED:
409 		break;
410 	case MPI_EVENT_RAID_RC_VOLUME_STATUS_CHANGED:
411 		if (mpt_vol != NULL) {
412 			if ((mpt_vol->flags & MPT_RVF_UP2DATE) != 0) {
413 				mpt_vol->flags &= ~MPT_RVF_UP2DATE;
414 			} else {
415 				/*
416 				 * Coalesce status messages into one
417 				 * per background run of our RAID thread.
418 				 * This removes "spurious" status messages
419 				 * from our output.
420 				 */
421 				print_event = 0;
422 			}
423 		}
424 		break;
425 	case MPI_EVENT_RAID_RC_VOLUME_SETTINGS_CHANGED:
426 	case MPI_EVENT_RAID_RC_VOLUME_PHYSDISK_CHANGED:
427 		mpt->raid_rescan++;
428 		if (mpt_vol != NULL) {
429 			mpt_vol->flags &= ~(MPT_RVF_UP2DATE|MPT_RVF_ANNOUNCED);
430 		}
431 		break;
432 	case MPI_EVENT_RAID_RC_PHYSDISK_CREATED:
433 	case MPI_EVENT_RAID_RC_PHYSDISK_DELETED:
434 		mpt->raid_rescan++;
435 		break;
436 	case MPI_EVENT_RAID_RC_PHYSDISK_SETTINGS_CHANGED:
437 	case MPI_EVENT_RAID_RC_PHYSDISK_STATUS_CHANGED:
438 		mpt->raid_rescan++;
439 		if (mpt_disk != NULL) {
440 			mpt_disk->flags &= ~MPT_RDF_UP2DATE;
441 		}
442 		break;
443 	case MPI_EVENT_RAID_RC_DOMAIN_VAL_NEEDED:
444 		mpt->raid_rescan++;
445 		break;
446 	case MPI_EVENT_RAID_RC_SMART_DATA:
447 	case MPI_EVENT_RAID_RC_REPLACE_ACTION_STARTED:
448 		break;
449 	}
450 
451 	if (print_event) {
452 		if (mpt_disk != NULL) {
453 			mpt_disk_prt(mpt, mpt_disk, NULL);
454 		} else if (mpt_vol != NULL) {
455 			mpt_vol_prt(mpt, mpt_vol, NULL);
456 		} else {
457 			mpt_prt(mpt, "Volume(%d:%d", raid_event->VolumeBus,
458 				raid_event->VolumeID);
459 
460 			if (raid_event->PhysDiskNum != 0xFF)
461 				mpt_prtc(mpt, ":%d): ",
462 					 raid_event->PhysDiskNum);
463 			else
464 				mpt_prtc(mpt, "): ");
465 		}
466 
467 		if (raid_event->ReasonCode >= NUM_ELEMENTS(raid_event_txt))
468 			mpt_prtc(mpt, "Unhandled RaidEvent %#x\n",
469 				 raid_event->ReasonCode);
470 		else
471 			mpt_prtc(mpt, "%s\n",
472 				 raid_event_txt[raid_event->ReasonCode]);
473 	}
474 
475 	if (raid_event->ReasonCode == MPI_EVENT_RAID_RC_SMART_DATA) {
476 		/* XXX Use CAM's print sense for this... */
477 		if (mpt_disk != NULL)
478 			mpt_disk_prt(mpt, mpt_disk, NULL);
479 		else
480 			mpt_prt(mpt, "Volume(%d:%d:%d: ",
481 			    raid_event->VolumeBus, raid_event->VolumeID,
482 			    raid_event->PhysDiskNum);
483 		mpt_prtc(mpt, "ASC 0x%x, ASCQ 0x%x)\n",
484 			 raid_event->ASC, raid_event->ASCQ);
485 	}
486 
487 	mpt_raid_wakeup(mpt);
488 	return (1);
489 }
490 
491 static void
492 mpt_raid_shutdown(struct mpt_softc *mpt)
493 {
494 	struct mpt_raid_volume *mpt_vol;
495 
496 	if (mpt->raid_mwce_setting != MPT_RAID_MWCE_REBUILD_ONLY) {
497 		return;
498 	}
499 
500 	mpt->raid_mwce_setting = MPT_RAID_MWCE_OFF;
501 	RAID_VOL_FOREACH(mpt, mpt_vol) {
502 		mpt_verify_mwce(mpt, mpt_vol);
503 	}
504 }
505 
506 static int
507 mpt_raid_reply_handler(struct mpt_softc *mpt, request_t *req,
508     uint32_t reply_desc, MSG_DEFAULT_REPLY *reply_frame)
509 {
510 	int free_req;
511 
512 	if (req == NULL)
513 		return (TRUE);
514 
515 	free_req = TRUE;
516 	if (reply_frame != NULL)
517 		free_req = mpt_raid_reply_frame_handler(mpt, req, reply_frame);
518 #ifdef NOTYET
519 	else if (req->ccb != NULL) {
520 		/* Complete Quiesce CCB with error... */
521 	}
522 #endif
523 
524 	req->state &= ~REQ_STATE_QUEUED;
525 	req->state |= REQ_STATE_DONE;
526 	TAILQ_REMOVE(&mpt->request_pending_list, req, links);
527 
528 	if ((req->state & REQ_STATE_NEED_WAKEUP) != 0) {
529 		wakeup(req);
530 	} else if (free_req) {
531 		mpt_free_request(mpt, req);
532 	}
533 
534 	return (TRUE);
535 }
536 
537 /*
538  * Parse additional completion information in the reply
539  * frame for RAID I/O requests.
540  */
541 static int
542 mpt_raid_reply_frame_handler(struct mpt_softc *mpt, request_t *req,
543     MSG_DEFAULT_REPLY *reply_frame)
544 {
545 	MSG_RAID_ACTION_REPLY *reply;
546 	struct mpt_raid_action_result *action_result;
547 	MSG_RAID_ACTION_REQUEST *rap;
548 
549 	reply = (MSG_RAID_ACTION_REPLY *)reply_frame;
550 	req->IOCStatus = le16toh(reply->IOCStatus);
551 	rap = (MSG_RAID_ACTION_REQUEST *)req->req_vbuf;
552 
553 	switch (rap->Action) {
554 	case MPI_RAID_ACTION_QUIESCE_PHYS_IO:
555 		mpt_prt(mpt, "QUIESCE PHYSIO DONE\n");
556 		break;
557 	case MPI_RAID_ACTION_ENABLE_PHYS_IO:
558 		mpt_prt(mpt, "ENABLY PHYSIO DONE\n");
559 		break;
560 	default:
561 		break;
562 	}
563 	action_result = REQ_TO_RAID_ACTION_RESULT(req);
564 	memcpy(&action_result->action_data, &reply->ActionData,
565 	    sizeof(action_result->action_data));
566 	action_result->action_status = le16toh(reply->ActionStatus);
567 	return (TRUE);
568 }
569 
570 /*
571  * Utiltity routine to perform a RAID action command;
572  */
573 int
574 mpt_issue_raid_req(struct mpt_softc *mpt, struct mpt_raid_volume *vol,
575 		   struct mpt_raid_disk *disk, request_t *req, u_int Action,
576 		   uint32_t ActionDataWord, bus_addr_t addr, bus_size_t len,
577 		   int write, int wait)
578 {
579 	MSG_RAID_ACTION_REQUEST *rap;
580 	SGE_SIMPLE32 *se;
581 
582 	rap = req->req_vbuf;
583 	memset(rap, 0, sizeof *rap);
584 	rap->Action = Action;
585 	rap->ActionDataWord = htole32(ActionDataWord);
586 	rap->Function = MPI_FUNCTION_RAID_ACTION;
587 	rap->VolumeID = vol->config_page->VolumeID;
588 	rap->VolumeBus = vol->config_page->VolumeBus;
589 	if (disk != 0)
590 		rap->PhysDiskNum = disk->config_page.PhysDiskNum;
591 	else
592 		rap->PhysDiskNum = 0xFF;
593 	se = (SGE_SIMPLE32 *)&rap->ActionDataSGE;
594 	se->Address = htole32(addr);
595 	MPI_pSGE_SET_LENGTH(se, len);
596 	MPI_pSGE_SET_FLAGS(se, (MPI_SGE_FLAGS_SIMPLE_ELEMENT |
597 	    MPI_SGE_FLAGS_LAST_ELEMENT | MPI_SGE_FLAGS_END_OF_BUFFER |
598 	    MPI_SGE_FLAGS_END_OF_LIST |
599 	    write ? MPI_SGE_FLAGS_HOST_TO_IOC : MPI_SGE_FLAGS_IOC_TO_HOST));
600 	se->FlagsLength = htole32(se->FlagsLength);
601 	rap->MsgContext = htole32(req->index | raid_handler_id);
602 
603 	mpt_check_doorbell(mpt);
604 	mpt_send_cmd(mpt, req);
605 
606 	if (wait) {
607 		return (mpt_wait_req(mpt, req, REQ_STATE_DONE, REQ_STATE_DONE,
608 				     /*sleep_ok*/FALSE, /*time_ms*/2000));
609 	} else {
610 		return (0);
611 	}
612 }
613 
614 /*************************** RAID Status Monitoring ***************************/
615 static int
616 mpt_spawn_raid_thread(struct mpt_softc *mpt)
617 {
618 	int error;
619 
620 	/*
621 	 * Freeze out any CAM transactions until our thread
622 	 * is able to run at least once.  We need to update
623 	 * our RAID pages before acception I/O or we may
624 	 * reject I/O to an ID we later determine is for a
625 	 * hidden physdisk.
626 	 */
627 	MPT_LOCK(mpt);
628 	xpt_freeze_simq(mpt->phydisk_sim, 1);
629 	MPT_UNLOCK(mpt);
630 	error = mpt_kthread_create(mpt_raid_thread, mpt,
631 	    &mpt->raid_thread, /*flags*/0, /*altstack*/0,
632 	    "mpt_raid%d", mpt->unit);
633 	if (error != 0) {
634 		MPT_LOCK(mpt);
635 		xpt_release_simq(mpt->phydisk_sim, /*run_queue*/FALSE);
636 		MPT_UNLOCK(mpt);
637 	}
638 	return (error);
639 }
640 
641 static void
642 mpt_terminate_raid_thread(struct mpt_softc *mpt)
643 {
644 
645 	if (mpt->raid_thread == NULL) {
646 		return;
647 	}
648 	mpt->shutdwn_raid = 1;
649 	wakeup(mpt->raid_volumes);
650 	/*
651 	 * Sleep on a slightly different location
652 	 * for this interlock just for added safety.
653 	 */
654 	mpt_sleep(mpt, &mpt->raid_thread, PUSER, "thtrm", 0);
655 }
656 
657 static void
658 mpt_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb)
659 {
660 
661 	xpt_free_path(ccb->ccb_h.path);
662 	xpt_free_ccb(ccb);
663 }
664 
665 static void
666 mpt_raid_thread(void *arg)
667 {
668 	struct mpt_softc *mpt;
669 	int firstrun;
670 
671 	mpt = (struct mpt_softc *)arg;
672 	firstrun = 1;
673 
674 	get_mplock();
675 	MPT_LOCK(mpt);
676 
677 	while (mpt->shutdwn_raid == 0) {
678 
679 		if (mpt->raid_wakeup == 0) {
680 			mpt_sleep(mpt, &mpt->raid_volumes, PUSER, "idle", 0);
681 			continue;
682 		}
683 
684 		mpt->raid_wakeup = 0;
685 
686 		if (mpt_refresh_raid_data(mpt)) {
687 			mpt_schedule_raid_refresh(mpt);	/* XX NOT QUITE RIGHT */
688 			continue;
689 		}
690 
691 		/*
692 		 * Now that we have our first snapshot of RAID data,
693 		 * allow CAM to access our physical disk bus.
694 		 */
695 		if (firstrun) {
696 			firstrun = 0;
697 			MPTLOCK_2_CAMLOCK(mpt);
698 			xpt_release_simq(mpt->phydisk_sim, TRUE);
699 			CAMLOCK_2_MPTLOCK(mpt);
700 		}
701 
702 		if (mpt->raid_rescan != 0) {
703 			union ccb *ccb;
704 			struct cam_path *path;
705 			int error;
706 
707 			mpt->raid_rescan = 0;
708 			MPT_UNLOCK(mpt);
709 
710 			ccb = xpt_alloc_ccb();
711 
712 			MPT_LOCK(mpt);
713 			error = xpt_create_path(&path, xpt_periph,
714 			    cam_sim_path(mpt->phydisk_sim),
715 			    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
716 			if (error != CAM_REQ_CMP) {
717 				xpt_free_ccb(ccb);
718 				mpt_prt(mpt, "Unable to rescan RAID Bus!\n");
719 			} else {
720 				xpt_setup_ccb(&ccb->ccb_h, path, 5);
721 				ccb->ccb_h.func_code = XPT_SCAN_BUS;
722 				ccb->ccb_h.cbfcnp = mpt_cam_rescan_callback;
723 				ccb->crcn.flags = CAM_FLAG_NONE;
724 				MPTLOCK_2_CAMLOCK(mpt);
725 				xpt_action(ccb);
726 				CAMLOCK_2_MPTLOCK(mpt);
727 			}
728 		}
729 	}
730 	mpt->raid_thread = NULL;
731 	wakeup(&mpt->raid_thread);
732 	MPT_UNLOCK(mpt);
733 	rel_mplock();
734 }
735 
736 #if 0
737 static void
738 mpt_raid_quiesce_timeout(void *arg)
739 {
740 	/* Complete the CCB with error */
741 	/* COWWWW */
742 }
743 
744 static timeout_t mpt_raid_quiesce_timeout;
745 cam_status
746 mpt_raid_quiesce_disk(struct mpt_softc *mpt, struct mpt_raid_disk *mpt_disk,
747 		      request_t *req)
748 {
749 	union ccb *ccb;
750 
751 	ccb = req->ccb;
752 	if ((mpt_disk->flags & MPT_RDF_QUIESCED) != 0)
753 		return (CAM_REQ_CMP);
754 
755 	if ((mpt_disk->flags & MPT_RDF_QUIESCING) == 0) {
756 		int rv;
757 
758 		mpt_disk->flags |= MPT_RDF_QUIESCING;
759 		xpt_freeze_devq(ccb->ccb_h.path, 1);
760 
761 		rv = mpt_issue_raid_req(mpt, mpt_disk->volume, mpt_disk, req,
762 					MPI_RAID_ACTION_QUIESCE_PHYS_IO,
763 					/*ActionData*/0, /*addr*/0,
764 					/*len*/0, /*write*/FALSE,
765 					/*wait*/FALSE);
766 		if (rv != 0)
767 			return (CAM_REQ_CMP_ERR);
768 
769 		mpt_req_timeout(req, mpt_raid_quiesce_timeout, ccb, 5 * hz);
770 #if 0
771 		if (rv == ETIMEDOUT) {
772 			mpt_disk_prt(mpt, mpt_disk, "mpt_raid_quiesce_disk: "
773 				     "Quiece Timed-out\n");
774 			xpt_release_devq(ccb->ccb_h.path, 1, /*run*/0);
775 			return (CAM_REQ_CMP_ERR);
776 		}
777 
778 		ar = REQ_TO_RAID_ACTION_RESULT(req);
779 		if (rv != 0
780 		 || REQ_IOCSTATUS(req) != MPI_IOCSTATUS_SUCCESS
781 		 || (ar->action_status != MPI_RAID_ACTION_ASTATUS_SUCCESS)) {
782 			mpt_disk_prt(mpt, mpt_disk, "Quiece Failed"
783 				    "%d:%x:%x\n", rv, req->IOCStatus,
784 				    ar->action_status);
785 			xpt_release_devq(ccb->ccb_h.path, 1, /*run*/0);
786 			return (CAM_REQ_CMP_ERR);
787 		}
788 #endif
789 		return (CAM_REQ_INPROG);
790 	}
791 	return (CAM_REQUEUE_REQ);
792 }
793 #endif
794 
795 /* XXX Ignores that there may be multiple busses/IOCs involved. */
796 cam_status
797 mpt_map_physdisk(struct mpt_softc *mpt, union ccb *ccb, u_int *tgt)
798 {
799 	struct mpt_raid_disk *mpt_disk;
800 
801 	mpt_disk = mpt->raid_disks + ccb->ccb_h.target_id;
802 	if (ccb->ccb_h.target_id < mpt->raid_max_disks
803 	 && (mpt_disk->flags & MPT_RDF_ACTIVE) != 0) {
804 		*tgt = mpt_disk->config_page.PhysDiskID;
805 		return (0);
806 	}
807 	mpt_lprt(mpt, MPT_PRT_DEBUG1, "mpt_map_physdisk(%d) - Not Active\n",
808 		 ccb->ccb_h.target_id);
809 	return (-1);
810 }
811 
812 /* XXX Ignores that there may be multiple busses/IOCs involved. */
813 int
814 mpt_is_raid_volume(struct mpt_softc *mpt, int tgt)
815 {
816 	CONFIG_PAGE_IOC_2_RAID_VOL *ioc_vol;
817 	CONFIG_PAGE_IOC_2_RAID_VOL *ioc_last_vol;
818 
819 	if (mpt->ioc_page2 == NULL || mpt->ioc_page2->MaxPhysDisks == 0) {
820 		return (0);
821 	}
822 	ioc_vol = mpt->ioc_page2->RaidVolume;
823 	ioc_last_vol = ioc_vol + mpt->ioc_page2->NumActiveVolumes;
824 	for (;ioc_vol != ioc_last_vol; ioc_vol++) {
825 		if (ioc_vol->VolumeID == tgt) {
826 			return (1);
827 		}
828 	}
829 	return (0);
830 }
831 
832 #if 0
833 static void
834 mpt_enable_vol(struct mpt_softc *mpt, struct mpt_raid_volume *mpt_vol,
835 	       int enable)
836 {
837 	request_t *req;
838 	struct mpt_raid_action_result *ar;
839 	CONFIG_PAGE_RAID_VOL_0 *vol_pg;
840 	int enabled;
841 	int rv;
842 
843 	vol_pg = mpt_vol->config_page;
844 	enabled = vol_pg->VolumeStatus.Flags & MPI_RAIDVOL0_STATUS_FLAG_ENABLED;
845 
846 	/*
847 	 * If the setting matches the configuration,
848 	 * there is nothing to do.
849 	 */
850 	if ((enabled && enable)
851 	 || (!enabled && !enable))
852 		return;
853 
854 	req = mpt_get_request(mpt, /*sleep_ok*/TRUE);
855 	if (req == NULL) {
856 		mpt_vol_prt(mpt, mpt_vol,
857 			    "mpt_enable_vol: Get request failed!\n");
858 		return;
859 	}
860 
861 	rv = mpt_issue_raid_req(mpt, mpt_vol, /*disk*/NULL, req,
862 				enable ? MPI_RAID_ACTION_ENABLE_VOLUME
863 				       : MPI_RAID_ACTION_DISABLE_VOLUME,
864 				/*data*/0, /*addr*/0, /*len*/0,
865 				/*write*/FALSE, /*wait*/TRUE);
866 	if (rv == ETIMEDOUT) {
867 		mpt_vol_prt(mpt, mpt_vol, "mpt_enable_vol: "
868 			    "%s Volume Timed-out\n",
869 			    enable ? "Enable" : "Disable");
870 		return;
871 	}
872 	ar = REQ_TO_RAID_ACTION_RESULT(req);
873 	if (rv != 0
874 	 || REQ_IOCSTATUS(req) != MPI_IOCSTATUS_SUCCESS
875 	 || (ar->action_status != MPI_RAID_ACTION_ASTATUS_SUCCESS)) {
876 		mpt_vol_prt(mpt, mpt_vol, "%s Volume Failed: %d:%x:%x\n",
877 			    enable ? "Enable" : "Disable",
878 			    rv, req->IOCStatus, ar->action_status);
879 	}
880 
881 	mpt_free_request(mpt, req);
882 }
883 #endif
884 
885 static void
886 mpt_verify_mwce(struct mpt_softc *mpt, struct mpt_raid_volume *mpt_vol)
887 {
888 	request_t *req;
889 	struct mpt_raid_action_result *ar;
890 	CONFIG_PAGE_RAID_VOL_0 *vol_pg;
891 	uint32_t data;
892 	int rv;
893 	int resyncing;
894 	int mwce;
895 
896 	vol_pg = mpt_vol->config_page;
897 	resyncing = vol_pg->VolumeStatus.Flags
898 		  & MPI_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS;
899 	mwce = vol_pg->VolumeSettings.Settings
900 	     & MPI_RAIDVOL0_SETTING_WRITE_CACHING_ENABLE;
901 
902 	/*
903 	 * If the setting matches the configuration,
904 	 * there is nothing to do.
905 	 */
906 	switch (mpt->raid_mwce_setting) {
907 	case MPT_RAID_MWCE_REBUILD_ONLY:
908 		if ((resyncing && mwce) || (!resyncing && !mwce)) {
909 			return;
910 		}
911 		mpt_vol->flags ^= MPT_RVF_WCE_CHANGED;
912 		if ((mpt_vol->flags & MPT_RVF_WCE_CHANGED) == 0) {
913 			/*
914 			 * Wait one more status update to see if
915 			 * resyncing gets enabled.  It gets disabled
916 			 * temporarilly when WCE is changed.
917 			 */
918 			return;
919 		}
920 		break;
921 	case MPT_RAID_MWCE_ON:
922 		if (mwce)
923 			return;
924 		break;
925 	case MPT_RAID_MWCE_OFF:
926 		if (!mwce)
927 			return;
928 		break;
929 	case MPT_RAID_MWCE_NC:
930 		return;
931 	}
932 
933 	req = mpt_get_request(mpt, /*sleep_ok*/TRUE);
934 	if (req == NULL) {
935 		mpt_vol_prt(mpt, mpt_vol,
936 			    "mpt_verify_mwce: Get request failed!\n");
937 		return;
938 	}
939 
940 	vol_pg->VolumeSettings.Settings ^=
941 	    MPI_RAIDVOL0_SETTING_WRITE_CACHING_ENABLE;
942 	memcpy(&data, &vol_pg->VolumeSettings, sizeof(data));
943 	vol_pg->VolumeSettings.Settings ^=
944 	    MPI_RAIDVOL0_SETTING_WRITE_CACHING_ENABLE;
945 	rv = mpt_issue_raid_req(mpt, mpt_vol, /*disk*/NULL, req,
946 				MPI_RAID_ACTION_CHANGE_VOLUME_SETTINGS,
947 				data, /*addr*/0, /*len*/0,
948 				/*write*/FALSE, /*wait*/TRUE);
949 	if (rv == ETIMEDOUT) {
950 		mpt_vol_prt(mpt, mpt_vol, "mpt_verify_mwce: "
951 			    "Write Cache Enable Timed-out\n");
952 		return;
953 	}
954 	ar = REQ_TO_RAID_ACTION_RESULT(req);
955 	if (rv != 0
956 	 || REQ_IOCSTATUS(req) != MPI_IOCSTATUS_SUCCESS
957 	 || (ar->action_status != MPI_RAID_ACTION_ASTATUS_SUCCESS)) {
958 		mpt_vol_prt(mpt, mpt_vol, "Write Cache Enable Failed: "
959 			    "%d:%x:%x\n", rv, req->IOCStatus,
960 			    ar->action_status);
961 	} else {
962 		vol_pg->VolumeSettings.Settings ^=
963 		    MPI_RAIDVOL0_SETTING_WRITE_CACHING_ENABLE;
964 	}
965 	mpt_free_request(mpt, req);
966 }
967 
968 static void
969 mpt_verify_resync_rate(struct mpt_softc *mpt, struct mpt_raid_volume *mpt_vol)
970 {
971 	request_t *req;
972 	struct mpt_raid_action_result *ar;
973 	CONFIG_PAGE_RAID_VOL_0	*vol_pg;
974 	u_int prio;
975 	int rv;
976 
977 	vol_pg = mpt_vol->config_page;
978 
979 	if (mpt->raid_resync_rate == MPT_RAID_RESYNC_RATE_NC)
980 		return;
981 
982 	/*
983 	 * If the current RAID resync rate does not
984 	 * match our configured rate, update it.
985 	 */
986 	prio = vol_pg->VolumeSettings.Settings
987 	     & MPI_RAIDVOL0_SETTING_PRIORITY_RESYNC;
988 	if (vol_pg->ResyncRate != 0
989 	 && vol_pg->ResyncRate != mpt->raid_resync_rate) {
990 
991 		req = mpt_get_request(mpt, /*sleep_ok*/TRUE);
992 		if (req == NULL) {
993 			mpt_vol_prt(mpt, mpt_vol, "mpt_verify_resync_rate: "
994 				    "Get request failed!\n");
995 			return;
996 		}
997 
998 		rv = mpt_issue_raid_req(mpt, mpt_vol, /*disk*/NULL, req,
999 					MPI_RAID_ACTION_SET_RESYNC_RATE,
1000 					mpt->raid_resync_rate, /*addr*/0,
1001 					/*len*/0, /*write*/FALSE, /*wait*/TRUE);
1002 		if (rv == ETIMEDOUT) {
1003 			mpt_vol_prt(mpt, mpt_vol, "mpt_refresh_raid_data: "
1004 				    "Resync Rate Setting Timed-out\n");
1005 			return;
1006 		}
1007 
1008 		ar = REQ_TO_RAID_ACTION_RESULT(req);
1009 		if (rv != 0
1010 		 || REQ_IOCSTATUS(req) != MPI_IOCSTATUS_SUCCESS
1011 		 || (ar->action_status != MPI_RAID_ACTION_ASTATUS_SUCCESS)) {
1012 			mpt_vol_prt(mpt, mpt_vol, "Resync Rate Setting Failed: "
1013 				    "%d:%x:%x\n", rv, req->IOCStatus,
1014 				    ar->action_status);
1015 		} else
1016 			vol_pg->ResyncRate = mpt->raid_resync_rate;
1017 		mpt_free_request(mpt, req);
1018 	} else if ((prio && mpt->raid_resync_rate < 128)
1019 		|| (!prio && mpt->raid_resync_rate >= 128)) {
1020 		uint32_t data;
1021 
1022 		req = mpt_get_request(mpt, /*sleep_ok*/TRUE);
1023 		if (req == NULL) {
1024 			mpt_vol_prt(mpt, mpt_vol, "mpt_verify_resync_rate: "
1025 				    "Get request failed!\n");
1026 			return;
1027 		}
1028 
1029 		vol_pg->VolumeSettings.Settings ^=
1030 		    MPI_RAIDVOL0_SETTING_PRIORITY_RESYNC;
1031 		memcpy(&data, &vol_pg->VolumeSettings, sizeof(data));
1032 		vol_pg->VolumeSettings.Settings ^=
1033 		    MPI_RAIDVOL0_SETTING_PRIORITY_RESYNC;
1034 		rv = mpt_issue_raid_req(mpt, mpt_vol, /*disk*/NULL, req,
1035 					MPI_RAID_ACTION_CHANGE_VOLUME_SETTINGS,
1036 					data, /*addr*/0, /*len*/0,
1037 					/*write*/FALSE, /*wait*/TRUE);
1038 		if (rv == ETIMEDOUT) {
1039 			mpt_vol_prt(mpt, mpt_vol, "mpt_refresh_raid_data: "
1040 				    "Resync Rate Setting Timed-out\n");
1041 			return;
1042 		}
1043 		ar = REQ_TO_RAID_ACTION_RESULT(req);
1044 		if (rv != 0
1045 		 || REQ_IOCSTATUS(req) != MPI_IOCSTATUS_SUCCESS
1046 		 || (ar->action_status != MPI_RAID_ACTION_ASTATUS_SUCCESS)) {
1047 			mpt_vol_prt(mpt, mpt_vol, "Resync Rate Setting Failed: "
1048 				    "%d:%x:%x\n", rv, req->IOCStatus,
1049 				    ar->action_status);
1050 		} else {
1051 			vol_pg->VolumeSettings.Settings ^=
1052 			    MPI_RAIDVOL0_SETTING_PRIORITY_RESYNC;
1053 		}
1054 
1055 		mpt_free_request(mpt, req);
1056 	}
1057 }
1058 
1059 static void
1060 mpt_adjust_queue_depth(struct mpt_softc *mpt, struct mpt_raid_volume *mpt_vol,
1061 		       struct cam_path *path)
1062 {
1063 	struct ccb_relsim crs;
1064 
1065 	xpt_setup_ccb(&crs.ccb_h, path, /*priority*/5);
1066 	crs.ccb_h.func_code = XPT_REL_SIMQ;
1067 	crs.release_flags = RELSIM_ADJUST_OPENINGS;
1068 	crs.openings = mpt->raid_queue_depth;
1069 	xpt_action((union ccb *)&crs);
1070 	if (crs.ccb_h.status != CAM_REQ_CMP)
1071 		mpt_vol_prt(mpt, mpt_vol, "mpt_adjust_queue_depth failed "
1072 			    "with CAM status %#x\n", crs.ccb_h.status);
1073 }
1074 
1075 static void
1076 mpt_announce_vol(struct mpt_softc *mpt, struct mpt_raid_volume *mpt_vol)
1077 {
1078 	CONFIG_PAGE_RAID_VOL_0 *vol_pg;
1079 	u_int i;
1080 
1081 	vol_pg = mpt_vol->config_page;
1082 	mpt_vol_prt(mpt, mpt_vol, "Settings (");
1083 	for (i = 1; i <= 0x8000; i <<= 1) {
1084 		switch (vol_pg->VolumeSettings.Settings & i) {
1085 		case MPI_RAIDVOL0_SETTING_WRITE_CACHING_ENABLE:
1086 			mpt_prtc(mpt, " Member-WCE");
1087 			break;
1088 		case MPI_RAIDVOL0_SETTING_OFFLINE_ON_SMART:
1089 			mpt_prtc(mpt, " Offline-On-SMART-Err");
1090 			break;
1091 		case MPI_RAIDVOL0_SETTING_AUTO_CONFIGURE:
1092 			mpt_prtc(mpt, " Hot-Plug-Spares");
1093 			break;
1094 		case MPI_RAIDVOL0_SETTING_PRIORITY_RESYNC:
1095 			mpt_prtc(mpt, " High-Priority-ReSync");
1096 			break;
1097 		default:
1098 			break;
1099 		}
1100 	}
1101 	mpt_prtc(mpt, " )\n");
1102 	if (vol_pg->VolumeSettings.HotSparePool != 0) {
1103 		mpt_vol_prt(mpt, mpt_vol, "Using Spare Pool%s",
1104 			    powerof2(vol_pg->VolumeSettings.HotSparePool)
1105 			  ? ":" : "s:");
1106 		for (i = 0; i < 8; i++) {
1107 			u_int mask;
1108 
1109 			mask = 0x1 << i;
1110 			if ((vol_pg->VolumeSettings.HotSparePool & mask) == 0)
1111 				continue;
1112 			mpt_prtc(mpt, " %d", i);
1113 		}
1114 		mpt_prtc(mpt, "\n");
1115 	}
1116 	mpt_vol_prt(mpt, mpt_vol, "%d Members:\n", vol_pg->NumPhysDisks);
1117 	for (i = 0; i < vol_pg->NumPhysDisks; i++){
1118 		struct mpt_raid_disk *mpt_disk;
1119 		CONFIG_PAGE_RAID_PHYS_DISK_0 *disk_pg;
1120 		int pt_bus = cam_sim_bus(mpt->phydisk_sim);
1121 		U8 f, s;
1122 
1123 		mpt_disk = mpt->raid_disks + vol_pg->PhysDisk[i].PhysDiskNum;
1124 		disk_pg = &mpt_disk->config_page;
1125 		mpt_prtc(mpt, "      ");
1126 		mpt_prtc(mpt, "(%s:%d:%d:0): ", device_get_nameunit(mpt->dev),
1127 			 pt_bus, disk_pg->PhysDiskID);
1128 		if (vol_pg->VolumeType == MPI_RAID_VOL_TYPE_IM) {
1129 			mpt_prtc(mpt, "%s", mpt_disk->member_number == 0?
1130 			    "Primary" : "Secondary");
1131 		} else {
1132 			mpt_prtc(mpt, "Stripe Position %d",
1133 				 mpt_disk->member_number);
1134 		}
1135 		f = disk_pg->PhysDiskStatus.Flags;
1136 		s = disk_pg->PhysDiskStatus.State;
1137 		if (f & MPI_PHYSDISK0_STATUS_FLAG_OUT_OF_SYNC) {
1138 			mpt_prtc(mpt, " Out of Sync");
1139 		}
1140 		if (f & MPI_PHYSDISK0_STATUS_FLAG_QUIESCED) {
1141 			mpt_prtc(mpt, " Quiesced");
1142 		}
1143 		if (f & MPI_PHYSDISK0_STATUS_FLAG_INACTIVE_VOLUME) {
1144 			mpt_prtc(mpt, " Inactive");
1145 		}
1146 		if (f & MPI_PHYSDISK0_STATUS_FLAG_OPTIMAL_PREVIOUS) {
1147 			mpt_prtc(mpt, " Was Optimal");
1148 		}
1149 		if (f & MPI_PHYSDISK0_STATUS_FLAG_NOT_OPTIMAL_PREVIOUS) {
1150 			mpt_prtc(mpt, " Was Non-Optimal");
1151 		}
1152 		switch (s) {
1153 		case MPI_PHYSDISK0_STATUS_ONLINE:
1154 			mpt_prtc(mpt, " Online");
1155 			break;
1156 		case MPI_PHYSDISK0_STATUS_MISSING:
1157 			mpt_prtc(mpt, " Missing");
1158 			break;
1159 		case MPI_PHYSDISK0_STATUS_NOT_COMPATIBLE:
1160 			mpt_prtc(mpt, " Incompatible");
1161 			break;
1162 		case MPI_PHYSDISK0_STATUS_FAILED:
1163 			mpt_prtc(mpt, " Failed");
1164 			break;
1165 		case MPI_PHYSDISK0_STATUS_INITIALIZING:
1166 			mpt_prtc(mpt, " Initializing");
1167 			break;
1168 		case MPI_PHYSDISK0_STATUS_OFFLINE_REQUESTED:
1169 			mpt_prtc(mpt, " Requested Offline");
1170 			break;
1171 		case MPI_PHYSDISK0_STATUS_FAILED_REQUESTED:
1172 			mpt_prtc(mpt, " Requested Failed");
1173 			break;
1174 		case MPI_PHYSDISK0_STATUS_OTHER_OFFLINE:
1175 		default:
1176 			mpt_prtc(mpt, " Offline Other (%x)", s);
1177 			break;
1178 		}
1179 		mpt_prtc(mpt, "\n");
1180 	}
1181 }
1182 
1183 static void
1184 mpt_announce_disk(struct mpt_softc *mpt, struct mpt_raid_disk *mpt_disk)
1185 {
1186 	CONFIG_PAGE_RAID_PHYS_DISK_0 *disk_pg;
1187 	int rd_bus = cam_sim_bus(mpt->sim);
1188 	int pt_bus = cam_sim_bus(mpt->phydisk_sim);
1189 	u_int i;
1190 
1191 	disk_pg = &mpt_disk->config_page;
1192 	mpt_disk_prt(mpt, mpt_disk,
1193 		     "Physical (%s:%d:%d:0), Pass-thru (%s:%d:%jd:0)\n",
1194 		     device_get_nameunit(mpt->dev), rd_bus,
1195 		     disk_pg->PhysDiskID, device_get_nameunit(mpt->dev),
1196 		     pt_bus, (intmax_t)(mpt_disk - mpt->raid_disks));
1197 	if (disk_pg->PhysDiskSettings.HotSparePool == 0)
1198 		return;
1199 	mpt_disk_prt(mpt, mpt_disk, "Member of Hot Spare Pool%s",
1200 		     powerof2(disk_pg->PhysDiskSettings.HotSparePool)
1201 		   ? ":" : "s:");
1202 	for (i = 0; i < 8; i++) {
1203 		u_int mask;
1204 
1205 		mask = 0x1 << i;
1206 		if ((disk_pg->PhysDiskSettings.HotSparePool & mask) == 0)
1207 			continue;
1208 		mpt_prtc(mpt, " %d", i);
1209 	}
1210 	mpt_prtc(mpt, "\n");
1211 }
1212 
1213 static void
1214 mpt_refresh_raid_disk(struct mpt_softc *mpt, struct mpt_raid_disk *mpt_disk,
1215 		      IOC_3_PHYS_DISK *ioc_disk)
1216 {
1217 	int rv;
1218 
1219 	rv = mpt_read_cfg_header(mpt, MPI_CONFIG_PAGETYPE_RAID_PHYSDISK,
1220 				 /*PageNumber*/0, ioc_disk->PhysDiskNum,
1221 				 &mpt_disk->config_page.Header,
1222 				 /*sleep_ok*/TRUE, /*timeout_ms*/5000);
1223 	if (rv != 0) {
1224 		mpt_prt(mpt, "mpt_refresh_raid_disk: "
1225 			"Failed to read RAID Disk Hdr(%d)\n",
1226 			ioc_disk->PhysDiskNum);
1227 		return;
1228 	}
1229 	rv = mpt_read_cur_cfg_page(mpt, ioc_disk->PhysDiskNum,
1230 				   &mpt_disk->config_page.Header,
1231 				   sizeof(mpt_disk->config_page),
1232 				   /*sleep_ok*/TRUE, /*timeout_ms*/5000);
1233 	if (rv != 0)
1234 		mpt_prt(mpt, "mpt_refresh_raid_disk: "
1235 			"Failed to read RAID Disk Page(%d)\n",
1236 			ioc_disk->PhysDiskNum);
1237 	mpt2host_config_page_raid_phys_disk_0(&mpt_disk->config_page);
1238 }
1239 
1240 static void
1241 mpt_refresh_raid_vol(struct mpt_softc *mpt, struct mpt_raid_volume *mpt_vol,
1242     CONFIG_PAGE_IOC_2_RAID_VOL *ioc_vol)
1243 {
1244 	CONFIG_PAGE_RAID_VOL_0 *vol_pg;
1245 	struct mpt_raid_action_result *ar;
1246 	request_t *req;
1247 	int rv;
1248 	int i;
1249 
1250 	vol_pg = mpt_vol->config_page;
1251 	mpt_vol->flags &= ~MPT_RVF_UP2DATE;
1252 
1253 	rv = mpt_read_cfg_header(mpt, MPI_CONFIG_PAGETYPE_RAID_VOLUME, 0,
1254 	    ioc_vol->VolumePageNumber, &vol_pg->Header, TRUE, 5000);
1255 	if (rv != 0) {
1256 		mpt_vol_prt(mpt, mpt_vol,
1257 		    "mpt_refresh_raid_vol: Failed to read RAID Vol Hdr(%d)\n",
1258 		    ioc_vol->VolumePageNumber);
1259 		return;
1260 	}
1261 
1262 	rv = mpt_read_cur_cfg_page(mpt, ioc_vol->VolumePageNumber,
1263 	    &vol_pg->Header, mpt->raid_page0_len, TRUE, 5000);
1264 	if (rv != 0) {
1265 		mpt_vol_prt(mpt, mpt_vol,
1266 		    "mpt_refresh_raid_vol: Failed to read RAID Vol Page(%d)\n",
1267 		    ioc_vol->VolumePageNumber);
1268 		return;
1269 	}
1270 	mpt2host_config_page_raid_vol_0(vol_pg);
1271 
1272 	mpt_vol->flags |= MPT_RVF_ACTIVE;
1273 
1274 	/* Update disk entry array data. */
1275 	for (i = 0; i < vol_pg->NumPhysDisks; i++) {
1276 		struct mpt_raid_disk *mpt_disk;
1277 		mpt_disk = mpt->raid_disks + vol_pg->PhysDisk[i].PhysDiskNum;
1278 		mpt_disk->volume = mpt_vol;
1279 		mpt_disk->member_number = vol_pg->PhysDisk[i].PhysDiskMap;
1280 		if (vol_pg->VolumeType == MPI_RAID_VOL_TYPE_IM) {
1281 			mpt_disk->member_number--;
1282 		}
1283 	}
1284 
1285 	if ((vol_pg->VolumeStatus.Flags
1286 	   & MPI_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS) == 0)
1287 		return;
1288 
1289 	req = mpt_get_request(mpt, TRUE);
1290 	if (req == NULL) {
1291 		mpt_vol_prt(mpt, mpt_vol,
1292 		    "mpt_refresh_raid_vol: Get request failed!\n");
1293 		return;
1294 	}
1295 	rv = mpt_issue_raid_req(mpt, mpt_vol, NULL, req,
1296 	    MPI_RAID_ACTION_INDICATOR_STRUCT, 0, 0, 0, FALSE, TRUE);
1297 	if (rv == ETIMEDOUT) {
1298 		mpt_vol_prt(mpt, mpt_vol,
1299 		    "mpt_refresh_raid_vol: Progress Indicator fetch timeout\n");
1300 		mpt_free_request(mpt, req);
1301 		return;
1302 	}
1303 
1304 	ar = REQ_TO_RAID_ACTION_RESULT(req);
1305 	if (rv == 0
1306 	 && ar->action_status == MPI_RAID_ACTION_ASTATUS_SUCCESS
1307 	 && REQ_IOCSTATUS(req) == MPI_IOCSTATUS_SUCCESS) {
1308 		memcpy(&mpt_vol->sync_progress,
1309 		       &ar->action_data.indicator_struct,
1310 		       sizeof(mpt_vol->sync_progress));
1311 		mpt2host_mpi_raid_vol_indicator(&mpt_vol->sync_progress);
1312 	} else {
1313 		mpt_vol_prt(mpt, mpt_vol,
1314 		    "mpt_refresh_raid_vol: Progress indicator fetch failed!\n");
1315 	}
1316 	mpt_free_request(mpt, req);
1317 }
1318 
1319 /*
1320  * Update in-core information about RAID support.  We update any entries
1321  * that didn't previously exists or have been marked as needing to
1322  * be updated by our event handler.  Interesting changes are displayed
1323  * to the console.
1324  */
1325 int
1326 mpt_refresh_raid_data(struct mpt_softc *mpt)
1327 {
1328 	CONFIG_PAGE_IOC_2_RAID_VOL *ioc_vol;
1329 	CONFIG_PAGE_IOC_2_RAID_VOL *ioc_last_vol;
1330 	IOC_3_PHYS_DISK *ioc_disk;
1331 	IOC_3_PHYS_DISK *ioc_last_disk;
1332 	CONFIG_PAGE_RAID_VOL_0	*vol_pg;
1333 	size_t len;
1334 	int rv;
1335 	int i;
1336 	u_int nonopt_volumes;
1337 
1338 	if (mpt->ioc_page2 == NULL || mpt->ioc_page3 == NULL) {
1339 		return (0);
1340 	}
1341 
1342 	/*
1343 	 * Mark all items as unreferenced by the configuration.
1344 	 * This allows us to find, report, and discard stale
1345 	 * entries.
1346 	 */
1347 	for (i = 0; i < mpt->ioc_page2->MaxPhysDisks; i++) {
1348 		mpt->raid_disks[i].flags &= ~MPT_RDF_REFERENCED;
1349 	}
1350 	for (i = 0; i < mpt->ioc_page2->MaxVolumes; i++) {
1351 		mpt->raid_volumes[i].flags &= ~MPT_RVF_REFERENCED;
1352 	}
1353 
1354 	/*
1355 	 * Get Physical Disk information.
1356 	 */
1357 	len = mpt->ioc_page3->Header.PageLength * sizeof(uint32_t);
1358 	rv = mpt_read_cur_cfg_page(mpt, /*PageAddress*/0,
1359 				   &mpt->ioc_page3->Header, len,
1360 				   /*sleep_ok*/TRUE, /*timeout_ms*/5000);
1361 	if (rv) {
1362 		mpt_prt(mpt,
1363 		    "mpt_refresh_raid_data: Failed to read IOC Page 3\n");
1364 		return (-1);
1365 	}
1366 	mpt2host_config_page_ioc3(mpt->ioc_page3);
1367 
1368 	ioc_disk = mpt->ioc_page3->PhysDisk;
1369 	ioc_last_disk = ioc_disk + mpt->ioc_page3->NumPhysDisks;
1370 	for (; ioc_disk != ioc_last_disk; ioc_disk++) {
1371 		struct mpt_raid_disk *mpt_disk;
1372 
1373 		mpt_disk = mpt->raid_disks + ioc_disk->PhysDiskNum;
1374 		mpt_disk->flags |= MPT_RDF_REFERENCED;
1375 		if ((mpt_disk->flags & (MPT_RDF_ACTIVE|MPT_RDF_UP2DATE))
1376 		 != (MPT_RDF_ACTIVE|MPT_RDF_UP2DATE)) {
1377 
1378 			mpt_refresh_raid_disk(mpt, mpt_disk, ioc_disk);
1379 
1380 		}
1381 		mpt_disk->flags |= MPT_RDF_ACTIVE;
1382 		mpt->raid_rescan++;
1383 	}
1384 
1385 	/*
1386 	 * Refresh volume data.
1387 	 */
1388 	len = mpt->ioc_page2->Header.PageLength * sizeof(uint32_t);
1389 	rv = mpt_read_cur_cfg_page(mpt, /*PageAddress*/0,
1390 				   &mpt->ioc_page2->Header, len,
1391 				   /*sleep_ok*/TRUE, /*timeout_ms*/5000);
1392 	if (rv) {
1393 		mpt_prt(mpt, "mpt_refresh_raid_data: "
1394 			"Failed to read IOC Page 2\n");
1395 		return (-1);
1396 	}
1397 	mpt2host_config_page_ioc2(mpt->ioc_page2);
1398 
1399 	ioc_vol = mpt->ioc_page2->RaidVolume;
1400 	ioc_last_vol = ioc_vol + mpt->ioc_page2->NumActiveVolumes;
1401 	for (;ioc_vol != ioc_last_vol; ioc_vol++) {
1402 		struct mpt_raid_volume *mpt_vol;
1403 
1404 		mpt_vol = mpt->raid_volumes + ioc_vol->VolumePageNumber;
1405 		mpt_vol->flags |= MPT_RVF_REFERENCED;
1406 		vol_pg = mpt_vol->config_page;
1407 		if (vol_pg == NULL)
1408 			continue;
1409 		if (((mpt_vol->flags & (MPT_RVF_ACTIVE|MPT_RVF_UP2DATE))
1410 		  != (MPT_RVF_ACTIVE|MPT_RVF_UP2DATE))
1411 		 || (vol_pg->VolumeStatus.Flags
1412 		   & MPI_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS) != 0) {
1413 
1414 			mpt_refresh_raid_vol(mpt, mpt_vol, ioc_vol);
1415 		}
1416 		mpt_vol->flags |= MPT_RVF_ACTIVE;
1417 	}
1418 
1419 	nonopt_volumes = 0;
1420 	for (i = 0; i < mpt->ioc_page2->MaxVolumes; i++) {
1421 		struct mpt_raid_volume *mpt_vol;
1422 		uint64_t total;
1423 		uint64_t left;
1424 		int m;
1425 		u_int prio;
1426 
1427 		mpt_vol = &mpt->raid_volumes[i];
1428 
1429 		if ((mpt_vol->flags & MPT_RVF_ACTIVE) == 0) {
1430 			continue;
1431 		}
1432 
1433 		vol_pg = mpt_vol->config_page;
1434 		if ((mpt_vol->flags & (MPT_RVF_REFERENCED|MPT_RVF_ANNOUNCED))
1435 		 == MPT_RVF_ANNOUNCED) {
1436 			mpt_vol_prt(mpt, mpt_vol, "No longer configured\n");
1437 			mpt_vol->flags = 0;
1438 			continue;
1439 		}
1440 
1441 		if ((mpt_vol->flags & MPT_RVF_ANNOUNCED) == 0) {
1442 			mpt_announce_vol(mpt, mpt_vol);
1443 			mpt_vol->flags |= MPT_RVF_ANNOUNCED;
1444 		}
1445 
1446 		if (vol_pg->VolumeStatus.State !=
1447 		    MPI_RAIDVOL0_STATUS_STATE_OPTIMAL)
1448 			nonopt_volumes++;
1449 
1450 		if ((mpt_vol->flags & MPT_RVF_UP2DATE) != 0)
1451 			continue;
1452 
1453 		mpt_vol->flags |= MPT_RVF_UP2DATE;
1454 		mpt_vol_prt(mpt, mpt_vol, "%s - %s\n",
1455 		    mpt_vol_type(mpt_vol), mpt_vol_state(mpt_vol));
1456 		mpt_verify_mwce(mpt, mpt_vol);
1457 
1458 		if (vol_pg->VolumeStatus.Flags == 0) {
1459 			continue;
1460 		}
1461 
1462 		mpt_vol_prt(mpt, mpt_vol, "Status (");
1463 		for (m = 1; m <= 0x80; m <<= 1) {
1464 			switch (vol_pg->VolumeStatus.Flags & m) {
1465 			case MPI_RAIDVOL0_STATUS_FLAG_ENABLED:
1466 				mpt_prtc(mpt, " Enabled");
1467 				break;
1468 			case MPI_RAIDVOL0_STATUS_FLAG_QUIESCED:
1469 				mpt_prtc(mpt, " Quiesced");
1470 				break;
1471 			case MPI_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS:
1472 				mpt_prtc(mpt, " Re-Syncing");
1473 				break;
1474 			case MPI_RAIDVOL0_STATUS_FLAG_VOLUME_INACTIVE:
1475 				mpt_prtc(mpt, " Inactive");
1476 				break;
1477 			default:
1478 				break;
1479 			}
1480 		}
1481 		mpt_prtc(mpt, " )\n");
1482 
1483 		if ((vol_pg->VolumeStatus.Flags
1484 		   & MPI_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS) == 0)
1485 			continue;
1486 
1487 		mpt_verify_resync_rate(mpt, mpt_vol);
1488 
1489 		left = MPT_U64_2_SCALAR(mpt_vol->sync_progress.BlocksRemaining);
1490 		total = MPT_U64_2_SCALAR(mpt_vol->sync_progress.TotalBlocks);
1491 		if (vol_pg->ResyncRate != 0) {
1492 
1493 			prio = ((u_int)vol_pg->ResyncRate * 100000) / 0xFF;
1494 			mpt_vol_prt(mpt, mpt_vol, "Rate %d.%d%%\n",
1495 			    prio / 1000, prio % 1000);
1496 		} else {
1497 			prio = vol_pg->VolumeSettings.Settings
1498 			     & MPI_RAIDVOL0_SETTING_PRIORITY_RESYNC;
1499 			mpt_vol_prt(mpt, mpt_vol, "%s Priority Re-Sync\n",
1500 			    prio ? "High" : "Low");
1501 		}
1502 		mpt_vol_prt(mpt, mpt_vol, "%ju of %ju "
1503 			    "blocks remaining\n", (uintmax_t)left,
1504 			    (uintmax_t)total);
1505 
1506 		/* Periodically report on sync progress. */
1507 		mpt_schedule_raid_refresh(mpt);
1508 	}
1509 
1510 	for (i = 0; i < mpt->ioc_page2->MaxPhysDisks; i++) {
1511 		struct mpt_raid_disk *mpt_disk;
1512 		CONFIG_PAGE_RAID_PHYS_DISK_0 *disk_pg;
1513 		int m;
1514 
1515 		mpt_disk = &mpt->raid_disks[i];
1516 		disk_pg = &mpt_disk->config_page;
1517 
1518 		if ((mpt_disk->flags & MPT_RDF_ACTIVE) == 0)
1519 			continue;
1520 
1521 		if ((mpt_disk->flags & (MPT_RDF_REFERENCED|MPT_RDF_ANNOUNCED))
1522 		 == MPT_RDF_ANNOUNCED) {
1523 			mpt_disk_prt(mpt, mpt_disk, "No longer configured\n");
1524 			mpt_disk->flags = 0;
1525 			mpt->raid_rescan++;
1526 			continue;
1527 		}
1528 
1529 		if ((mpt_disk->flags & MPT_RDF_ANNOUNCED) == 0) {
1530 
1531 			mpt_announce_disk(mpt, mpt_disk);
1532 			mpt_disk->flags |= MPT_RVF_ANNOUNCED;
1533 		}
1534 
1535 		if ((mpt_disk->flags & MPT_RDF_UP2DATE) != 0)
1536 			continue;
1537 
1538 		mpt_disk->flags |= MPT_RDF_UP2DATE;
1539 		mpt_disk_prt(mpt, mpt_disk, "%s\n", mpt_disk_state(mpt_disk));
1540 		if (disk_pg->PhysDiskStatus.Flags == 0)
1541 			continue;
1542 
1543 		mpt_disk_prt(mpt, mpt_disk, "Status (");
1544 		for (m = 1; m <= 0x80; m <<= 1) {
1545 			switch (disk_pg->PhysDiskStatus.Flags & m) {
1546 			case MPI_PHYSDISK0_STATUS_FLAG_OUT_OF_SYNC:
1547 				mpt_prtc(mpt, " Out-Of-Sync");
1548 				break;
1549 			case MPI_PHYSDISK0_STATUS_FLAG_QUIESCED:
1550 				mpt_prtc(mpt, " Quiesced");
1551 				break;
1552 			default:
1553 				break;
1554 			}
1555 		}
1556 		mpt_prtc(mpt, " )\n");
1557 	}
1558 
1559 	mpt->raid_nonopt_volumes = nonopt_volumes;
1560 	return (0);
1561 }
1562 
1563 static void
1564 mpt_raid_timer(void *arg)
1565 {
1566 	struct mpt_softc *mpt;
1567 
1568 	mpt = (struct mpt_softc *)arg;
1569 	MPT_LOCK(mpt);
1570 	mpt_raid_wakeup(mpt);
1571 	MPT_UNLOCK(mpt);
1572 }
1573 
1574 void
1575 mpt_schedule_raid_refresh(struct mpt_softc *mpt)
1576 {
1577 	callout_reset(&mpt->raid_timer, MPT_RAID_SYNC_REPORT_INTERVAL,
1578 		      mpt_raid_timer, mpt);
1579 }
1580 
1581 void
1582 mpt_raid_free_mem(struct mpt_softc *mpt)
1583 {
1584 
1585 	if (mpt->raid_volumes) {
1586 		struct mpt_raid_volume *mpt_raid;
1587 		int i;
1588 		for (i = 0; i < mpt->raid_max_volumes; i++) {
1589 			mpt_raid = &mpt->raid_volumes[i];
1590 			if (mpt_raid->config_page) {
1591 				kfree(mpt_raid->config_page, M_DEVBUF);
1592 				mpt_raid->config_page = NULL;
1593 			}
1594 		}
1595 		kfree(mpt->raid_volumes, M_DEVBUF);
1596 		mpt->raid_volumes = NULL;
1597 	}
1598 	if (mpt->raid_disks) {
1599 		kfree(mpt->raid_disks, M_DEVBUF);
1600 		mpt->raid_disks = NULL;
1601 	}
1602 	if (mpt->ioc_page2) {
1603 		kfree(mpt->ioc_page2, M_DEVBUF);
1604 		mpt->ioc_page2 = NULL;
1605 	}
1606 	if (mpt->ioc_page3) {
1607 		kfree(mpt->ioc_page3, M_DEVBUF);
1608 		mpt->ioc_page3 = NULL;
1609 	}
1610 	mpt->raid_max_volumes =  0;
1611 	mpt->raid_max_disks =  0;
1612 }
1613 
1614 #if __FreeBSD_version >= 500000
1615 static int
1616 mpt_raid_set_vol_resync_rate(struct mpt_softc *mpt, u_int rate)
1617 {
1618 	struct mpt_raid_volume *mpt_vol;
1619 
1620 	if ((rate > MPT_RAID_RESYNC_RATE_MAX
1621 	  || rate < MPT_RAID_RESYNC_RATE_MIN)
1622 	 && rate != MPT_RAID_RESYNC_RATE_NC)
1623 		return (EINVAL);
1624 
1625 	MPT_LOCK(mpt);
1626 	mpt->raid_resync_rate = rate;
1627 	RAID_VOL_FOREACH(mpt, mpt_vol) {
1628 		if ((mpt_vol->flags & MPT_RVF_ACTIVE) == 0) {
1629 			continue;
1630 		}
1631 		mpt_verify_resync_rate(mpt, mpt_vol);
1632 	}
1633 	MPT_UNLOCK(mpt);
1634 	return (0);
1635 }
1636 
1637 static int
1638 mpt_raid_set_vol_queue_depth(struct mpt_softc *mpt, u_int vol_queue_depth)
1639 {
1640 	struct mpt_raid_volume *mpt_vol;
1641 
1642 	if (vol_queue_depth > 255 || vol_queue_depth < 1)
1643 		return (EINVAL);
1644 
1645 	MPT_LOCK(mpt);
1646 	mpt->raid_queue_depth = vol_queue_depth;
1647 	RAID_VOL_FOREACH(mpt, mpt_vol) {
1648 		struct cam_path *path;
1649 		int error;
1650 
1651 		if ((mpt_vol->flags & MPT_RVF_ACTIVE) == 0)
1652 			continue;
1653 
1654 		mpt->raid_rescan = 0;
1655 
1656 		MPTLOCK_2_CAMLOCK(mpt);
1657 		error = xpt_create_path(&path, xpt_periph,
1658 					cam_sim_path(mpt->sim),
1659 					mpt_vol->config_page->VolumeID,
1660 					/*lun*/0);
1661 		if (error != CAM_REQ_CMP) {
1662 			CAMLOCK_2_MPTLOCK(mpt);
1663 			mpt_vol_prt(mpt, mpt_vol, "Unable to allocate path!\n");
1664 			continue;
1665 		}
1666 		mpt_adjust_queue_depth(mpt, mpt_vol, path);
1667 		xpt_free_path(path);
1668 		CAMLOCK_2_MPTLOCK(mpt);
1669 	}
1670 	MPT_UNLOCK(mpt);
1671 	return (0);
1672 }
1673 
1674 static int
1675 mpt_raid_set_vol_mwce(struct mpt_softc *mpt, mpt_raid_mwce_t mwce)
1676 {
1677 	struct mpt_raid_volume *mpt_vol;
1678 	int force_full_resync;
1679 
1680 	MPT_LOCK(mpt);
1681 	if (mwce == mpt->raid_mwce_setting) {
1682 		MPT_UNLOCK(mpt);
1683 		return (0);
1684 	}
1685 
1686 	/*
1687 	 * Catch MWCE being left on due to a failed shutdown.  Since
1688 	 * sysctls cannot be set by the loader, we treat the first
1689 	 * setting of this varible specially and force a full volume
1690 	 * resync if MWCE is enabled and a resync is in progress.
1691 	 */
1692 	force_full_resync = 0;
1693 	if (mpt->raid_mwce_set == 0
1694 	 && mpt->raid_mwce_setting == MPT_RAID_MWCE_NC
1695 	 && mwce == MPT_RAID_MWCE_REBUILD_ONLY)
1696 		force_full_resync = 1;
1697 
1698 	mpt->raid_mwce_setting = mwce;
1699 	RAID_VOL_FOREACH(mpt, mpt_vol) {
1700 		CONFIG_PAGE_RAID_VOL_0 *vol_pg;
1701 		int resyncing;
1702 		int mwce;
1703 
1704 		if ((mpt_vol->flags & MPT_RVF_ACTIVE) == 0)
1705 			continue;
1706 
1707 		vol_pg = mpt_vol->config_page;
1708 		resyncing = vol_pg->VolumeStatus.Flags
1709 			  & MPI_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS;
1710 		mwce = vol_pg->VolumeSettings.Settings
1711 		     & MPI_RAIDVOL0_SETTING_WRITE_CACHING_ENABLE;
1712 		if (force_full_resync && resyncing && mwce) {
1713 
1714 			/*
1715 			 * XXX disable/enable volume should force a resync,
1716 			 *     but we'll need to queice, drain, and restart
1717 			 *     I/O to do that.
1718 			 */
1719 			mpt_vol_prt(mpt, mpt_vol, "WARNING - Unsafe shutdown "
1720 				    "detected.  Suggest full resync.\n");
1721 		}
1722 		mpt_verify_mwce(mpt, mpt_vol);
1723 	}
1724 	mpt->raid_mwce_set = 1;
1725 	MPT_UNLOCK(mpt);
1726 	return (0);
1727 }
1728 const char *mpt_vol_mwce_strs[] =
1729 {
1730 	"On",
1731 	"Off",
1732 	"On-During-Rebuild",
1733 	"NC"
1734 };
1735 
1736 static int
1737 mpt_raid_sysctl_vol_member_wce(SYSCTL_HANDLER_ARGS)
1738 {
1739 	char inbuf[20];
1740 	struct mpt_softc *mpt;
1741 	const char *str;
1742 	int error;
1743 	u_int size;
1744 	u_int i;
1745 
1746 	GIANT_REQUIRED;
1747 
1748 	mpt = (struct mpt_softc *)arg1;
1749 	str = mpt_vol_mwce_strs[mpt->raid_mwce_setting];
1750 	error = SYSCTL_OUT(req, str, strlen(str) + 1);
1751 	if (error || !req->newptr) {
1752 		return (error);
1753 	}
1754 
1755 	size = req->newlen - req->newidx;
1756 	if (size >= sizeof(inbuf)) {
1757 		return (EINVAL);
1758 	}
1759 
1760 	error = SYSCTL_IN(req, inbuf, size);
1761 	if (error) {
1762 		return (error);
1763 	}
1764 	inbuf[size] = '\0';
1765 	for (i = 0; i < NUM_ELEMENTS(mpt_vol_mwce_strs); i++) {
1766 		if (strcmp(mpt_vol_mwce_strs[i], inbuf) == 0) {
1767 			return (mpt_raid_set_vol_mwce(mpt, i));
1768 		}
1769 	}
1770 	return (EINVAL);
1771 }
1772 
1773 static int
1774 mpt_raid_sysctl_vol_resync_rate(SYSCTL_HANDLER_ARGS)
1775 {
1776 	struct mpt_softc *mpt;
1777 	u_int raid_resync_rate;
1778 	int error;
1779 
1780 	GIANT_REQUIRED;
1781 
1782 	mpt = (struct mpt_softc *)arg1;
1783 	raid_resync_rate = mpt->raid_resync_rate;
1784 
1785 	error = sysctl_handle_int(oidp, &raid_resync_rate, 0, req);
1786 	if (error || !req->newptr) {
1787 		return error;
1788 	}
1789 
1790 	return (mpt_raid_set_vol_resync_rate(mpt, raid_resync_rate));
1791 }
1792 
1793 static int
1794 mpt_raid_sysctl_vol_queue_depth(SYSCTL_HANDLER_ARGS)
1795 {
1796 	struct mpt_softc *mpt;
1797 	u_int raid_queue_depth;
1798 	int error;
1799 
1800 	GIANT_REQUIRED;
1801 
1802 	mpt = (struct mpt_softc *)arg1;
1803 	raid_queue_depth = mpt->raid_queue_depth;
1804 
1805 	error = sysctl_handle_int(oidp, &raid_queue_depth, 0, req);
1806 	if (error || !req->newptr) {
1807 		return error;
1808 	}
1809 
1810 	return (mpt_raid_set_vol_queue_depth(mpt, raid_queue_depth));
1811 }
1812 
1813 static void
1814 mpt_raid_sysctl_attach(struct mpt_softc *mpt)
1815 {
1816 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(mpt->dev);
1817 	struct sysctl_oid *tree = device_get_sysctl_tree(mpt->dev);
1818 
1819 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
1820 			"vol_member_wce", CTLTYPE_STRING | CTLFLAG_RW, mpt, 0,
1821 			mpt_raid_sysctl_vol_member_wce, "A",
1822 			"volume member WCE(On,Off,On-During-Rebuild,NC)");
1823 
1824 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
1825 			"vol_queue_depth", CTLTYPE_INT | CTLFLAG_RW, mpt, 0,
1826 			mpt_raid_sysctl_vol_queue_depth, "I",
1827 			"default volume queue depth");
1828 
1829 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
1830 			"vol_resync_rate", CTLTYPE_INT | CTLFLAG_RW, mpt, 0,
1831 			mpt_raid_sysctl_vol_resync_rate, "I",
1832 			"volume resync priority (0 == NC, 1 - 255)");
1833 	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
1834 			"nonoptimal_volumes", CTLFLAG_RD,
1835 			&mpt->raid_nonopt_volumes, 0,
1836 			"number of nonoptimal volumes");
1837 }
1838 #endif
1839