xref: /freebsd/sys/geom/vinum/geom_vinum_plex.c (revision c3aadfb9)
1 /*-
2  * Copyright (c) 2004 Lukas Ertl
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bio.h>
32 #include <sys/kernel.h>
33 #include <sys/kthread.h>
34 #include <sys/libkern.h>
35 #include <sys/lock.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/mutex.h>
39 #include <sys/systm.h>
40 
41 #include <geom/geom.h>
42 #include <geom/vinum/geom_vinum_var.h>
43 #include <geom/vinum/geom_vinum_raid5.h>
44 #include <geom/vinum/geom_vinum.h>
45 
46 static void gv_plex_completed_request(struct gv_plex *, struct bio *);
47 static void gv_plex_normal_request(struct gv_plex *, struct bio *);
48 static void gv_plex_worker(void *);
49 
50 /* XXX: is this the place to catch dying subdisks? */
51 static void
52 gv_plex_orphan(struct g_consumer *cp)
53 {
54 	struct g_geom *gp;
55 	struct gv_plex *p;
56 	int error;
57 
58 	g_topology_assert();
59 	gp = cp->geom;
60 	g_trace(G_T_TOPOLOGY, "gv_plex_orphan(%s)", gp->name);
61 
62 	if (cp->acr != 0 || cp->acw != 0 || cp->ace != 0)
63 		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
64 	error = cp->provider->error;
65 	if (error == 0)
66 		error = ENXIO;
67 	g_detach(cp);
68 	g_destroy_consumer(cp);
69 	if (!LIST_EMPTY(&gp->consumer))
70 		return;
71 
72 	p = gp->softc;
73 	if (p != NULL) {
74 		gv_kill_plex_thread(p);
75 		p->geom = NULL;
76 		p->provider = NULL;
77 		p->consumer = NULL;
78 	}
79 	gp->softc = NULL;
80 	g_wither_geom(gp, error);
81 }
82 
83 void
84 gv_plex_done(struct bio *bp)
85 {
86 	struct gv_plex *p;
87 	struct gv_bioq *bq;
88 
89 	p = bp->bio_from->geom->softc;
90 	bp->bio_cflags |= GV_BIO_DONE;
91 	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
92 	bq->bp = bp;
93 	mtx_lock(&p->bqueue_mtx);
94 	TAILQ_INSERT_TAIL(&p->bqueue, bq, queue);
95 	wakeup(p);
96 	mtx_unlock(&p->bqueue_mtx);
97 }
98 
99 /* Find the correct subdisk to send the bio to and build a bio to send. */
100 static int
101 gv_plexbuffer(struct gv_plex *p, struct bio *bp, caddr_t addr, off_t boff, off_t bcount)
102 {
103 	struct g_geom *gp;
104 	struct gv_sd *s;
105 	struct bio *cbp, *pbp;
106 	int i, sdno;
107 	off_t len_left, real_len, real_off;
108 	off_t stripeend, stripeno, stripestart;
109 
110 	if (p == NULL || LIST_EMPTY(&p->subdisks))
111 		return (ENXIO);
112 
113 	s = NULL;
114 	gp = bp->bio_to->geom;
115 
116 	/*
117 	 * We only handle concatenated and striped plexes here.  RAID5 plexes
118 	 * are handled in build_raid5_request().
119 	 */
120 	switch (p->org) {
121 	case GV_PLEX_CONCAT:
122 		/*
123 		 * Find the subdisk where this request starts.  The subdisks in
124 		 * this list must be ordered by plex_offset.
125 		 */
126 		LIST_FOREACH(s, &p->subdisks, in_plex) {
127 			if (s->plex_offset <= boff &&
128 			    s->plex_offset + s->size > boff)
129 				break;
130 		}
131 		/* Subdisk not found. */
132 		if (s == NULL)
133 			return (ENXIO);
134 
135 		/* Calculate corresponding offsets on disk. */
136 		real_off = boff - s->plex_offset;
137 		len_left = s->size - real_off;
138 		real_len = (bcount > len_left) ? len_left : bcount;
139 		break;
140 
141 	case GV_PLEX_STRIPED:
142 		/* The number of the stripe where the request starts. */
143 		stripeno = boff / p->stripesize;
144 
145 		/* The number of the subdisk where the stripe resides. */
146 		sdno = stripeno % p->sdcount;
147 
148 		/* Find the right subdisk. */
149 		i = 0;
150 		LIST_FOREACH(s, &p->subdisks, in_plex) {
151 			if (i == sdno)
152 				break;
153 			i++;
154 		}
155 
156 		/* Subdisk not found. */
157 		if (s == NULL)
158 			return (ENXIO);
159 
160 		/* The offset of the stripe from the start of the subdisk. */
161 		stripestart = (stripeno / p->sdcount) *
162 		    p->stripesize;
163 
164 		/* The offset at the end of the stripe. */
165 		stripeend = stripestart + p->stripesize;
166 
167 		/* The offset of the request on this subdisk. */
168 		real_off = boff - (stripeno * p->stripesize) +
169 		    stripestart;
170 
171 		/* The length left in this stripe. */
172 		len_left = stripeend - real_off;
173 
174 		real_len = (bcount <= len_left) ? bcount : len_left;
175 		break;
176 
177 	default:
178 		return (EINVAL);
179 	}
180 
181 	/* Now check if we can handle the request on this subdisk. */
182 	switch (s->state) {
183 	case GV_SD_UP:
184 		/* If the subdisk is up, just continue. */
185 		break;
186 
187 	case GV_SD_STALE:
188 		if (!(bp->bio_cflags & GV_BIO_SYNCREQ))
189 			return (ENXIO);
190 
191 		printf("GEOM_VINUM: sd %s is initializing\n", s->name);
192 		gv_set_sd_state(s, GV_SD_INITIALIZING, GV_SETSTATE_FORCE);
193 		break;
194 
195 	case GV_SD_INITIALIZING:
196 		if (bp->bio_cmd == BIO_READ)
197 			return (ENXIO);
198 		break;
199 
200 	default:
201 		/* All other subdisk states mean it's not accessible. */
202 		return (ENXIO);
203 	}
204 
205 	/* Clone the bio and adjust the offsets and sizes. */
206 	cbp = g_clone_bio(bp);
207 	if (cbp == NULL)
208 		return (ENOMEM);
209 	cbp->bio_offset = real_off;
210 	cbp->bio_length = real_len;
211 	cbp->bio_data = addr;
212 	cbp->bio_done = g_std_done;
213 	cbp->bio_caller2 = s->consumer;
214 	if ((bp->bio_cflags & GV_BIO_SYNCREQ)) {
215 		cbp->bio_cflags |= GV_BIO_SYNCREQ;
216 		cbp->bio_done = gv_plex_done;
217 	}
218 
219 	if (bp->bio_driver1 == NULL) {
220 		bp->bio_driver1 = cbp;
221 	} else {
222 		pbp = bp->bio_driver1;
223 		while (pbp->bio_caller1 != NULL)
224 			pbp = pbp->bio_caller1;
225 		pbp->bio_caller1 = cbp;
226 	}
227 
228 	return (0);
229 }
230 
231 static void
232 gv_plex_start(struct bio *bp)
233 {
234 	struct gv_plex *p;
235 	struct gv_bioq *bq;
236 
237 	switch(bp->bio_cmd) {
238 	case BIO_READ:
239 	case BIO_WRITE:
240 	case BIO_DELETE:
241 		break;
242 	case BIO_GETATTR:
243 	default:
244 		g_io_deliver(bp, EOPNOTSUPP);
245 		return;
246 	}
247 
248 	/*
249 	 * We cannot handle this request if too many of our subdisks are
250 	 * inaccessible.
251 	 */
252 	p = bp->bio_to->geom->softc;
253 	if ((p->state < GV_PLEX_DEGRADED) &&
254 	    !(bp->bio_cflags & GV_BIO_SYNCREQ)) {
255 		g_io_deliver(bp, ENXIO);
256 		return;
257 	}
258 
259 	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
260 	bq->bp = bp;
261 	mtx_lock(&p->bqueue_mtx);
262 	TAILQ_INSERT_TAIL(&p->bqueue, bq, queue);
263 	wakeup(p);
264 	mtx_unlock(&p->bqueue_mtx);
265 }
266 
267 static void
268 gv_plex_worker(void *arg)
269 {
270 	struct bio *bp;
271 	struct gv_plex *p;
272 	struct gv_sd *s;
273 	struct gv_bioq *bq;
274 
275 	p = arg;
276 	KASSERT(p != NULL, ("NULL p"));
277 
278 	mtx_lock(&p->bqueue_mtx);
279 	for (;;) {
280 		/* We were signaled to exit. */
281 		if (p->flags & GV_PLEX_THREAD_DIE)
282 			break;
283 
284 		/* Take the first BIO from our queue. */
285 		bq = TAILQ_FIRST(&p->bqueue);
286 		if (bq == NULL) {
287 			msleep(p, &p->bqueue_mtx, PRIBIO, "-", hz/10);
288 			continue;
289 		}
290 		TAILQ_REMOVE(&p->bqueue, bq, queue);
291 		mtx_unlock(&p->bqueue_mtx);
292 
293 		bp = bq->bp;
294 
295 		/* A completed request. */
296 		if (bp->bio_cflags & GV_BIO_DONE) {
297 			g_free(bq);
298 
299 			if (bp->bio_cflags & GV_BIO_SYNCREQ ||
300 			    bp->bio_cflags & GV_BIO_REBUILD) {
301 				s = bp->bio_to->private;
302 				if (bp->bio_error == 0)
303 					s->initialized += bp->bio_length;
304 				if (s->initialized >= s->size) {
305 					g_topology_lock();
306 					gv_set_sd_state(s, GV_SD_UP,
307 					    GV_SETSTATE_CONFIG);
308 					g_topology_unlock();
309 					s->initialized = 0;
310 				}
311 			}
312 
313 			if (bp->bio_cflags & GV_BIO_SYNCREQ)
314 				g_std_done(bp);
315 			else
316 				gv_plex_completed_request(p, bp);
317 		/*
318 		 * A sub-request that was hold back because it interfered with
319 		 * another sub-request.
320 		 */
321 		} else if (bp->bio_cflags & GV_BIO_ONHOLD) {
322 			/* Is it still locked out? */
323 			if (gv_stripe_active(p, bp)) {
324 				mtx_lock(&p->bqueue_mtx);
325 				TAILQ_INSERT_TAIL(&p->bqueue, bq, queue);
326 				mtx_unlock(&p->bqueue_mtx);
327 			} else {
328 				g_free(bq);
329 				bp->bio_cflags &= ~GV_BIO_ONHOLD;
330 				g_io_request(bp, bp->bio_caller2);
331 			}
332 
333 		/* A normal request to this plex. */
334 		} else {
335 			g_free(bq);
336 			gv_plex_normal_request(p, bp);
337 		}
338 
339 		mtx_lock(&p->bqueue_mtx);
340 	}
341 	mtx_unlock(&p->bqueue_mtx);
342 	p->flags |= GV_PLEX_THREAD_DEAD;
343 	wakeup(p);
344 
345 	kthread_exit(ENXIO);
346 }
347 
348 void
349 gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
350 {
351 	struct bio *cbp, *pbp;
352 	struct gv_bioq *bq, *bq2;
353 	struct gv_raid5_packet *wp;
354 	int i;
355 
356 	wp = bp->bio_driver1;
357 
358 	switch (bp->bio_parent->bio_cmd) {
359 	case BIO_READ:
360 		if (wp == NULL)
361 			break;
362 
363 		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
364 			if (bq->bp == bp) {
365 				TAILQ_REMOVE(&wp->bits, bq, queue);
366 				g_free(bq);
367 				for (i = 0; i < wp->length; i++)
368 					wp->data[i] ^= bp->bio_data[i];
369 				break;
370 			}
371 		}
372 		if (TAILQ_EMPTY(&wp->bits)) {
373 			bp->bio_parent->bio_completed += wp->length;
374 			if (wp->lockbase != -1)
375 				TAILQ_REMOVE(&p->packets, wp, list);
376 			g_free(wp);
377 		}
378 
379 		break;
380 
381  	case BIO_WRITE:
382 		if (wp == NULL)
383 			break;
384 
385 		/* Check if we need to handle parity data. */
386 		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
387 			if (bq->bp == bp) {
388 				TAILQ_REMOVE(&wp->bits, bq, queue);
389 				g_free(bq);
390 				cbp = wp->parity;
391 				if (cbp != NULL) {
392 					for (i = 0; i < wp->length; i++)
393 						cbp->bio_data[i] ^=
394 						    bp->bio_data[i];
395 				}
396 				break;
397 			}
398 		}
399 
400 		/* Handle parity data. */
401 		if (TAILQ_EMPTY(&wp->bits)) {
402 			if (wp->waiting != NULL) {
403 				pbp = wp->waiting;
404 				wp->waiting = NULL;
405 				cbp = wp->parity;
406 				for (i = 0; i < wp->length; i++)
407 					cbp->bio_data[i] ^= pbp->bio_data[i];
408 				g_io_request(pbp, pbp->bio_caller2);
409 			} else if (wp->parity != NULL) {
410 				cbp = wp->parity;
411 				wp->parity = NULL;
412 				g_io_request(cbp, cbp->bio_caller2);
413 			} else {
414 				bp->bio_parent->bio_completed += wp->length;
415 				TAILQ_REMOVE(&p->packets, wp, list);
416 				g_free(wp);
417 			}
418 		}
419 
420 		break;
421 	}
422 
423 	pbp = bp->bio_parent;
424 	if (pbp->bio_error == 0)
425 		pbp->bio_error = bp->bio_error;
426 
427 	/* When the original request is finished, we deliver it. */
428 	pbp->bio_inbed++;
429 	if (pbp->bio_inbed == pbp->bio_children)
430 		g_io_deliver(pbp, pbp->bio_error);
431 
432 	/* Clean up what we allocated. */
433 	if (bp->bio_cflags & GV_BIO_MALLOC)
434 		g_free(bp->bio_data);
435 	g_destroy_bio(bp);
436 }
437 
438 void
439 gv_plex_normal_request(struct gv_plex *p, struct bio *bp)
440 {
441 	struct bio *cbp, *pbp;
442 	struct gv_bioq *bq, *bq2;
443 	struct gv_raid5_packet *wp, *wp2;
444 	caddr_t addr;
445 	off_t bcount, boff;
446 	int err;
447 
448 	bcount = bp->bio_length;
449 	addr = bp->bio_data;
450 	boff = bp->bio_offset;
451 
452 	/* Walk over the whole length of the request, we might split it up. */
453 	while (bcount > 0) {
454 		wp = NULL;
455 
456  		/*
457 		 * RAID5 plexes need special treatment, as a single write
458 		 * request involves several read/write sub-requests.
459  		 */
460 		if (p->org == GV_PLEX_RAID5) {
461 			wp = g_malloc(sizeof(*wp), M_WAITOK | M_ZERO);
462 			wp->bio = bp;
463 			TAILQ_INIT(&wp->bits);
464 
465 			if (bp->bio_cflags & GV_BIO_REBUILD)
466 				err = gv_rebuild_raid5(p, wp, bp, addr,
467 				    boff, bcount);
468 			else
469 				err = gv_build_raid5_req(p, wp, bp, addr,
470 				    boff, bcount);
471 
472  			/*
473 			 * Building the sub-request failed, we probably need to
474 			 * clean up a lot.
475  			 */
476  			if (err) {
477 				printf("GEOM_VINUM: plex request failed for ");
478 				g_print_bio(bp);
479 				printf("\n");
480 				TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
481 					TAILQ_REMOVE(&wp->bits, bq, queue);
482 					g_free(bq);
483 				}
484 				if (wp->waiting != NULL) {
485 					if (wp->waiting->bio_cflags &
486 					    GV_BIO_MALLOC)
487 						g_free(wp->waiting->bio_data);
488 					g_destroy_bio(wp->waiting);
489 				}
490 				if (wp->parity != NULL) {
491 					if (wp->parity->bio_cflags &
492 					    GV_BIO_MALLOC)
493 						g_free(wp->parity->bio_data);
494 					g_destroy_bio(wp->parity);
495 				}
496 				g_free(wp);
497 
498 				TAILQ_FOREACH_SAFE(wp, &p->packets, list, wp2) {
499 					if (wp->bio == bp) {
500 						TAILQ_REMOVE(&p->packets, wp,
501 						    list);
502 						TAILQ_FOREACH_SAFE(bq,
503 						    &wp->bits, queue, bq2) {
504 							TAILQ_REMOVE(&wp->bits,
505 							    bq, queue);
506 							g_free(bq);
507 						}
508 						g_free(wp);
509 					}
510 				}
511 
512 				cbp = bp->bio_driver1;
513 				while (cbp != NULL) {
514 					pbp = cbp->bio_caller1;
515 					if (cbp->bio_cflags & GV_BIO_MALLOC)
516 						g_free(cbp->bio_data);
517 					g_destroy_bio(cbp);
518 					cbp = pbp;
519 				}
520 
521 				g_io_deliver(bp, err);
522  				return;
523  			}
524 
525 			if (TAILQ_EMPTY(&wp->bits))
526 				g_free(wp);
527 			else if (wp->lockbase != -1)
528 				TAILQ_INSERT_TAIL(&p->packets, wp, list);
529 
530 		/*
531 		 * Requests to concatenated and striped plexes go straight
532 		 * through.
533 		 */
534 		} else {
535 			err = gv_plexbuffer(p, bp, addr, boff, bcount);
536 
537 			/* Building the sub-request failed. */
538 			if (err) {
539 				printf("GEOM_VINUM: plex request failed for ");
540 				g_print_bio(bp);
541 				printf("\n");
542 				cbp = bp->bio_driver1;
543 				while (cbp != NULL) {
544 					pbp = cbp->bio_caller1;
545 					g_destroy_bio(cbp);
546 					cbp = pbp;
547 				}
548 				g_io_deliver(bp, err);
549 				return;
550 			}
551 		}
552 
553 		/* Abuse bio_caller1 as linked list. */
554 		pbp = bp->bio_driver1;
555 		while (pbp->bio_caller1 != NULL)
556 			pbp = pbp->bio_caller1;
557 		bcount -= pbp->bio_length;
558 		addr += pbp->bio_length;
559 		boff += pbp->bio_length;
560 	}
561 
562 	/* Fire off all sub-requests. */
563 	pbp = bp->bio_driver1;
564 	while (pbp != NULL) {
565 		/*
566 		 * RAID5 sub-requests need to come in correct order, otherwise
567 		 * we trip over the parity, as it might be overwritten by
568 		 * another sub-request.
569 		 */
570 		if (pbp->bio_driver1 != NULL &&
571 		    gv_stripe_active(p, pbp)) {
572 			pbp->bio_cflags |= GV_BIO_ONHOLD;
573 			bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
574 			bq->bp = pbp;
575 			mtx_lock(&p->bqueue_mtx);
576 			TAILQ_INSERT_TAIL(&p->bqueue, bq, queue);
577 			mtx_unlock(&p->bqueue_mtx);
578 		} else
579 			g_io_request(pbp, pbp->bio_caller2);
580 		pbp = pbp->bio_caller1;
581 	}
582 }
583 
584 static int
585 gv_plex_access(struct g_provider *pp, int dr, int dw, int de)
586 {
587 	struct g_geom *gp;
588 	struct g_consumer *cp, *cp2;
589 	int error;
590 
591 	gp = pp->geom;
592 
593 	error = ENXIO;
594 	LIST_FOREACH(cp, &gp->consumer, consumer) {
595 		error = g_access(cp, dr, dw, de);
596 		if (error) {
597 			LIST_FOREACH(cp2, &gp->consumer, consumer) {
598 				if (cp == cp2)
599 					break;
600 				g_access(cp2, -dr, -dw, -de);
601 			}
602 			return (error);
603 		}
604 	}
605 	return (error);
606 }
607 
608 static struct g_geom *
609 gv_plex_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
610 {
611 	struct g_geom *gp;
612 	struct g_consumer *cp, *cp2;
613 	struct g_provider *pp2;
614 	struct gv_plex *p;
615 	struct gv_sd *s;
616 	struct gv_softc *sc;
617 	int error;
618 
619 	g_trace(G_T_TOPOLOGY, "gv_plex_taste(%s, %s)", mp->name, pp->name);
620 	g_topology_assert();
621 
622 	/* We only want to attach to subdisks. */
623 	if (strcmp(pp->geom->class->name, "VINUMDRIVE"))
624 		return (NULL);
625 
626 	/* Find the VINUM class and its associated geom. */
627 	gp = find_vinum_geom();
628 	if (gp == NULL)
629 		return (NULL);
630 	sc = gp->softc;
631 	KASSERT(sc != NULL, ("gv_plex_taste: NULL sc"));
632 
633 	/* Find out which subdisk the offered provider corresponds to. */
634 	s = pp->private;
635 	KASSERT(s != NULL, ("gv_plex_taste: NULL s"));
636 
637 	/* Now find the correct plex where this subdisk belongs to. */
638 	p = gv_find_plex(sc, s->plex);
639 	KASSERT(p != NULL, ("gv_plex_taste: NULL p"));
640 
641 	/*
642 	 * Add this subdisk to this plex.  Since we trust the on-disk
643 	 * configuration, we don't check the given value (should we?).
644 	 * XXX: shouldn't be done here
645 	 */
646 	gv_sd_to_plex(p, s, 0);
647 
648 	/* Now check if there's already a geom for this plex. */
649 	gp = p->geom;
650 
651 	/* Yes, there is already a geom, so we just add the consumer. */
652 	if (gp != NULL) {
653 		cp2 = LIST_FIRST(&gp->consumer);
654 		/* Need to attach a new consumer to this subdisk. */
655 		cp = g_new_consumer(gp);
656 		error = g_attach(cp, pp);
657 		if (error) {
658 			printf("geom_vinum: couldn't attach consumer to %s\n",
659 			    pp->name);
660 			g_destroy_consumer(cp);
661 			return (NULL);
662 		}
663 		/* Adjust the access counts of the new consumer. */
664 		if ((cp2 != NULL) && (cp2->acr || cp2->acw || cp2->ace)) {
665 			error = g_access(cp, cp2->acr, cp2->acw, cp2->ace);
666 			if (error) {
667 				printf("geom_vinum: couldn't set access counts"
668 				    " for consumer on %s\n", pp->name);
669 				g_detach(cp);
670 				g_destroy_consumer(cp);
671 				return (NULL);
672 			}
673 		}
674 		s->consumer = cp;
675 
676 		/* Adjust the size of the providers this plex has. */
677 		LIST_FOREACH(pp2, &gp->provider, provider)
678 			pp2->mediasize = p->size;
679 
680 		/* Update the size of the volume this plex is attached to. */
681 		if (p->vol_sc != NULL)
682 			gv_update_vol_size(p->vol_sc, p->size);
683 
684 		return (NULL);
685 
686 	/* We need to create a new geom. */
687 	} else {
688 		gp = g_new_geomf(mp, "%s", p->name);
689 		gp->start = gv_plex_start;
690 		gp->orphan = gv_plex_orphan;
691 		gp->access = gv_plex_access;
692 		gp->softc = p;
693 		p->geom = gp;
694 
695 		TAILQ_INIT(&p->packets);
696 		TAILQ_INIT(&p->bqueue);
697 		mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF);
698 		kthread_create(gv_plex_worker, p, NULL, 0, 0, "gv_p %s",
699 		    p->name);
700 		p->flags |= GV_PLEX_THREAD_ACTIVE;
701 
702 		/* Attach a consumer to this provider. */
703 		cp = g_new_consumer(gp);
704 		g_attach(cp, pp);
705 		s->consumer = cp;
706 
707 		/* Create a provider for the outside world. */
708 		pp2 = g_new_providerf(gp, "gvinum/plex/%s", p->name);
709 		pp2->mediasize = p->size;
710 		pp2->sectorsize = pp->sectorsize;
711 		p->provider = pp2;
712 		g_error_provider(pp2, 0);
713 		return (gp);
714 	}
715 }
716 
717 static int
718 gv_plex_destroy_geom(struct gctl_req *req, struct g_class *mp,
719     struct g_geom *gp)
720 {
721 	struct gv_plex *p;
722 
723 	g_trace(G_T_TOPOLOGY, "gv_plex_destroy_geom: %s", gp->name);
724 	g_topology_assert();
725 
726 	p = gp->softc;
727 
728 	KASSERT(p != NULL, ("gv_plex_destroy_geom: null p of '%s'", gp->name));
729 
730 	/*
731 	 * If this is a RAID5 plex, check if its worker thread is still active
732 	 * and signal it to self destruct.
733 	 */
734 	gv_kill_plex_thread(p);
735 	/* g_free(sc); */
736 	g_wither_geom(gp, ENXIO);
737 	return (0);
738 }
739 
740 #define	VINUMPLEX_CLASS_NAME "VINUMPLEX"
741 
742 static struct g_class g_vinum_plex_class = {
743 	.name = VINUMPLEX_CLASS_NAME,
744 	.version = G_VERSION,
745 	.taste = gv_plex_taste,
746 	.destroy_geom = gv_plex_destroy_geom,
747 };
748 
749 DECLARE_GEOM_CLASS(g_vinum_plex_class, g_vinum_plex);
750