1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
23 * All rights reserved.
24 *
25 * Portions Copyright (c) 2012 Martin Matuska <mm@FreeBSD.org>
26 */
27
28 #include <sys/zfs_context.h>
29 #include <sys/param.h>
30 #include <sys/kernel.h>
31 #include <sys/bio.h>
32 #include <sys/buf.h>
33 #include <sys/file.h>
34 #include <sys/spa.h>
35 #include <sys/spa_impl.h>
36 #include <sys/vdev_impl.h>
37 #include <sys/vdev_os.h>
38 #include <sys/fs/zfs.h>
39 #include <sys/zio.h>
40 #include <vm/vm_page.h>
41 #include <geom/geom.h>
42 #include <geom/geom_disk.h>
43 #include <geom/geom_int.h>
44
45 #ifndef g_topology_locked
46 #define g_topology_locked() sx_xlocked(&topology_lock)
47 #endif
48
49 /*
50 * Virtual device vector for GEOM.
51 */
52
53 static g_attrchanged_t vdev_geom_attrchanged;
54 struct g_class zfs_vdev_class = {
55 .name = "ZFS::VDEV",
56 .version = G_VERSION,
57 .attrchanged = vdev_geom_attrchanged,
58 };
59
60 struct consumer_vdev_elem {
61 SLIST_ENTRY(consumer_vdev_elem) elems;
62 vdev_t *vd;
63 };
64
65 SLIST_HEAD(consumer_priv_t, consumer_vdev_elem);
66 _Static_assert(
67 sizeof (((struct g_consumer *)NULL)->private) ==
68 sizeof (struct consumer_priv_t *),
69 "consumer_priv_t* can't be stored in g_consumer.private");
70
71 DECLARE_GEOM_CLASS(zfs_vdev_class, zfs_vdev);
72
73 SYSCTL_DECL(_vfs_zfs_vdev);
74 /* Don't send BIO_FLUSH. */
75 static int vdev_geom_bio_flush_disable;
76 SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, bio_flush_disable, CTLFLAG_RWTUN,
77 &vdev_geom_bio_flush_disable, 0, "Disable BIO_FLUSH");
78 /* Don't send BIO_DELETE. */
79 static int vdev_geom_bio_delete_disable;
80 SYSCTL_INT(_vfs_zfs_vdev, OID_AUTO, bio_delete_disable, CTLFLAG_RWTUN,
81 &vdev_geom_bio_delete_disable, 0, "Disable BIO_DELETE");
82
83 /* Declare local functions */
84 static void vdev_geom_detach(struct g_consumer *cp, boolean_t open_for_read);
85
86 /*
87 * Thread local storage used to indicate when a thread is probing geoms
88 * for their guids. If NULL, this thread is not tasting geoms. If non NULL,
89 * it is looking for a replacement for the vdev_t* that is its value.
90 */
91 uint_t zfs_geom_probe_vdev_key;
92
93 static void
vdev_geom_set_physpath(vdev_t * vd,struct g_consumer * cp,boolean_t do_null_update)94 vdev_geom_set_physpath(vdev_t *vd, struct g_consumer *cp,
95 boolean_t do_null_update)
96 {
97 boolean_t needs_update = B_FALSE;
98 char *physpath;
99 int error, physpath_len;
100
101 physpath_len = MAXPATHLEN;
102 physpath = g_malloc(physpath_len, M_WAITOK|M_ZERO);
103 error = g_io_getattr("GEOM::physpath", cp, &physpath_len, physpath);
104 if (error == 0) {
105 char *old_physpath;
106
107 /* g_topology lock ensures that vdev has not been closed */
108 g_topology_assert();
109 old_physpath = vd->vdev_physpath;
110 vd->vdev_physpath = spa_strdup(physpath);
111
112 if (old_physpath != NULL) {
113 needs_update = (strcmp(old_physpath,
114 vd->vdev_physpath) != 0);
115 spa_strfree(old_physpath);
116 } else
117 needs_update = do_null_update;
118 }
119 g_free(physpath);
120
121 /*
122 * If the physical path changed, update the config.
123 * Only request an update for previously unset physpaths if
124 * requested by the caller.
125 */
126 if (needs_update)
127 spa_async_request(vd->vdev_spa, SPA_ASYNC_CONFIG_UPDATE);
128
129 }
130
131 static void
vdev_geom_attrchanged(struct g_consumer * cp,const char * attr)132 vdev_geom_attrchanged(struct g_consumer *cp, const char *attr)
133 {
134 struct consumer_priv_t *priv;
135 struct consumer_vdev_elem *elem;
136
137 priv = (struct consumer_priv_t *)&cp->private;
138 if (SLIST_EMPTY(priv))
139 return;
140
141 SLIST_FOREACH(elem, priv, elems) {
142 vdev_t *vd = elem->vd;
143 if (strcmp(attr, "GEOM::physpath") == 0) {
144 vdev_geom_set_physpath(vd, cp, /* null_update */B_TRUE);
145 return;
146 }
147 }
148 }
149
150 static void
vdev_geom_resize(struct g_consumer * cp)151 vdev_geom_resize(struct g_consumer *cp)
152 {
153 struct consumer_priv_t *priv;
154 struct consumer_vdev_elem *elem;
155 spa_t *spa;
156 vdev_t *vd;
157
158 priv = (struct consumer_priv_t *)&cp->private;
159 if (SLIST_EMPTY(priv))
160 return;
161
162 SLIST_FOREACH(elem, priv, elems) {
163 vd = elem->vd;
164 if (vd->vdev_state != VDEV_STATE_HEALTHY)
165 continue;
166 spa = vd->vdev_spa;
167 if (!spa->spa_autoexpand)
168 continue;
169 vdev_online(spa, vd->vdev_guid, ZFS_ONLINE_EXPAND, NULL);
170 }
171 }
172
173 static void
vdev_geom_orphan(struct g_consumer * cp)174 vdev_geom_orphan(struct g_consumer *cp)
175 {
176 struct consumer_priv_t *priv;
177 // cppcheck-suppress uninitvar
178 struct consumer_vdev_elem *elem;
179
180 g_topology_assert();
181
182 priv = (struct consumer_priv_t *)&cp->private;
183 if (SLIST_EMPTY(priv))
184 /* Vdev close in progress. Ignore the event. */
185 return;
186
187 /*
188 * Orphan callbacks occur from the GEOM event thread.
189 * Concurrent with this call, new I/O requests may be
190 * working their way through GEOM about to find out
191 * (only once executed by the g_down thread) that we've
192 * been orphaned from our disk provider. These I/Os
193 * must be retired before we can detach our consumer.
194 * This is most easily achieved by acquiring the
195 * SPA ZIO configuration lock as a writer, but doing
196 * so with the GEOM topology lock held would cause
197 * a lock order reversal. Instead, rely on the SPA's
198 * async removal support to invoke a close on this
199 * vdev once it is safe to do so.
200 */
201 SLIST_FOREACH(elem, priv, elems) {
202 // cppcheck-suppress uninitvar
203 vdev_t *vd = elem->vd;
204
205 vd->vdev_remove_wanted = B_TRUE;
206 spa_async_request(vd->vdev_spa, SPA_ASYNC_REMOVE);
207 }
208 }
209
210 static struct g_consumer *
vdev_geom_attach(struct g_provider * pp,vdev_t * vd,boolean_t sanity)211 vdev_geom_attach(struct g_provider *pp, vdev_t *vd, boolean_t sanity)
212 {
213 struct g_geom *gp;
214 struct g_consumer *cp;
215 int error;
216
217 g_topology_assert();
218
219 ZFS_LOG(1, "Attaching to %s.", pp->name);
220
221 if (sanity) {
222 if (pp->sectorsize > VDEV_PAD_SIZE || !ISP2(pp->sectorsize)) {
223 ZFS_LOG(1, "Failing attach of %s. "
224 "Incompatible sectorsize %d\n",
225 pp->name, pp->sectorsize);
226 return (NULL);
227 } else if (pp->mediasize < SPA_MINDEVSIZE) {
228 ZFS_LOG(1, "Failing attach of %s. "
229 "Incompatible mediasize %ju\n",
230 pp->name, pp->mediasize);
231 return (NULL);
232 }
233 }
234
235 /* Do we have geom already? No? Create one. */
236 LIST_FOREACH(gp, &zfs_vdev_class.geom, geom) {
237 if (gp->flags & G_GEOM_WITHER)
238 continue;
239 if (strcmp(gp->name, "zfs::vdev") != 0)
240 continue;
241 break;
242 }
243 if (gp == NULL) {
244 gp = g_new_geomf(&zfs_vdev_class, "zfs::vdev");
245 gp->orphan = vdev_geom_orphan;
246 gp->attrchanged = vdev_geom_attrchanged;
247 gp->resize = vdev_geom_resize;
248 cp = g_new_consumer(gp);
249 error = g_attach(cp, pp);
250 if (error != 0) {
251 ZFS_LOG(1, "%s(%d): g_attach failed: %d\n", __func__,
252 __LINE__, error);
253 vdev_geom_detach(cp, B_FALSE);
254 return (NULL);
255 }
256 error = g_access(cp, 1, 0, 1);
257 if (error != 0) {
258 ZFS_LOG(1, "%s(%d): g_access failed: %d\n", __func__,
259 __LINE__, error);
260 vdev_geom_detach(cp, B_FALSE);
261 return (NULL);
262 }
263 ZFS_LOG(1, "Created geom and consumer for %s.", pp->name);
264 } else {
265 /* Check if we are already connected to this provider. */
266 LIST_FOREACH(cp, &gp->consumer, consumer) {
267 if (cp->provider == pp) {
268 ZFS_LOG(1, "Found consumer for %s.", pp->name);
269 break;
270 }
271 }
272 if (cp == NULL) {
273 cp = g_new_consumer(gp);
274 error = g_attach(cp, pp);
275 if (error != 0) {
276 ZFS_LOG(1, "%s(%d): g_attach failed: %d\n",
277 __func__, __LINE__, error);
278 vdev_geom_detach(cp, B_FALSE);
279 return (NULL);
280 }
281 error = g_access(cp, 1, 0, 1);
282 if (error != 0) {
283 ZFS_LOG(1, "%s(%d): g_access failed: %d\n",
284 __func__, __LINE__, error);
285 vdev_geom_detach(cp, B_FALSE);
286 return (NULL);
287 }
288 ZFS_LOG(1, "Created consumer for %s.", pp->name);
289 } else {
290 error = g_access(cp, 1, 0, 1);
291 if (error != 0) {
292 ZFS_LOG(1, "%s(%d): g_access failed: %d\n",
293 __func__, __LINE__, error);
294 return (NULL);
295 }
296 ZFS_LOG(1, "Used existing consumer for %s.", pp->name);
297 }
298 }
299
300 if (vd != NULL)
301 vd->vdev_tsd = cp;
302
303 cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
304 return (cp);
305 }
306
307 static void
vdev_geom_detach(struct g_consumer * cp,boolean_t open_for_read)308 vdev_geom_detach(struct g_consumer *cp, boolean_t open_for_read)
309 {
310 struct g_geom *gp;
311
312 g_topology_assert();
313
314 ZFS_LOG(1, "Detaching from %s.",
315 cp->provider && cp->provider->name ? cp->provider->name : "NULL");
316
317 gp = cp->geom;
318 if (open_for_read)
319 g_access(cp, -1, 0, -1);
320 /* Destroy consumer on last close. */
321 if (cp->acr == 0 && cp->ace == 0) {
322 if (cp->acw > 0)
323 g_access(cp, 0, -cp->acw, 0);
324 if (cp->provider != NULL) {
325 ZFS_LOG(1, "Destroying consumer for %s.",
326 cp->provider->name ? cp->provider->name : "NULL");
327 g_detach(cp);
328 }
329 g_destroy_consumer(cp);
330 }
331 /* Destroy geom if there are no consumers left. */
332 if (LIST_EMPTY(&gp->consumer)) {
333 ZFS_LOG(1, "Destroyed geom %s.", gp->name);
334 g_wither_geom(gp, ENXIO);
335 }
336 }
337
338 static void
vdev_geom_close_locked(vdev_t * vd)339 vdev_geom_close_locked(vdev_t *vd)
340 {
341 struct g_consumer *cp;
342 struct consumer_priv_t *priv;
343 struct consumer_vdev_elem *elem, *elem_temp;
344
345 g_topology_assert();
346
347 cp = vd->vdev_tsd;
348 vd->vdev_delayed_close = B_FALSE;
349 if (cp == NULL)
350 return;
351
352 ZFS_LOG(1, "Closing access to %s.", cp->provider->name);
353 KASSERT(cp->private != NULL, ("%s: cp->private is NULL", __func__));
354 priv = (struct consumer_priv_t *)&cp->private;
355 vd->vdev_tsd = NULL;
356 SLIST_FOREACH_SAFE(elem, priv, elems, elem_temp) {
357 if (elem->vd == vd) {
358 SLIST_REMOVE(priv, elem, consumer_vdev_elem, elems);
359 g_free(elem);
360 }
361 }
362
363 vdev_geom_detach(cp, B_TRUE);
364 }
365
366 /*
367 * Issue one or more bios to the vdev in parallel
368 * cmds, datas, offsets, errors, and sizes are arrays of length ncmds. Each IO
369 * operation is described by parallel entries from each array. There may be
370 * more bios actually issued than entries in the array
371 */
372 static void
vdev_geom_io(struct g_consumer * cp,int * cmds,void ** datas,off_t * offsets,off_t * sizes,int * errors,int ncmds)373 vdev_geom_io(struct g_consumer *cp, int *cmds, void **datas, off_t *offsets,
374 off_t *sizes, int *errors, int ncmds)
375 {
376 struct bio **bios;
377 uint8_t *p;
378 off_t off, maxio, s, end;
379 int i, n_bios, j;
380 size_t bios_size;
381
382 #if __FreeBSD_version > 1300130
383 maxio = maxphys - (maxphys % cp->provider->sectorsize);
384 #else
385 maxio = MAXPHYS - (MAXPHYS % cp->provider->sectorsize);
386 #endif
387 n_bios = 0;
388
389 /* How many bios are required for all commands ? */
390 for (i = 0; i < ncmds; i++)
391 n_bios += (sizes[i] + maxio - 1) / maxio;
392
393 /* Allocate memory for the bios */
394 bios_size = n_bios * sizeof (struct bio *);
395 bios = kmem_zalloc(bios_size, KM_SLEEP);
396
397 /* Prepare and issue all of the bios */
398 for (i = j = 0; i < ncmds; i++) {
399 off = offsets[i];
400 p = datas[i];
401 s = sizes[i];
402 end = off + s;
403 ASSERT0(off % cp->provider->sectorsize);
404 ASSERT0(s % cp->provider->sectorsize);
405
406 for (; off < end; off += maxio, p += maxio, s -= maxio, j++) {
407 bios[j] = g_alloc_bio();
408 bios[j]->bio_cmd = cmds[i];
409 bios[j]->bio_done = NULL;
410 bios[j]->bio_offset = off;
411 bios[j]->bio_length = MIN(s, maxio);
412 bios[j]->bio_data = (caddr_t)p;
413 g_io_request(bios[j], cp);
414 }
415 }
416 ASSERT3S(j, ==, n_bios);
417
418 /* Wait for all of the bios to complete, and clean them up */
419 for (i = j = 0; i < ncmds; i++) {
420 off = offsets[i];
421 s = sizes[i];
422 end = off + s;
423
424 for (; off < end; off += maxio, s -= maxio, j++) {
425 errors[i] = biowait(bios[j], "vdev_geom_io") ||
426 errors[i];
427 g_destroy_bio(bios[j]);
428 }
429 }
430 kmem_free(bios, bios_size);
431 }
432
433 /*
434 * Read the vdev config from a device. Return the number of valid labels that
435 * were found. The vdev config will be returned in config if and only if at
436 * least one valid label was found.
437 */
438 static int
vdev_geom_read_config(struct g_consumer * cp,nvlist_t ** configp)439 vdev_geom_read_config(struct g_consumer *cp, nvlist_t **configp)
440 {
441 struct g_provider *pp;
442 nvlist_t *config;
443 vdev_phys_t *vdev_lists[VDEV_LABELS];
444 char *buf;
445 size_t buflen;
446 uint64_t psize, state, txg;
447 off_t offsets[VDEV_LABELS];
448 off_t size;
449 off_t sizes[VDEV_LABELS];
450 int cmds[VDEV_LABELS];
451 int errors[VDEV_LABELS];
452 int l, nlabels;
453
454 g_topology_assert_not();
455
456 pp = cp->provider;
457 ZFS_LOG(1, "Reading config from %s...", pp->name);
458
459 psize = pp->mediasize;
460 psize = P2ALIGN_TYPED(psize, sizeof (vdev_label_t), uint64_t);
461
462 size = sizeof (*vdev_lists[0]) + pp->sectorsize -
463 ((sizeof (*vdev_lists[0]) - 1) % pp->sectorsize) - 1;
464
465 buflen = sizeof (vdev_lists[0]->vp_nvlist);
466
467 /* Create all of the IO requests */
468 for (l = 0; l < VDEV_LABELS; l++) {
469 cmds[l] = BIO_READ;
470 vdev_lists[l] = kmem_alloc(size, KM_SLEEP);
471 offsets[l] = vdev_label_offset(psize, l, 0) + VDEV_SKIP_SIZE;
472 sizes[l] = size;
473 errors[l] = 0;
474 ASSERT0(offsets[l] % pp->sectorsize);
475 }
476
477 /* Issue the IO requests */
478 vdev_geom_io(cp, cmds, (void**)vdev_lists, offsets, sizes, errors,
479 VDEV_LABELS);
480
481 /* Parse the labels */
482 config = *configp = NULL;
483 nlabels = 0;
484 for (l = 0; l < VDEV_LABELS; l++) {
485 if (errors[l] != 0)
486 continue;
487
488 buf = vdev_lists[l]->vp_nvlist;
489
490 if (nvlist_unpack(buf, buflen, &config, 0) != 0)
491 continue;
492
493 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
494 &state) != 0 || state > POOL_STATE_L2CACHE) {
495 nvlist_free(config);
496 continue;
497 }
498
499 if (state != POOL_STATE_SPARE &&
500 state != POOL_STATE_L2CACHE &&
501 (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG,
502 &txg) != 0 || txg == 0)) {
503 nvlist_free(config);
504 continue;
505 }
506
507 if (*configp != NULL)
508 nvlist_free(*configp);
509 *configp = config;
510 nlabels++;
511 }
512
513 /* Free the label storage */
514 for (l = 0; l < VDEV_LABELS; l++)
515 kmem_free(vdev_lists[l], size);
516
517 return (nlabels);
518 }
519
520 static void
resize_configs(nvlist_t *** configs,uint64_t * count,uint64_t id)521 resize_configs(nvlist_t ***configs, uint64_t *count, uint64_t id)
522 {
523 nvlist_t **new_configs;
524 uint64_t i;
525
526 if (id < *count)
527 return;
528 new_configs = kmem_zalloc((id + 1) * sizeof (nvlist_t *),
529 KM_SLEEP);
530 for (i = 0; i < *count; i++)
531 new_configs[i] = (*configs)[i];
532 if (*configs != NULL)
533 kmem_free(*configs, *count * sizeof (void *));
534 *configs = new_configs;
535 *count = id + 1;
536 }
537
538 static void
process_vdev_config(nvlist_t *** configs,uint64_t * count,nvlist_t * cfg,const char * name,uint64_t * known_pool_guid)539 process_vdev_config(nvlist_t ***configs, uint64_t *count, nvlist_t *cfg,
540 const char *name, uint64_t *known_pool_guid)
541 {
542 nvlist_t *vdev_tree;
543 uint64_t pool_guid;
544 uint64_t vdev_guid;
545 uint64_t id, txg, known_txg;
546 const char *pname;
547
548 if (nvlist_lookup_string(cfg, ZPOOL_CONFIG_POOL_NAME, &pname) != 0 ||
549 strcmp(pname, name) != 0)
550 goto ignore;
551
552 if (nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_POOL_GUID, &pool_guid) != 0)
553 goto ignore;
554
555 if (nvlist_lookup_uint64(cfg, ZPOOL_CONFIG_TOP_GUID, &vdev_guid) != 0)
556 goto ignore;
557
558 if (nvlist_lookup_nvlist(cfg, ZPOOL_CONFIG_VDEV_TREE, &vdev_tree) != 0)
559 goto ignore;
560
561 if (nvlist_lookup_uint64(vdev_tree, ZPOOL_CONFIG_ID, &id) != 0)
562 goto ignore;
563
564 txg = fnvlist_lookup_uint64(cfg, ZPOOL_CONFIG_POOL_TXG);
565
566 if (*known_pool_guid != 0) {
567 if (pool_guid != *known_pool_guid)
568 goto ignore;
569 } else
570 *known_pool_guid = pool_guid;
571
572 resize_configs(configs, count, id);
573
574 if ((*configs)[id] != NULL) {
575 known_txg = fnvlist_lookup_uint64((*configs)[id],
576 ZPOOL_CONFIG_POOL_TXG);
577 if (txg <= known_txg)
578 goto ignore;
579 nvlist_free((*configs)[id]);
580 }
581
582 (*configs)[id] = cfg;
583 return;
584
585 ignore:
586 nvlist_free(cfg);
587 }
588
589 int
vdev_geom_read_pool_label(const char * name,nvlist_t *** configs,uint64_t * count)590 vdev_geom_read_pool_label(const char *name,
591 nvlist_t ***configs, uint64_t *count)
592 {
593 struct g_class *mp;
594 struct g_geom *gp;
595 struct g_provider *pp;
596 struct g_consumer *zcp;
597 nvlist_t *vdev_cfg;
598 uint64_t pool_guid;
599 int nlabels;
600
601 DROP_GIANT();
602 g_topology_lock();
603
604 *configs = NULL;
605 *count = 0;
606 pool_guid = 0;
607 LIST_FOREACH(mp, &g_classes, class) {
608 if (mp == &zfs_vdev_class)
609 continue;
610 LIST_FOREACH(gp, &mp->geom, geom) {
611 if (gp->flags & G_GEOM_WITHER)
612 continue;
613 LIST_FOREACH(pp, &gp->provider, provider) {
614 if (pp->flags & G_PF_WITHER)
615 continue;
616 zcp = vdev_geom_attach(pp, NULL, B_TRUE);
617 if (zcp == NULL)
618 continue;
619 g_topology_unlock();
620 nlabels = vdev_geom_read_config(zcp, &vdev_cfg);
621 g_topology_lock();
622 vdev_geom_detach(zcp, B_TRUE);
623 if (nlabels == 0)
624 continue;
625 ZFS_LOG(1, "successfully read vdev config");
626
627 process_vdev_config(configs, count,
628 vdev_cfg, name, &pool_guid);
629 }
630 }
631 }
632 g_topology_unlock();
633 PICKUP_GIANT();
634
635 return (*count > 0 ? 0 : ENOENT);
636 }
637
638 enum match {
639 NO_MATCH = 0, /* No matching labels found */
640 TOPGUID_MATCH = 1, /* Labels match top guid, not vdev guid */
641 ZERO_MATCH = 1, /* Should never be returned */
642 ONE_MATCH = 2, /* 1 label matching the vdev_guid */
643 TWO_MATCH = 3, /* 2 label matching the vdev_guid */
644 THREE_MATCH = 4, /* 3 label matching the vdev_guid */
645 FULL_MATCH = 5 /* all labels match the vdev_guid */
646 };
647
648 static enum match
vdev_attach_ok(vdev_t * vd,struct g_provider * pp)649 vdev_attach_ok(vdev_t *vd, struct g_provider *pp)
650 {
651 nvlist_t *config;
652 uint64_t pool_guid, top_guid, vdev_guid;
653 struct g_consumer *cp;
654 int nlabels;
655
656 cp = vdev_geom_attach(pp, NULL, B_TRUE);
657 if (cp == NULL) {
658 ZFS_LOG(1, "Unable to attach tasting instance to %s.",
659 pp->name);
660 return (NO_MATCH);
661 }
662 g_topology_unlock();
663 nlabels = vdev_geom_read_config(cp, &config);
664 g_topology_lock();
665 vdev_geom_detach(cp, B_TRUE);
666 if (nlabels == 0) {
667 ZFS_LOG(1, "Unable to read config from %s.", pp->name);
668 return (NO_MATCH);
669 }
670
671 pool_guid = 0;
672 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid);
673 top_guid = 0;
674 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TOP_GUID, &top_guid);
675 vdev_guid = 0;
676 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_GUID, &vdev_guid);
677 nvlist_free(config);
678
679 /*
680 * Check that the label's pool guid matches the desired guid.
681 * Inactive spares and L2ARCs do not have any pool guid in the label.
682 */
683 if (pool_guid != 0 && pool_guid != spa_guid(vd->vdev_spa)) {
684 ZFS_LOG(1, "pool guid mismatch for provider %s: %ju != %ju.",
685 pp->name,
686 (uintmax_t)spa_guid(vd->vdev_spa), (uintmax_t)pool_guid);
687 return (NO_MATCH);
688 }
689
690 /*
691 * Check that the label's vdev guid matches the desired guid.
692 * The second condition handles possible race on vdev detach, when
693 * remaining vdev receives GUID of destroyed top level mirror vdev.
694 */
695 if (vdev_guid == vd->vdev_guid) {
696 ZFS_LOG(1, "guids match for provider %s.", pp->name);
697 return (ZERO_MATCH + nlabels);
698 } else if (top_guid == vd->vdev_guid && vd == vd->vdev_top) {
699 ZFS_LOG(1, "top vdev guid match for provider %s.", pp->name);
700 return (TOPGUID_MATCH);
701 }
702 ZFS_LOG(1, "vdev guid mismatch for provider %s: %ju != %ju.",
703 pp->name, (uintmax_t)vd->vdev_guid, (uintmax_t)vdev_guid);
704 return (NO_MATCH);
705 }
706
707 static struct g_consumer *
vdev_geom_attach_by_guids(vdev_t * vd)708 vdev_geom_attach_by_guids(vdev_t *vd)
709 {
710 struct g_class *mp;
711 struct g_geom *gp;
712 struct g_provider *pp, *best_pp;
713 struct g_consumer *cp;
714 const char *vdpath;
715 enum match match, best_match;
716
717 g_topology_assert();
718
719 vdpath = vd->vdev_path + sizeof ("/dev/") - 1;
720 cp = NULL;
721 best_pp = NULL;
722 best_match = NO_MATCH;
723 LIST_FOREACH(mp, &g_classes, class) {
724 if (mp == &zfs_vdev_class)
725 continue;
726 LIST_FOREACH(gp, &mp->geom, geom) {
727 if (gp->flags & G_GEOM_WITHER)
728 continue;
729 LIST_FOREACH(pp, &gp->provider, provider) {
730 match = vdev_attach_ok(vd, pp);
731 if (match > best_match) {
732 best_match = match;
733 best_pp = pp;
734 } else if (match == best_match) {
735 if (strcmp(pp->name, vdpath) == 0) {
736 best_pp = pp;
737 }
738 }
739 if (match == FULL_MATCH)
740 goto out;
741 }
742 }
743 }
744
745 out:
746 if (best_pp) {
747 cp = vdev_geom_attach(best_pp, vd, B_TRUE);
748 if (cp == NULL) {
749 printf("ZFS WARNING: Unable to attach to %s.\n",
750 best_pp->name);
751 }
752 }
753 return (cp);
754 }
755
756 static struct g_consumer *
vdev_geom_open_by_guids(vdev_t * vd)757 vdev_geom_open_by_guids(vdev_t *vd)
758 {
759 struct g_consumer *cp;
760 char *buf;
761 size_t len;
762
763 g_topology_assert();
764
765 ZFS_LOG(1, "Searching by guids [%ju:%ju].",
766 (uintmax_t)spa_guid(vd->vdev_spa), (uintmax_t)vd->vdev_guid);
767 cp = vdev_geom_attach_by_guids(vd);
768 if (cp != NULL) {
769 len = strlen(cp->provider->name) + strlen("/dev/") + 1;
770 buf = kmem_alloc(len, KM_SLEEP);
771
772 snprintf(buf, len, "/dev/%s", cp->provider->name);
773 spa_strfree(vd->vdev_path);
774 vd->vdev_path = buf;
775
776 ZFS_LOG(1, "Attach by guid [%ju:%ju] succeeded, provider %s.",
777 (uintmax_t)spa_guid(vd->vdev_spa),
778 (uintmax_t)vd->vdev_guid, cp->provider->name);
779 } else {
780 ZFS_LOG(1, "Search by guid [%ju:%ju] failed.",
781 (uintmax_t)spa_guid(vd->vdev_spa),
782 (uintmax_t)vd->vdev_guid);
783 }
784
785 return (cp);
786 }
787
788 static struct g_consumer *
vdev_geom_open_by_path(vdev_t * vd,int check_guid)789 vdev_geom_open_by_path(vdev_t *vd, int check_guid)
790 {
791 struct g_provider *pp;
792 struct g_consumer *cp;
793
794 g_topology_assert();
795
796 cp = NULL;
797 pp = g_provider_by_name(vd->vdev_path + sizeof ("/dev/") - 1);
798 if (pp != NULL) {
799 ZFS_LOG(1, "Found provider by name %s.", vd->vdev_path);
800 if (!check_guid || vdev_attach_ok(vd, pp) == FULL_MATCH)
801 cp = vdev_geom_attach(pp, vd, B_FALSE);
802 }
803
804 return (cp);
805 }
806
807 static int
vdev_geom_open(vdev_t * vd,uint64_t * psize,uint64_t * max_psize,uint64_t * logical_ashift,uint64_t * physical_ashift)808 vdev_geom_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize,
809 uint64_t *logical_ashift, uint64_t *physical_ashift)
810 {
811 struct g_provider *pp;
812 struct g_consumer *cp;
813 int error, has_trim;
814 uint16_t rate;
815
816 /*
817 * Set the TLS to indicate downstack that we
818 * should not access zvols
819 */
820 VERIFY0(tsd_set(zfs_geom_probe_vdev_key, vd));
821
822 /*
823 * We must have a pathname, and it must be absolute.
824 */
825 if (vd->vdev_path == NULL || strncmp(vd->vdev_path, "/dev/", 5) != 0) {
826 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
827 return (EINVAL);
828 }
829
830 /*
831 * Reopen the device if it's not currently open. Otherwise,
832 * just update the physical size of the device.
833 */
834 if ((cp = vd->vdev_tsd) != NULL) {
835 ASSERT(vd->vdev_reopening);
836 goto skip_open;
837 }
838
839 DROP_GIANT();
840 g_topology_lock();
841 error = 0;
842
843 if (vd->vdev_spa->spa_is_splitting ||
844 ((vd->vdev_prevstate == VDEV_STATE_UNKNOWN &&
845 (vd->vdev_spa->spa_load_state == SPA_LOAD_NONE ||
846 vd->vdev_spa->spa_load_state == SPA_LOAD_CREATE)))) {
847 /*
848 * We are dealing with a vdev that hasn't been previously
849 * opened (since boot), and we are not loading an
850 * existing pool configuration. This looks like a
851 * vdev add operation to a new or existing pool.
852 * Assume the user really wants to do this, and find
853 * GEOM provider by its name, ignoring GUID mismatches.
854 *
855 * XXPOLICY: It would be safer to only allow a device
856 * that is unlabeled or labeled but missing
857 * GUID information to be opened in this fashion,
858 * unless we are doing a split, in which case we
859 * should allow any guid.
860 */
861 cp = vdev_geom_open_by_path(vd, 0);
862 } else {
863 /*
864 * Try using the recorded path for this device, but only
865 * accept it if its label data contains the expected GUIDs.
866 */
867 cp = vdev_geom_open_by_path(vd, 1);
868 if (cp == NULL) {
869 /*
870 * The device at vd->vdev_path doesn't have the
871 * expected GUIDs. The disks might have merely
872 * moved around so try all other GEOM providers
873 * to find one with the right GUIDs.
874 */
875 cp = vdev_geom_open_by_guids(vd);
876 }
877 }
878
879 /* Clear the TLS now that tasting is done */
880 VERIFY0(tsd_set(zfs_geom_probe_vdev_key, NULL));
881
882 if (cp == NULL) {
883 ZFS_LOG(1, "Vdev %s not found.", vd->vdev_path);
884 error = ENOENT;
885 } else {
886 struct consumer_priv_t *priv;
887 struct consumer_vdev_elem *elem;
888 int spamode;
889
890 priv = (struct consumer_priv_t *)&cp->private;
891 if (cp->private == NULL)
892 SLIST_INIT(priv);
893 elem = g_malloc(sizeof (*elem), M_WAITOK|M_ZERO);
894 elem->vd = vd;
895 SLIST_INSERT_HEAD(priv, elem, elems);
896
897 spamode = spa_mode(vd->vdev_spa);
898 if (cp->provider->sectorsize > VDEV_PAD_SIZE ||
899 !ISP2(cp->provider->sectorsize)) {
900 ZFS_LOG(1, "Provider %s has unsupported sectorsize.",
901 cp->provider->name);
902
903 vdev_geom_close_locked(vd);
904 error = EINVAL;
905 cp = NULL;
906 } else if (cp->acw == 0 && (spamode & FWRITE) != 0) {
907 int i;
908
909 for (i = 0; i < 5; i++) {
910 error = g_access(cp, 0, 1, 0);
911 if (error == 0)
912 break;
913 g_topology_unlock();
914 tsleep(vd, 0, "vdev", hz / 2);
915 g_topology_lock();
916 }
917 if (error != 0) {
918 printf("ZFS WARNING: Unable to open %s for "
919 "writing (error=%d).\n",
920 cp->provider->name, error);
921 vdev_geom_close_locked(vd);
922 cp = NULL;
923 }
924 }
925 }
926
927 /* Fetch initial physical path information for this device. */
928 if (cp != NULL) {
929 vdev_geom_attrchanged(cp, "GEOM::physpath");
930
931 /* Set other GEOM characteristics */
932 vdev_geom_set_physpath(vd, cp, /* do_null_update */B_FALSE);
933 }
934
935 g_topology_unlock();
936 PICKUP_GIANT();
937 if (cp == NULL) {
938 vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
939 vdev_dbgmsg(vd, "vdev_geom_open: failed to open [error=%d]",
940 error);
941 return (error);
942 }
943 skip_open:
944 pp = cp->provider;
945
946 /*
947 * Determine the actual size of the device.
948 */
949 *max_psize = *psize = pp->mediasize;
950
951 /*
952 * Determine the device's minimum transfer size and preferred
953 * transfer size.
954 */
955 *logical_ashift = highbit(MAX(pp->sectorsize, SPA_MINBLOCKSIZE)) - 1;
956 *physical_ashift = 0;
957 if (pp->stripesize && pp->stripesize > (1 << *logical_ashift) &&
958 ISP2(pp->stripesize) && pp->stripeoffset == 0)
959 *physical_ashift = highbit(pp->stripesize) - 1;
960
961 /*
962 * Clear the nowritecache settings, so that on a vdev_reopen()
963 * we will try again.
964 */
965 vd->vdev_nowritecache = B_FALSE;
966
967 /* Inform the ZIO pipeline that we are non-rotational. */
968 error = g_getattr("GEOM::rotation_rate", cp, &rate);
969 if (error == 0 && rate == DISK_RR_NON_ROTATING)
970 vd->vdev_nonrot = B_TRUE;
971 else
972 vd->vdev_nonrot = B_FALSE;
973
974 /* Set when device reports it supports TRIM. */
975 error = g_getattr("GEOM::candelete", cp, &has_trim);
976 vd->vdev_has_trim = (error == 0 && has_trim);
977
978 /* Set when device reports it supports secure TRIM. */
979 /* unavailable on FreeBSD */
980 vd->vdev_has_securetrim = B_FALSE;
981
982 return (0);
983 }
984
985 static void
vdev_geom_close(vdev_t * vd)986 vdev_geom_close(vdev_t *vd)
987 {
988 struct g_consumer *cp;
989 boolean_t locked;
990
991 cp = vd->vdev_tsd;
992
993 DROP_GIANT();
994 locked = g_topology_locked();
995 if (!locked)
996 g_topology_lock();
997
998 if (!vd->vdev_reopening ||
999 (cp != NULL && ((cp->flags & G_CF_ORPHAN) != 0 ||
1000 (cp->provider != NULL && cp->provider->error != 0))))
1001 vdev_geom_close_locked(vd);
1002
1003 if (!locked)
1004 g_topology_unlock();
1005 PICKUP_GIANT();
1006 }
1007
1008 static void
vdev_geom_io_intr(struct bio * bp)1009 vdev_geom_io_intr(struct bio *bp)
1010 {
1011 vdev_t *vd;
1012 zio_t *zio;
1013
1014 zio = bp->bio_caller1;
1015 vd = zio->io_vd;
1016 zio->io_error = bp->bio_error;
1017 if (zio->io_error == 0 && bp->bio_resid != 0)
1018 zio->io_error = SET_ERROR(EIO);
1019
1020 switch (zio->io_error) {
1021 case ENOTSUP:
1022 /*
1023 * If we get ENOTSUP for BIO_FLUSH or BIO_DELETE we know
1024 * that future attempts will never succeed. In this case
1025 * we set a persistent flag so that we don't bother with
1026 * requests in the future.
1027 */
1028 switch (bp->bio_cmd) {
1029 case BIO_FLUSH:
1030 vd->vdev_nowritecache = B_TRUE;
1031 break;
1032 case BIO_DELETE:
1033 break;
1034 }
1035 break;
1036 case ENXIO:
1037 if (!vd->vdev_remove_wanted) {
1038 /*
1039 * If provider's error is set we assume it is being
1040 * removed.
1041 */
1042 if (bp->bio_to->error != 0) {
1043 vd->vdev_remove_wanted = B_TRUE;
1044 spa_async_request(zio->io_spa,
1045 SPA_ASYNC_REMOVE);
1046 } else if (!vd->vdev_delayed_close) {
1047 vd->vdev_delayed_close = B_TRUE;
1048 }
1049 }
1050 break;
1051 }
1052
1053 /*
1054 * We have to split bio freeing into two parts, because the ABD code
1055 * cannot be called in this context and vdev_op_io_done is not called
1056 * for ZIO_TYPE_FLUSH zio-s.
1057 */
1058 if (zio->io_type != ZIO_TYPE_READ && zio->io_type != ZIO_TYPE_WRITE) {
1059 g_destroy_bio(bp);
1060 zio->io_bio = NULL;
1061 }
1062 zio_delay_interrupt(zio);
1063 }
1064
1065 struct vdev_geom_check_unmapped_cb_state {
1066 int pages;
1067 uint_t end;
1068 };
1069
1070 /*
1071 * Callback to check the ABD segment size/alignment and count the pages.
1072 * GEOM requires data buffer to look virtually contiguous. It means only
1073 * the first page of the buffer may not start and only the last may not
1074 * end on a page boundary. All other physical pages must be full.
1075 */
1076 static int
vdev_geom_check_unmapped_cb(void * buf,size_t len,void * priv)1077 vdev_geom_check_unmapped_cb(void *buf, size_t len, void *priv)
1078 {
1079 struct vdev_geom_check_unmapped_cb_state *s = priv;
1080 vm_offset_t off = (vm_offset_t)buf & PAGE_MASK;
1081
1082 if (s->pages != 0 && off != 0)
1083 return (1);
1084 if (s->end != 0)
1085 return (1);
1086 s->end = (off + len) & PAGE_MASK;
1087 s->pages += (off + len + PAGE_MASK) >> PAGE_SHIFT;
1088 return (0);
1089 }
1090
1091 /*
1092 * Check whether we can use unmapped I/O for this ZIO on this device to
1093 * avoid data copying between scattered and/or gang ABD buffer and linear.
1094 */
1095 static int
vdev_geom_check_unmapped(zio_t * zio,struct g_consumer * cp)1096 vdev_geom_check_unmapped(zio_t *zio, struct g_consumer *cp)
1097 {
1098 struct vdev_geom_check_unmapped_cb_state s;
1099
1100 /* If unmapped I/O is administratively disabled, respect that. */
1101 if (!unmapped_buf_allowed)
1102 return (0);
1103
1104 /* If the buffer is already linear, then nothing to do here. */
1105 if (abd_is_linear(zio->io_abd))
1106 return (0);
1107
1108 /*
1109 * If unmapped I/O is not supported by the GEOM provider,
1110 * then we can't do anything and have to copy the data.
1111 */
1112 if ((cp->provider->flags & G_PF_ACCEPT_UNMAPPED) == 0)
1113 return (0);
1114
1115 /* Check the buffer chunks sizes/alignments and count pages. */
1116 s.pages = s.end = 0;
1117 if (abd_iterate_func(zio->io_abd, 0, zio->io_size,
1118 vdev_geom_check_unmapped_cb, &s))
1119 return (0);
1120 return (s.pages);
1121 }
1122
1123 /*
1124 * Callback to translate the ABD segment into array of physical pages.
1125 */
1126 static int
vdev_geom_fill_unmap_cb(void * buf,size_t len,void * priv)1127 vdev_geom_fill_unmap_cb(void *buf, size_t len, void *priv)
1128 {
1129 struct bio *bp = priv;
1130 vm_offset_t addr = (vm_offset_t)buf;
1131 vm_offset_t end = addr + len;
1132
1133 if (bp->bio_ma_n == 0) {
1134 bp->bio_ma_offset = addr & PAGE_MASK;
1135 addr &= ~PAGE_MASK;
1136 } else {
1137 ASSERT0(P2PHASE(addr, PAGE_SIZE));
1138 }
1139 do {
1140 bp->bio_ma[bp->bio_ma_n++] =
1141 PHYS_TO_VM_PAGE(pmap_kextract(addr));
1142 addr += PAGE_SIZE;
1143 } while (addr < end);
1144 return (0);
1145 }
1146
1147 static void
vdev_geom_io_start(zio_t * zio)1148 vdev_geom_io_start(zio_t *zio)
1149 {
1150 vdev_t *vd;
1151 struct g_consumer *cp;
1152 struct bio *bp;
1153
1154 vd = zio->io_vd;
1155
1156 if (zio->io_type == ZIO_TYPE_FLUSH) {
1157 /* XXPOLICY */
1158 if (!vdev_readable(vd)) {
1159 zio->io_error = SET_ERROR(ENXIO);
1160 zio_interrupt(zio);
1161 return;
1162 }
1163
1164 if (zfs_nocacheflush || vdev_geom_bio_flush_disable) {
1165 zio_execute(zio);
1166 return;
1167 }
1168
1169 if (vd->vdev_nowritecache) {
1170 zio->io_error = SET_ERROR(ENOTSUP);
1171 zio_execute(zio);
1172 return;
1173 }
1174 } else if (zio->io_type == ZIO_TYPE_TRIM) {
1175 if (vdev_geom_bio_delete_disable) {
1176 zio_execute(zio);
1177 return;
1178 }
1179 }
1180
1181 ASSERT(zio->io_type == ZIO_TYPE_READ ||
1182 zio->io_type == ZIO_TYPE_WRITE ||
1183 zio->io_type == ZIO_TYPE_TRIM ||
1184 zio->io_type == ZIO_TYPE_FLUSH);
1185
1186 cp = vd->vdev_tsd;
1187 if (cp == NULL) {
1188 zio->io_error = SET_ERROR(ENXIO);
1189 zio_interrupt(zio);
1190 return;
1191 }
1192 bp = g_alloc_bio();
1193 bp->bio_caller1 = zio;
1194 switch (zio->io_type) {
1195 case ZIO_TYPE_READ:
1196 case ZIO_TYPE_WRITE:
1197 zio->io_target_timestamp = zio_handle_io_delay(zio);
1198 bp->bio_offset = zio->io_offset;
1199 bp->bio_length = zio->io_size;
1200 if (zio->io_type == ZIO_TYPE_READ)
1201 bp->bio_cmd = BIO_READ;
1202 else
1203 bp->bio_cmd = BIO_WRITE;
1204
1205 /*
1206 * If possible, represent scattered and/or gang ABD buffer to
1207 * GEOM as an array of physical pages. It allows to satisfy
1208 * requirement of virtually contiguous buffer without copying.
1209 */
1210 int pgs = vdev_geom_check_unmapped(zio, cp);
1211 if (pgs > 0) {
1212 bp->bio_ma = malloc(sizeof (struct vm_page *) * pgs,
1213 M_DEVBUF, M_WAITOK);
1214 bp->bio_ma_n = 0;
1215 bp->bio_ma_offset = 0;
1216 abd_iterate_func(zio->io_abd, 0, zio->io_size,
1217 vdev_geom_fill_unmap_cb, bp);
1218 bp->bio_data = unmapped_buf;
1219 bp->bio_flags |= BIO_UNMAPPED;
1220 } else {
1221 if (zio->io_type == ZIO_TYPE_READ) {
1222 bp->bio_data = abd_borrow_buf(zio->io_abd,
1223 zio->io_size);
1224 } else {
1225 bp->bio_data = abd_borrow_buf_copy(zio->io_abd,
1226 zio->io_size);
1227 }
1228 }
1229 break;
1230 case ZIO_TYPE_TRIM:
1231 bp->bio_cmd = BIO_DELETE;
1232 bp->bio_data = NULL;
1233 bp->bio_offset = zio->io_offset;
1234 bp->bio_length = zio->io_size;
1235 break;
1236 case ZIO_TYPE_FLUSH:
1237 bp->bio_cmd = BIO_FLUSH;
1238 bp->bio_data = NULL;
1239 bp->bio_offset = cp->provider->mediasize;
1240 bp->bio_length = 0;
1241 break;
1242 default:
1243 panic("invalid zio->io_type: %d\n", zio->io_type);
1244 }
1245 bp->bio_done = vdev_geom_io_intr;
1246 zio->io_bio = bp;
1247
1248 g_io_request(bp, cp);
1249 }
1250
1251 static void
vdev_geom_io_done(zio_t * zio)1252 vdev_geom_io_done(zio_t *zio)
1253 {
1254 struct bio *bp = zio->io_bio;
1255
1256 if (zio->io_type != ZIO_TYPE_READ && zio->io_type != ZIO_TYPE_WRITE) {
1257 ASSERT3P(bp, ==, NULL);
1258 return;
1259 }
1260
1261 if (bp == NULL) {
1262 ASSERT3S(zio->io_error, ==, ENXIO);
1263 return;
1264 }
1265
1266 if (bp->bio_ma != NULL) {
1267 free(bp->bio_ma, M_DEVBUF);
1268 } else {
1269 if (zio->io_type == ZIO_TYPE_READ) {
1270 abd_return_buf_copy(zio->io_abd, bp->bio_data,
1271 zio->io_size);
1272 } else {
1273 abd_return_buf(zio->io_abd, bp->bio_data,
1274 zio->io_size);
1275 }
1276 }
1277
1278 g_destroy_bio(bp);
1279 zio->io_bio = NULL;
1280 }
1281
1282 static void
vdev_geom_hold(vdev_t * vd)1283 vdev_geom_hold(vdev_t *vd)
1284 {
1285 }
1286
1287 static void
vdev_geom_rele(vdev_t * vd)1288 vdev_geom_rele(vdev_t *vd)
1289 {
1290 }
1291
1292 vdev_ops_t vdev_disk_ops = {
1293 .vdev_op_init = NULL,
1294 .vdev_op_fini = NULL,
1295 .vdev_op_open = vdev_geom_open,
1296 .vdev_op_close = vdev_geom_close,
1297 .vdev_op_asize = vdev_default_asize,
1298 .vdev_op_min_asize = vdev_default_min_asize,
1299 .vdev_op_min_alloc = NULL,
1300 .vdev_op_io_start = vdev_geom_io_start,
1301 .vdev_op_io_done = vdev_geom_io_done,
1302 .vdev_op_state_change = NULL,
1303 .vdev_op_need_resilver = NULL,
1304 .vdev_op_hold = vdev_geom_hold,
1305 .vdev_op_rele = vdev_geom_rele,
1306 .vdev_op_remap = NULL,
1307 .vdev_op_xlate = vdev_default_xlate,
1308 .vdev_op_rebuild_asize = NULL,
1309 .vdev_op_metaslab_init = NULL,
1310 .vdev_op_config_generate = NULL,
1311 .vdev_op_nparity = NULL,
1312 .vdev_op_ndisks = NULL,
1313 .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
1314 .vdev_op_leaf = B_TRUE /* leaf vdev */
1315 };
1316