1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy * CDDL HEADER START
3eda14cbcSMatt Macy *
4eda14cbcSMatt Macy * The contents of this file are subject to the terms of the
5eda14cbcSMatt Macy * Common Development and Distribution License (the "License").
6eda14cbcSMatt Macy * You may not use this file except in compliance with the License.
7eda14cbcSMatt Macy *
8eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9271171e0SMartin Matuska * or https://opensource.org/licenses/CDDL-1.0.
10eda14cbcSMatt Macy * See the License for the specific language governing permissions
11eda14cbcSMatt Macy * and limitations under the License.
12eda14cbcSMatt Macy *
13eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each
14eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the
16eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying
17eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner]
18eda14cbcSMatt Macy *
19eda14cbcSMatt Macy * CDDL HEADER END
20eda14cbcSMatt Macy */
21eda14cbcSMatt Macy /*
22eda14cbcSMatt Macy * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
23eda14cbcSMatt Macy * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
24eda14cbcSMatt Macy * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
25eda14cbcSMatt Macy * LLNL-CODE-403049.
26eda14cbcSMatt Macy * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
27783d3ff6SMartin Matuska * Copyright (c) 2023, 2024, Klara Inc.
28eda14cbcSMatt Macy */
29eda14cbcSMatt Macy
30eda14cbcSMatt Macy #include <sys/zfs_context.h>
31eda14cbcSMatt Macy #include <sys/spa_impl.h>
32eda14cbcSMatt Macy #include <sys/vdev_disk.h>
33eda14cbcSMatt Macy #include <sys/vdev_impl.h>
34eda14cbcSMatt Macy #include <sys/vdev_trim.h>
35eda14cbcSMatt Macy #include <sys/abd.h>
36eda14cbcSMatt Macy #include <sys/fs/zfs.h>
37eda14cbcSMatt Macy #include <sys/zio.h>
382c48331dSMatt Macy #include <linux/blkpg.h>
39eda14cbcSMatt Macy #include <linux/msdos_fs.h>
40eda14cbcSMatt Macy #include <linux/vfs_compat.h>
41681ce946SMartin Matuska #ifdef HAVE_LINUX_BLK_CGROUP_HEADER
42681ce946SMartin Matuska #include <linux/blk-cgroup.h>
43681ce946SMartin Matuska #endif
44eda14cbcSMatt Macy
45fd45b686SMartin Matuska /*
46fd45b686SMartin Matuska * Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying
47fd45b686SMartin Matuska * block_device. Since it carries the block_device inside, its convenient to
481719886fSMartin Matuska * just use the handle as a proxy.
491719886fSMartin Matuska *
501719886fSMartin Matuska * Linux 6.9.x uses a file for the same purpose.
511719886fSMartin Matuska *
521719886fSMartin Matuska * For pre-6.8, we just emulate this with a cast, since we don't need any of
531719886fSMartin Matuska * the other fields inside the handle.
54fd45b686SMartin Matuska */
551719886fSMartin Matuska #if defined(HAVE_BDEV_OPEN_BY_PATH)
56fd45b686SMartin Matuska typedef struct bdev_handle zfs_bdev_handle_t;
57fd45b686SMartin Matuska #define BDH_BDEV(bdh) ((bdh)->bdev)
58fd45b686SMartin Matuska #define BDH_IS_ERR(bdh) (IS_ERR(bdh))
59fd45b686SMartin Matuska #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
60fd45b686SMartin Matuska #define BDH_ERR_PTR(err) (ERR_PTR(err))
611719886fSMartin Matuska #elif defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
621719886fSMartin Matuska typedef struct file zfs_bdev_handle_t;
631719886fSMartin Matuska #define BDH_BDEV(bdh) (file_bdev(bdh))
641719886fSMartin Matuska #define BDH_IS_ERR(bdh) (IS_ERR(bdh))
651719886fSMartin Matuska #define BDH_PTR_ERR(bdh) (PTR_ERR(bdh))
661719886fSMartin Matuska #define BDH_ERR_PTR(err) (ERR_PTR(err))
67fd45b686SMartin Matuska #else
68fd45b686SMartin Matuska typedef void zfs_bdev_handle_t;
69fd45b686SMartin Matuska #define BDH_BDEV(bdh) ((struct block_device *)bdh)
70fd45b686SMartin Matuska #define BDH_IS_ERR(bdh) (IS_ERR(BDH_BDEV(bdh)))
71fd45b686SMartin Matuska #define BDH_PTR_ERR(bdh) (PTR_ERR(BDH_BDEV(bdh)))
72fd45b686SMartin Matuska #define BDH_ERR_PTR(err) (ERR_PTR(err))
73fd45b686SMartin Matuska #endif
74fd45b686SMartin Matuska
75eda14cbcSMatt Macy typedef struct vdev_disk {
76fd45b686SMartin Matuska zfs_bdev_handle_t *vd_bdh;
77eda14cbcSMatt Macy krwlock_t vd_lock;
78eda14cbcSMatt Macy } vdev_disk_t;
79eda14cbcSMatt Macy
80eda14cbcSMatt Macy /*
81783d3ff6SMartin Matuska * Maximum number of segments to add to a bio (min 4). If this is higher than
82783d3ff6SMartin Matuska * the maximum allowed by the device queue or the kernel itself, it will be
83783d3ff6SMartin Matuska * clamped. Setting it to zero will cause the kernel's ideal size to be used.
84783d3ff6SMartin Matuska */
85783d3ff6SMartin Matuska uint_t zfs_vdev_disk_max_segs = 0;
86783d3ff6SMartin Matuska
87783d3ff6SMartin Matuska /*
88eda14cbcSMatt Macy * Unique identifier for the exclusive vdev holder.
89eda14cbcSMatt Macy */
90eda14cbcSMatt Macy static void *zfs_vdev_holder = VDEV_HOLDER;
91eda14cbcSMatt Macy
92eda14cbcSMatt Macy /*
93eda14cbcSMatt Macy * Wait up to zfs_vdev_open_timeout_ms milliseconds before determining the
94eda14cbcSMatt Macy * device is missing. The missing path may be transient since the links
95eda14cbcSMatt Macy * can be briefly removed and recreated in response to udev events.
96eda14cbcSMatt Macy */
97dbd5678dSMartin Matuska static uint_t zfs_vdev_open_timeout_ms = 1000;
98eda14cbcSMatt Macy
99eda14cbcSMatt Macy /*
100eda14cbcSMatt Macy * Size of the "reserved" partition, in blocks.
101eda14cbcSMatt Macy */
102eda14cbcSMatt Macy #define EFI_MIN_RESV_SIZE (16 * 1024)
103eda14cbcSMatt Macy
104eda14cbcSMatt Macy /*
105dbd5678dSMartin Matuska * BIO request failfast mask.
106dbd5678dSMartin Matuska */
107dbd5678dSMartin Matuska
108dbd5678dSMartin Matuska static unsigned int zfs_vdev_failfast_mask = 1;
109dbd5678dSMartin Matuska
110783d3ff6SMartin Matuska /*
111783d3ff6SMartin Matuska * Convert SPA mode flags into bdev open mode flags.
112783d3ff6SMartin Matuska */
113315ee00fSMartin Matuska #ifdef HAVE_BLK_MODE_T
114783d3ff6SMartin Matuska typedef blk_mode_t vdev_bdev_mode_t;
115783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_READ BLK_OPEN_READ
116783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_WRITE BLK_OPEN_WRITE
117783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_EXCL BLK_OPEN_EXCL
118783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_MASK (BLK_OPEN_READ|BLK_OPEN_WRITE|BLK_OPEN_EXCL)
119315ee00fSMartin Matuska #else
120783d3ff6SMartin Matuska typedef fmode_t vdev_bdev_mode_t;
121783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_READ FMODE_READ
122783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_WRITE FMODE_WRITE
123783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_EXCL FMODE_EXCL
124783d3ff6SMartin Matuska #define VDEV_BDEV_MODE_MASK (FMODE_READ|FMODE_WRITE|FMODE_EXCL)
125315ee00fSMartin Matuska #endif
126783d3ff6SMartin Matuska
127783d3ff6SMartin Matuska static vdev_bdev_mode_t
vdev_bdev_mode(spa_mode_t smode)128783d3ff6SMartin Matuska vdev_bdev_mode(spa_mode_t smode)
129eda14cbcSMatt Macy {
130783d3ff6SMartin Matuska ASSERT3U(smode, !=, SPA_MODE_UNINIT);
131783d3ff6SMartin Matuska ASSERT0(smode & ~(SPA_MODE_READ|SPA_MODE_WRITE));
132315ee00fSMartin Matuska
133783d3ff6SMartin Matuska vdev_bdev_mode_t bmode = VDEV_BDEV_MODE_EXCL;
134315ee00fSMartin Matuska
135783d3ff6SMartin Matuska if (smode & SPA_MODE_READ)
136783d3ff6SMartin Matuska bmode |= VDEV_BDEV_MODE_READ;
137b356da80SMartin Matuska
138783d3ff6SMartin Matuska if (smode & SPA_MODE_WRITE)
139783d3ff6SMartin Matuska bmode |= VDEV_BDEV_MODE_WRITE;
140eda14cbcSMatt Macy
141783d3ff6SMartin Matuska ASSERT(bmode & VDEV_BDEV_MODE_MASK);
142783d3ff6SMartin Matuska ASSERT0(bmode & ~VDEV_BDEV_MODE_MASK);
143eda14cbcSMatt Macy
144783d3ff6SMartin Matuska return (bmode);
145eda14cbcSMatt Macy }
146eda14cbcSMatt Macy
147eda14cbcSMatt Macy /*
148eda14cbcSMatt Macy * Returns the usable capacity (in bytes) for the partition or disk.
149eda14cbcSMatt Macy */
150eda14cbcSMatt Macy static uint64_t
bdev_capacity(struct block_device * bdev)151eda14cbcSMatt Macy bdev_capacity(struct block_device *bdev)
152eda14cbcSMatt Macy {
153eda14cbcSMatt Macy return (i_size_read(bdev->bd_inode));
154eda14cbcSMatt Macy }
155eda14cbcSMatt Macy
1567877fdebSMatt Macy #if !defined(HAVE_BDEV_WHOLE)
1577877fdebSMatt Macy static inline struct block_device *
bdev_whole(struct block_device * bdev)1587877fdebSMatt Macy bdev_whole(struct block_device *bdev)
1597877fdebSMatt Macy {
1607877fdebSMatt Macy return (bdev->bd_contains);
1617877fdebSMatt Macy }
1627877fdebSMatt Macy #endif
1637877fdebSMatt Macy
164271171e0SMartin Matuska #if defined(HAVE_BDEVNAME)
165271171e0SMartin Matuska #define vdev_bdevname(bdev, name) bdevname(bdev, name)
166271171e0SMartin Matuska #else
167271171e0SMartin Matuska static inline void
vdev_bdevname(struct block_device * bdev,char * name)168271171e0SMartin Matuska vdev_bdevname(struct block_device *bdev, char *name)
169271171e0SMartin Matuska {
170271171e0SMartin Matuska snprintf(name, BDEVNAME_SIZE, "%pg", bdev);
171271171e0SMartin Matuska }
172271171e0SMartin Matuska #endif
173271171e0SMartin Matuska
174eda14cbcSMatt Macy /*
175eda14cbcSMatt Macy * Returns the maximum expansion capacity of the block device (in bytes).
176eda14cbcSMatt Macy *
177eda14cbcSMatt Macy * It is possible to expand a vdev when it has been created as a wholedisk
178eda14cbcSMatt Macy * and the containing block device has increased in capacity. Or when the
179eda14cbcSMatt Macy * partition containing the pool has been manually increased in size.
180eda14cbcSMatt Macy *
181eda14cbcSMatt Macy * This function is only responsible for calculating the potential expansion
182eda14cbcSMatt Macy * size so it can be reported by 'zpool list'. The efi_use_whole_disk() is
183eda14cbcSMatt Macy * responsible for verifying the expected partition layout in the wholedisk
184eda14cbcSMatt Macy * case, and updating the partition table if appropriate. Once the partition
185eda14cbcSMatt Macy * size has been increased the additional capacity will be visible using
186eda14cbcSMatt Macy * bdev_capacity().
187eda14cbcSMatt Macy *
188eda14cbcSMatt Macy * The returned maximum expansion capacity is always expected to be larger, or
189eda14cbcSMatt Macy * at the very least equal, to its usable capacity to prevent overestimating
190eda14cbcSMatt Macy * the pool expandsize.
191eda14cbcSMatt Macy */
192eda14cbcSMatt Macy static uint64_t
bdev_max_capacity(struct block_device * bdev,uint64_t wholedisk)193eda14cbcSMatt Macy bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk)
194eda14cbcSMatt Macy {
195eda14cbcSMatt Macy uint64_t psize;
196eda14cbcSMatt Macy int64_t available;
197eda14cbcSMatt Macy
1987877fdebSMatt Macy if (wholedisk && bdev != bdev_whole(bdev)) {
199eda14cbcSMatt Macy /*
200eda14cbcSMatt Macy * When reporting maximum expansion capacity for a wholedisk
201eda14cbcSMatt Macy * deduct any capacity which is expected to be lost due to
202eda14cbcSMatt Macy * alignment restrictions. Over reporting this value isn't
203eda14cbcSMatt Macy * harmful and would only result in slightly less capacity
204eda14cbcSMatt Macy * than expected post expansion.
205eda14cbcSMatt Macy * The estimated available space may be slightly smaller than
206eda14cbcSMatt Macy * bdev_capacity() for devices where the number of sectors is
207eda14cbcSMatt Macy * not a multiple of the alignment size and the partition layout
208eda14cbcSMatt Macy * is keeping less than PARTITION_END_ALIGNMENT bytes after the
209eda14cbcSMatt Macy * "reserved" EFI partition: in such cases return the device
210eda14cbcSMatt Macy * usable capacity.
211eda14cbcSMatt Macy */
2127877fdebSMatt Macy available = i_size_read(bdev_whole(bdev)->bd_inode) -
213eda14cbcSMatt Macy ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK +
214eda14cbcSMatt Macy PARTITION_END_ALIGNMENT) << SECTOR_BITS);
215eda14cbcSMatt Macy psize = MAX(available, bdev_capacity(bdev));
216eda14cbcSMatt Macy } else {
217eda14cbcSMatt Macy psize = bdev_capacity(bdev);
218eda14cbcSMatt Macy }
219eda14cbcSMatt Macy
220eda14cbcSMatt Macy return (psize);
221eda14cbcSMatt Macy }
222eda14cbcSMatt Macy
223eda14cbcSMatt Macy static void
vdev_disk_error(zio_t * zio)224eda14cbcSMatt Macy vdev_disk_error(zio_t *zio)
225eda14cbcSMatt Macy {
226eda14cbcSMatt Macy /*
227eda14cbcSMatt Macy * This function can be called in interrupt context, for instance while
228eda14cbcSMatt Macy * handling IRQs coming from a misbehaving disk device; use printk()
229eda14cbcSMatt Macy * which is safe from any context.
230eda14cbcSMatt Macy */
231eda14cbcSMatt Macy printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d "
232dbd5678dSMartin Matuska "offset=%llu size=%llu flags=%llu\n", spa_name(zio->io_spa),
233eda14cbcSMatt Macy zio->io_vd->vdev_path, zio->io_error, zio->io_type,
234eda14cbcSMatt Macy (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size,
235eda14cbcSMatt Macy zio->io_flags);
236eda14cbcSMatt Macy }
237eda14cbcSMatt Macy
238be181ee2SMartin Matuska static void
vdev_disk_kobj_evt_post(vdev_t * v)239be181ee2SMartin Matuska vdev_disk_kobj_evt_post(vdev_t *v)
240be181ee2SMartin Matuska {
241be181ee2SMartin Matuska vdev_disk_t *vd = v->vdev_tsd;
242fd45b686SMartin Matuska if (vd && vd->vd_bdh) {
243fd45b686SMartin Matuska spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh));
244be181ee2SMartin Matuska } else {
245be181ee2SMartin Matuska vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
246be181ee2SMartin Matuska v->vdev_path);
247be181ee2SMartin Matuska }
248be181ee2SMartin Matuska }
249be181ee2SMartin Matuska
250fd45b686SMartin Matuska static zfs_bdev_handle_t *
vdev_blkdev_get_by_path(const char * path,spa_mode_t smode,void * holder)251783d3ff6SMartin Matuska vdev_blkdev_get_by_path(const char *path, spa_mode_t smode, void *holder)
252315ee00fSMartin Matuska {
253783d3ff6SMartin Matuska vdev_bdev_mode_t bmode = vdev_bdev_mode(smode);
254783d3ff6SMartin Matuska
2551719886fSMartin Matuska #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
2561719886fSMartin Matuska return (bdev_file_open_by_path(path, bmode, holder, NULL));
2571719886fSMartin Matuska #elif defined(HAVE_BDEV_OPEN_BY_PATH)
258783d3ff6SMartin Matuska return (bdev_open_by_path(path, bmode, holder, NULL));
259fd45b686SMartin Matuska #elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
260783d3ff6SMartin Matuska return (blkdev_get_by_path(path, bmode, holder, NULL));
261315ee00fSMartin Matuska #else
262783d3ff6SMartin Matuska return (blkdev_get_by_path(path, bmode, holder));
263315ee00fSMartin Matuska #endif
264315ee00fSMartin Matuska }
265315ee00fSMartin Matuska
266315ee00fSMartin Matuska static void
vdev_blkdev_put(zfs_bdev_handle_t * bdh,spa_mode_t smode,void * holder)267783d3ff6SMartin Matuska vdev_blkdev_put(zfs_bdev_handle_t *bdh, spa_mode_t smode, void *holder)
268315ee00fSMartin Matuska {
269fd45b686SMartin Matuska #if defined(HAVE_BDEV_RELEASE)
270fd45b686SMartin Matuska return (bdev_release(bdh));
271fd45b686SMartin Matuska #elif defined(HAVE_BLKDEV_PUT_HOLDER)
272fd45b686SMartin Matuska return (blkdev_put(BDH_BDEV(bdh), holder));
2731719886fSMartin Matuska #elif defined(HAVE_BLKDEV_PUT)
274783d3ff6SMartin Matuska return (blkdev_put(BDH_BDEV(bdh), vdev_bdev_mode(smode)));
2751719886fSMartin Matuska #else
2761719886fSMartin Matuska fput(bdh);
277315ee00fSMartin Matuska #endif
278315ee00fSMartin Matuska }
279315ee00fSMartin Matuska
280eda14cbcSMatt Macy static int
vdev_disk_open(vdev_t * v,uint64_t * psize,uint64_t * max_psize,uint64_t * logical_ashift,uint64_t * physical_ashift)281eda14cbcSMatt Macy vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
282eda14cbcSMatt Macy uint64_t *logical_ashift, uint64_t *physical_ashift)
283eda14cbcSMatt Macy {
284fd45b686SMartin Matuska zfs_bdev_handle_t *bdh;
285783d3ff6SMartin Matuska spa_mode_t smode = spa_mode(v->vdev_spa);
286eda14cbcSMatt Macy hrtime_t timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms);
287eda14cbcSMatt Macy vdev_disk_t *vd;
288eda14cbcSMatt Macy
289eda14cbcSMatt Macy /* Must have a pathname and it must be absolute. */
290eda14cbcSMatt Macy if (v->vdev_path == NULL || v->vdev_path[0] != '/') {
291eda14cbcSMatt Macy v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
292eda14cbcSMatt Macy vdev_dbgmsg(v, "invalid vdev_path");
293eda14cbcSMatt Macy return (SET_ERROR(EINVAL));
294eda14cbcSMatt Macy }
295eda14cbcSMatt Macy
296eda14cbcSMatt Macy /*
297eda14cbcSMatt Macy * Reopen the device if it is currently open. When expanding a
2982c48331dSMatt Macy * partition force re-scanning the partition table if userland
2992c48331dSMatt Macy * did not take care of this already. We need to do this while closed
300eda14cbcSMatt Macy * in order to get an accurate updated block device size. Then
301eda14cbcSMatt Macy * since udev may need to recreate the device links increase the
302eda14cbcSMatt Macy * open retry timeout before reporting the device as unavailable.
303eda14cbcSMatt Macy */
304eda14cbcSMatt Macy vd = v->vdev_tsd;
305eda14cbcSMatt Macy if (vd) {
306eda14cbcSMatt Macy char disk_name[BDEVNAME_SIZE + 6] = "/dev/";
307eda14cbcSMatt Macy boolean_t reread_part = B_FALSE;
308eda14cbcSMatt Macy
309eda14cbcSMatt Macy rw_enter(&vd->vd_lock, RW_WRITER);
310fd45b686SMartin Matuska bdh = vd->vd_bdh;
311fd45b686SMartin Matuska vd->vd_bdh = NULL;
312eda14cbcSMatt Macy
313fd45b686SMartin Matuska if (bdh) {
314fd45b686SMartin Matuska struct block_device *bdev = BDH_BDEV(bdh);
3157877fdebSMatt Macy if (v->vdev_expanding && bdev != bdev_whole(bdev)) {
316271171e0SMartin Matuska vdev_bdevname(bdev_whole(bdev), disk_name + 5);
3172c48331dSMatt Macy /*
3182c48331dSMatt Macy * If userland has BLKPG_RESIZE_PARTITION,
3192c48331dSMatt Macy * then it should have updated the partition
3202c48331dSMatt Macy * table already. We can detect this by
3212c48331dSMatt Macy * comparing our current physical size
3222c48331dSMatt Macy * with that of the device. If they are
3232c48331dSMatt Macy * the same, then we must not have
3242c48331dSMatt Macy * BLKPG_RESIZE_PARTITION or it failed to
3252c48331dSMatt Macy * update the partition table online. We
3262c48331dSMatt Macy * fallback to rescanning the partition
3272c48331dSMatt Macy * table from the kernel below. However,
3282c48331dSMatt Macy * if the capacity already reflects the
3292c48331dSMatt Macy * updated partition, then we skip
3302c48331dSMatt Macy * rescanning the partition table here.
3312c48331dSMatt Macy */
3322c48331dSMatt Macy if (v->vdev_psize == bdev_capacity(bdev))
333eda14cbcSMatt Macy reread_part = B_TRUE;
334eda14cbcSMatt Macy }
335eda14cbcSMatt Macy
336783d3ff6SMartin Matuska vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
337eda14cbcSMatt Macy }
338eda14cbcSMatt Macy
339eda14cbcSMatt Macy if (reread_part) {
340783d3ff6SMartin Matuska bdh = vdev_blkdev_get_by_path(disk_name, smode,
341fd45b686SMartin Matuska zfs_vdev_holder);
342fd45b686SMartin Matuska if (!BDH_IS_ERR(bdh)) {
343fd45b686SMartin Matuska int error =
344fd45b686SMartin Matuska vdev_bdev_reread_part(BDH_BDEV(bdh));
345783d3ff6SMartin Matuska vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
346eda14cbcSMatt Macy if (error == 0) {
347eda14cbcSMatt Macy timeout = MSEC2NSEC(
348eda14cbcSMatt Macy zfs_vdev_open_timeout_ms * 2);
349eda14cbcSMatt Macy }
350eda14cbcSMatt Macy }
351eda14cbcSMatt Macy }
352eda14cbcSMatt Macy } else {
353eda14cbcSMatt Macy vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP);
354eda14cbcSMatt Macy
355eda14cbcSMatt Macy rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL);
356eda14cbcSMatt Macy rw_enter(&vd->vd_lock, RW_WRITER);
357eda14cbcSMatt Macy }
358eda14cbcSMatt Macy
359eda14cbcSMatt Macy /*
360eda14cbcSMatt Macy * Devices are always opened by the path provided at configuration
361eda14cbcSMatt Macy * time. This means that if the provided path is a udev by-id path
362eda14cbcSMatt Macy * then drives may be re-cabled without an issue. If the provided
363eda14cbcSMatt Macy * path is a udev by-path path, then the physical location information
364eda14cbcSMatt Macy * will be preserved. This can be critical for more complicated
365eda14cbcSMatt Macy * configurations where drives are located in specific physical
366eda14cbcSMatt Macy * locations to maximize the systems tolerance to component failure.
367eda14cbcSMatt Macy *
368eda14cbcSMatt Macy * Alternatively, you can provide your own udev rule to flexibly map
369eda14cbcSMatt Macy * the drives as you see fit. It is not advised that you use the
370eda14cbcSMatt Macy * /dev/[hd]d devices which may be reordered due to probing order.
371eda14cbcSMatt Macy * Devices in the wrong locations will be detected by the higher
372eda14cbcSMatt Macy * level vdev validation.
373eda14cbcSMatt Macy *
374eda14cbcSMatt Macy * The specified paths may be briefly removed and recreated in
375eda14cbcSMatt Macy * response to udev events. This should be exceptionally unlikely
376eda14cbcSMatt Macy * because the zpool command makes every effort to verify these paths
377eda14cbcSMatt Macy * have already settled prior to reaching this point. Therefore,
378eda14cbcSMatt Macy * a ENOENT failure at this point is highly likely to be transient
379eda14cbcSMatt Macy * and it is reasonable to sleep and retry before giving up. In
380eda14cbcSMatt Macy * practice delays have been observed to be on the order of 100ms.
381681ce946SMartin Matuska *
382681ce946SMartin Matuska * When ERESTARTSYS is returned it indicates the block device is
383681ce946SMartin Matuska * a zvol which could not be opened due to the deadlock detection
384681ce946SMartin Matuska * logic in zvol_open(). Extend the timeout and retry the open
385681ce946SMartin Matuska * subsequent attempts are expected to eventually succeed.
386eda14cbcSMatt Macy */
387eda14cbcSMatt Macy hrtime_t start = gethrtime();
388fd45b686SMartin Matuska bdh = BDH_ERR_PTR(-ENXIO);
389fd45b686SMartin Matuska while (BDH_IS_ERR(bdh) && ((gethrtime() - start) < timeout)) {
390783d3ff6SMartin Matuska bdh = vdev_blkdev_get_by_path(v->vdev_path, smode,
391fd45b686SMartin Matuska zfs_vdev_holder);
392fd45b686SMartin Matuska if (unlikely(BDH_PTR_ERR(bdh) == -ENOENT)) {
393be181ee2SMartin Matuska /*
394be181ee2SMartin Matuska * There is no point of waiting since device is removed
395be181ee2SMartin Matuska * explicitly
396be181ee2SMartin Matuska */
397be181ee2SMartin Matuska if (v->vdev_removed)
398be181ee2SMartin Matuska break;
399be181ee2SMartin Matuska
400*aca928a5SMartin Matuska schedule_timeout_interruptible(MSEC_TO_TICK(10));
401fd45b686SMartin Matuska } else if (unlikely(BDH_PTR_ERR(bdh) == -ERESTARTSYS)) {
402681ce946SMartin Matuska timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
403681ce946SMartin Matuska continue;
404fd45b686SMartin Matuska } else if (BDH_IS_ERR(bdh)) {
405eda14cbcSMatt Macy break;
406eda14cbcSMatt Macy }
407eda14cbcSMatt Macy }
408eda14cbcSMatt Macy
409fd45b686SMartin Matuska if (BDH_IS_ERR(bdh)) {
410fd45b686SMartin Matuska int error = -BDH_PTR_ERR(bdh);
411eda14cbcSMatt Macy vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", error,
412eda14cbcSMatt Macy (u_longlong_t)(gethrtime() - start),
413eda14cbcSMatt Macy (u_longlong_t)timeout);
414fd45b686SMartin Matuska vd->vd_bdh = NULL;
415eda14cbcSMatt Macy v->vdev_tsd = vd;
416eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
417eda14cbcSMatt Macy return (SET_ERROR(error));
418eda14cbcSMatt Macy } else {
419fd45b686SMartin Matuska vd->vd_bdh = bdh;
420eda14cbcSMatt Macy v->vdev_tsd = vd;
421eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
422eda14cbcSMatt Macy }
423eda14cbcSMatt Macy
424fd45b686SMartin Matuska struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
425fd45b686SMartin Matuska
426eda14cbcSMatt Macy /* Determine the physical block size */
427fd45b686SMartin Matuska int physical_block_size = bdev_physical_block_size(bdev);
428eda14cbcSMatt Macy
429eda14cbcSMatt Macy /* Determine the logical block size */
430fd45b686SMartin Matuska int logical_block_size = bdev_logical_block_size(bdev);
431eda14cbcSMatt Macy
432b985c9caSMartin Matuska /*
433b985c9caSMartin Matuska * If the device has a write cache, clear the nowritecache flag,
434b985c9caSMartin Matuska * so that we start issuing flush requests again.
435b985c9caSMartin Matuska */
436b985c9caSMartin Matuska v->vdev_nowritecache = !zfs_bdev_has_write_cache(bdev);
437eda14cbcSMatt Macy
438eda14cbcSMatt Macy /* Set when device reports it supports TRIM. */
439fd45b686SMartin Matuska v->vdev_has_trim = bdev_discard_supported(bdev);
440eda14cbcSMatt Macy
441eda14cbcSMatt Macy /* Set when device reports it supports secure TRIM. */
442fd45b686SMartin Matuska v->vdev_has_securetrim = bdev_secure_discard_supported(bdev);
443eda14cbcSMatt Macy
444eda14cbcSMatt Macy /* Inform the ZIO pipeline that we are non-rotational */
445fd45b686SMartin Matuska v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev));
446eda14cbcSMatt Macy
447eda14cbcSMatt Macy /* Physical volume size in bytes for the partition */
448fd45b686SMartin Matuska *psize = bdev_capacity(bdev);
449eda14cbcSMatt Macy
450eda14cbcSMatt Macy /* Physical volume size in bytes including possible expansion space */
451fd45b686SMartin Matuska *max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk);
452eda14cbcSMatt Macy
453eda14cbcSMatt Macy /* Based on the minimum sector size set the block size */
454eda14cbcSMatt Macy *physical_ashift = highbit64(MAX(physical_block_size,
455eda14cbcSMatt Macy SPA_MINBLOCKSIZE)) - 1;
456eda14cbcSMatt Macy
457eda14cbcSMatt Macy *logical_ashift = highbit64(MAX(logical_block_size,
458eda14cbcSMatt Macy SPA_MINBLOCKSIZE)) - 1;
459eda14cbcSMatt Macy
460eda14cbcSMatt Macy return (0);
461eda14cbcSMatt Macy }
462eda14cbcSMatt Macy
463eda14cbcSMatt Macy static void
vdev_disk_close(vdev_t * v)464eda14cbcSMatt Macy vdev_disk_close(vdev_t *v)
465eda14cbcSMatt Macy {
466eda14cbcSMatt Macy vdev_disk_t *vd = v->vdev_tsd;
467eda14cbcSMatt Macy
468eda14cbcSMatt Macy if (v->vdev_reopening || vd == NULL)
469eda14cbcSMatt Macy return;
470eda14cbcSMatt Macy
471783d3ff6SMartin Matuska if (vd->vd_bdh != NULL)
472fd45b686SMartin Matuska vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
473315ee00fSMartin Matuska zfs_vdev_holder);
474eda14cbcSMatt Macy
475eda14cbcSMatt Macy rw_destroy(&vd->vd_lock);
476eda14cbcSMatt Macy kmem_free(vd, sizeof (vdev_disk_t));
477eda14cbcSMatt Macy v->vdev_tsd = NULL;
478eda14cbcSMatt Macy }
479eda14cbcSMatt Macy
480eda14cbcSMatt Macy static inline void
vdev_submit_bio_impl(struct bio * bio)481eda14cbcSMatt Macy vdev_submit_bio_impl(struct bio *bio)
482eda14cbcSMatt Macy {
483eda14cbcSMatt Macy #ifdef HAVE_1ARG_SUBMIT_BIO
48481b22a98SMartin Matuska (void) submit_bio(bio);
485eda14cbcSMatt Macy #else
486c03c5b1cSMartin Matuska (void) submit_bio(bio_data_dir(bio), bio);
487eda14cbcSMatt Macy #endif
488eda14cbcSMatt Macy }
489eda14cbcSMatt Macy
490c40487d4SMatt Macy /*
491c40487d4SMatt Macy * preempt_schedule_notrace is GPL-only which breaks the ZFS build, so
492c40487d4SMatt Macy * replace it with preempt_schedule under the following condition:
493c40487d4SMatt Macy */
494c40487d4SMatt Macy #if defined(CONFIG_ARM64) && \
495c40487d4SMatt Macy defined(CONFIG_PREEMPTION) && \
496c40487d4SMatt Macy defined(CONFIG_BLK_CGROUP)
497c40487d4SMatt Macy #define preempt_schedule_notrace(x) preempt_schedule(x)
498c40487d4SMatt Macy #endif
499c40487d4SMatt Macy
500e3aa18adSMartin Matuska /*
501e3aa18adSMartin Matuska * As for the Linux 5.18 kernel bio_alloc() expects a block_device struct
502e3aa18adSMartin Matuska * as an argument removing the need to set it with bio_set_dev(). This
503e3aa18adSMartin Matuska * removes the need for all of the following compatibility code.
504e3aa18adSMartin Matuska */
505e3aa18adSMartin Matuska #if !defined(HAVE_BIO_ALLOC_4ARG)
506e3aa18adSMartin Matuska
507eda14cbcSMatt Macy #ifdef HAVE_BIO_SET_DEV
508eda14cbcSMatt Macy #if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY)
509eda14cbcSMatt Macy /*
510eda14cbcSMatt Macy * The Linux 5.5 kernel updated percpu_ref_tryget() which is inlined by
511eda14cbcSMatt Macy * blkg_tryget() to use rcu_read_lock() instead of rcu_read_lock_sched().
512eda14cbcSMatt Macy * As a side effect the function was converted to GPL-only. Define our
513eda14cbcSMatt Macy * own version when needed which uses rcu_read_lock_sched().
514716fd348SMartin Matuska *
515716fd348SMartin Matuska * The Linux 5.17 kernel split linux/blk-cgroup.h into a private and a public
516716fd348SMartin Matuska * part, moving blkg_tryget into the private one. Define our own version.
517eda14cbcSMatt Macy */
518716fd348SMartin Matuska #if defined(HAVE_BLKG_TRYGET_GPL_ONLY) || !defined(HAVE_BLKG_TRYGET)
519eda14cbcSMatt Macy static inline bool
vdev_blkg_tryget(struct blkcg_gq * blkg)520eda14cbcSMatt Macy vdev_blkg_tryget(struct blkcg_gq *blkg)
521eda14cbcSMatt Macy {
522eda14cbcSMatt Macy struct percpu_ref *ref = &blkg->refcnt;
523eda14cbcSMatt Macy unsigned long __percpu *count;
524eda14cbcSMatt Macy bool rc;
525eda14cbcSMatt Macy
526eda14cbcSMatt Macy rcu_read_lock_sched();
527eda14cbcSMatt Macy
528eda14cbcSMatt Macy if (__ref_is_percpu(ref, &count)) {
529eda14cbcSMatt Macy this_cpu_inc(*count);
530eda14cbcSMatt Macy rc = true;
531eda14cbcSMatt Macy } else {
5327877fdebSMatt Macy #ifdef ZFS_PERCPU_REF_COUNT_IN_DATA
5337877fdebSMatt Macy rc = atomic_long_inc_not_zero(&ref->data->count);
5347877fdebSMatt Macy #else
535eda14cbcSMatt Macy rc = atomic_long_inc_not_zero(&ref->count);
5367877fdebSMatt Macy #endif
537eda14cbcSMatt Macy }
538eda14cbcSMatt Macy
539eda14cbcSMatt Macy rcu_read_unlock_sched();
540eda14cbcSMatt Macy
541eda14cbcSMatt Macy return (rc);
542eda14cbcSMatt Macy }
543716fd348SMartin Matuska #else
544eda14cbcSMatt Macy #define vdev_blkg_tryget(bg) blkg_tryget(bg)
545eda14cbcSMatt Macy #endif
546681ce946SMartin Matuska #ifdef HAVE_BIO_SET_DEV_MACRO
547eda14cbcSMatt Macy /*
548eda14cbcSMatt Macy * The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the
549eda14cbcSMatt Macy * GPL-only bio_associate_blkg() symbol thus inadvertently converting
550eda14cbcSMatt Macy * the entire macro. Provide a minimal version which always assigns the
551eda14cbcSMatt Macy * request queue's root_blkg to the bio.
552eda14cbcSMatt Macy */
553eda14cbcSMatt Macy static inline void
vdev_bio_associate_blkg(struct bio * bio)554eda14cbcSMatt Macy vdev_bio_associate_blkg(struct bio *bio)
555eda14cbcSMatt Macy {
556caed7b1cSMartin Matuska #if defined(HAVE_BIO_BDEV_DISK)
557caed7b1cSMartin Matuska struct request_queue *q = bio->bi_bdev->bd_disk->queue;
558caed7b1cSMartin Matuska #else
559eda14cbcSMatt Macy struct request_queue *q = bio->bi_disk->queue;
560caed7b1cSMartin Matuska #endif
561eda14cbcSMatt Macy
562eda14cbcSMatt Macy ASSERT3P(q, !=, NULL);
563eda14cbcSMatt Macy ASSERT3P(bio->bi_blkg, ==, NULL);
564eda14cbcSMatt Macy
565eda14cbcSMatt Macy if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
566eda14cbcSMatt Macy bio->bi_blkg = q->root_blkg;
567eda14cbcSMatt Macy }
568681ce946SMartin Matuska
569eda14cbcSMatt Macy #define bio_associate_blkg vdev_bio_associate_blkg
570681ce946SMartin Matuska #else
571681ce946SMartin Matuska static inline void
vdev_bio_set_dev(struct bio * bio,struct block_device * bdev)572681ce946SMartin Matuska vdev_bio_set_dev(struct bio *bio, struct block_device *bdev)
573681ce946SMartin Matuska {
574681ce946SMartin Matuska #if defined(HAVE_BIO_BDEV_DISK)
575681ce946SMartin Matuska struct request_queue *q = bdev->bd_disk->queue;
576681ce946SMartin Matuska #else
577681ce946SMartin Matuska struct request_queue *q = bio->bi_disk->queue;
578681ce946SMartin Matuska #endif
579681ce946SMartin Matuska bio_clear_flag(bio, BIO_REMAPPED);
580681ce946SMartin Matuska if (bio->bi_bdev != bdev)
581681ce946SMartin Matuska bio_clear_flag(bio, BIO_THROTTLED);
582681ce946SMartin Matuska bio->bi_bdev = bdev;
583681ce946SMartin Matuska
584681ce946SMartin Matuska ASSERT3P(q, !=, NULL);
585681ce946SMartin Matuska ASSERT3P(bio->bi_blkg, ==, NULL);
586681ce946SMartin Matuska
587681ce946SMartin Matuska if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
588681ce946SMartin Matuska bio->bi_blkg = q->root_blkg;
589681ce946SMartin Matuska }
590681ce946SMartin Matuska #define bio_set_dev vdev_bio_set_dev
591681ce946SMartin Matuska #endif
592eda14cbcSMatt Macy #endif
593eda14cbcSMatt Macy #else
594eda14cbcSMatt Macy /*
595eda14cbcSMatt Macy * Provide a bio_set_dev() helper macro for pre-Linux 4.14 kernels.
596eda14cbcSMatt Macy */
597eda14cbcSMatt Macy static inline void
bio_set_dev(struct bio * bio,struct block_device * bdev)598eda14cbcSMatt Macy bio_set_dev(struct bio *bio, struct block_device *bdev)
599eda14cbcSMatt Macy {
600eda14cbcSMatt Macy bio->bi_bdev = bdev;
601eda14cbcSMatt Macy }
602eda14cbcSMatt Macy #endif /* HAVE_BIO_SET_DEV */
603e3aa18adSMartin Matuska #endif /* !HAVE_BIO_ALLOC_4ARG */
604eda14cbcSMatt Macy
605eda14cbcSMatt Macy static inline void
vdev_submit_bio(struct bio * bio)606eda14cbcSMatt Macy vdev_submit_bio(struct bio *bio)
607eda14cbcSMatt Macy {
608eda14cbcSMatt Macy struct bio_list *bio_list = current->bio_list;
609eda14cbcSMatt Macy current->bio_list = NULL;
610eda14cbcSMatt Macy vdev_submit_bio_impl(bio);
611eda14cbcSMatt Macy current->bio_list = bio_list;
612eda14cbcSMatt Macy }
613eda14cbcSMatt Macy
614e3aa18adSMartin Matuska static inline struct bio *
vdev_bio_alloc(struct block_device * bdev,gfp_t gfp_mask,unsigned short nr_vecs)615e3aa18adSMartin Matuska vdev_bio_alloc(struct block_device *bdev, gfp_t gfp_mask,
616e3aa18adSMartin Matuska unsigned short nr_vecs)
617e3aa18adSMartin Matuska {
618e3aa18adSMartin Matuska struct bio *bio;
619e3aa18adSMartin Matuska
620da5137abSMartin Matuska #ifdef HAVE_BIO_ALLOC_4ARG
621e3aa18adSMartin Matuska bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask);
622e3aa18adSMartin Matuska #else
623e3aa18adSMartin Matuska bio = bio_alloc(gfp_mask, nr_vecs);
624e3aa18adSMartin Matuska if (likely(bio != NULL))
625e3aa18adSMartin Matuska bio_set_dev(bio, bdev);
626da5137abSMartin Matuska #endif
627da5137abSMartin Matuska
628e3aa18adSMartin Matuska return (bio);
629e3aa18adSMartin Matuska }
630e3aa18adSMartin Matuska
631783d3ff6SMartin Matuska static inline uint_t
vdev_bio_max_segs(struct block_device * bdev)632783d3ff6SMartin Matuska vdev_bio_max_segs(struct block_device *bdev)
633783d3ff6SMartin Matuska {
634783d3ff6SMartin Matuska /*
635783d3ff6SMartin Matuska * Smallest of the device max segs and the tuneable max segs. Minimum
636783d3ff6SMartin Matuska * 4, so there's room to finish split pages if they come up.
637783d3ff6SMartin Matuska */
638783d3ff6SMartin Matuska const uint_t dev_max_segs = queue_max_segments(bdev_get_queue(bdev));
639783d3ff6SMartin Matuska const uint_t tune_max_segs = (zfs_vdev_disk_max_segs > 0) ?
640783d3ff6SMartin Matuska MAX(4, zfs_vdev_disk_max_segs) : dev_max_segs;
641783d3ff6SMartin Matuska const uint_t max_segs = MIN(tune_max_segs, dev_max_segs);
642783d3ff6SMartin Matuska
643783d3ff6SMartin Matuska #ifdef HAVE_BIO_MAX_SEGS
644783d3ff6SMartin Matuska return (bio_max_segs(max_segs));
645783d3ff6SMartin Matuska #else
646783d3ff6SMartin Matuska return (MIN(max_segs, BIO_MAX_PAGES));
647783d3ff6SMartin Matuska #endif
648783d3ff6SMartin Matuska }
649783d3ff6SMartin Matuska
650783d3ff6SMartin Matuska static inline uint_t
vdev_bio_max_bytes(struct block_device * bdev)651783d3ff6SMartin Matuska vdev_bio_max_bytes(struct block_device *bdev)
652783d3ff6SMartin Matuska {
653783d3ff6SMartin Matuska return (queue_max_sectors(bdev_get_queue(bdev)) << 9);
654783d3ff6SMartin Matuska }
655783d3ff6SMartin Matuska
656783d3ff6SMartin Matuska
657783d3ff6SMartin Matuska /*
658783d3ff6SMartin Matuska * Virtual block IO object (VBIO)
659783d3ff6SMartin Matuska *
660783d3ff6SMartin Matuska * Linux block IO (BIO) objects have a limit on how many data segments (pages)
661783d3ff6SMartin Matuska * they can hold. Depending on how they're allocated and structured, a large
662783d3ff6SMartin Matuska * ZIO can require more than one BIO to be submitted to the kernel, which then
663783d3ff6SMartin Matuska * all have to complete before we can return the completed ZIO back to ZFS.
664783d3ff6SMartin Matuska *
665783d3ff6SMartin Matuska * A VBIO is a wrapper around multiple BIOs, carrying everything needed to
666783d3ff6SMartin Matuska * translate a ZIO down into the kernel block layer and back again.
667783d3ff6SMartin Matuska *
668783d3ff6SMartin Matuska * Note that these are only used for data ZIOs (read/write). Meta-operations
669783d3ff6SMartin Matuska * (flush/trim) don't need multiple BIOs and so can just make the call
670783d3ff6SMartin Matuska * directly.
671783d3ff6SMartin Matuska */
672783d3ff6SMartin Matuska typedef struct {
673783d3ff6SMartin Matuska zio_t *vbio_zio; /* parent zio */
674783d3ff6SMartin Matuska
675783d3ff6SMartin Matuska struct block_device *vbio_bdev; /* blockdev to submit bios to */
676783d3ff6SMartin Matuska
677783d3ff6SMartin Matuska abd_t *vbio_abd; /* abd carrying borrowed linear buf */
678783d3ff6SMartin Matuska
679783d3ff6SMartin Matuska uint_t vbio_max_segs; /* max segs per bio */
680783d3ff6SMartin Matuska
681783d3ff6SMartin Matuska uint_t vbio_max_bytes; /* max bytes per bio */
682783d3ff6SMartin Matuska uint_t vbio_lbs_mask; /* logical block size mask */
683783d3ff6SMartin Matuska
684783d3ff6SMartin Matuska uint64_t vbio_offset; /* start offset of next bio */
685783d3ff6SMartin Matuska
686783d3ff6SMartin Matuska struct bio *vbio_bio; /* pointer to the current bio */
687783d3ff6SMartin Matuska int vbio_flags; /* bio flags */
688783d3ff6SMartin Matuska } vbio_t;
689783d3ff6SMartin Matuska
690783d3ff6SMartin Matuska static vbio_t *
vbio_alloc(zio_t * zio,struct block_device * bdev,int flags)691783d3ff6SMartin Matuska vbio_alloc(zio_t *zio, struct block_device *bdev, int flags)
692783d3ff6SMartin Matuska {
693783d3ff6SMartin Matuska vbio_t *vbio = kmem_zalloc(sizeof (vbio_t), KM_SLEEP);
694783d3ff6SMartin Matuska
695783d3ff6SMartin Matuska vbio->vbio_zio = zio;
696783d3ff6SMartin Matuska vbio->vbio_bdev = bdev;
697783d3ff6SMartin Matuska vbio->vbio_abd = NULL;
698783d3ff6SMartin Matuska vbio->vbio_max_segs = vdev_bio_max_segs(bdev);
699783d3ff6SMartin Matuska vbio->vbio_max_bytes = vdev_bio_max_bytes(bdev);
700783d3ff6SMartin Matuska vbio->vbio_lbs_mask = ~(bdev_logical_block_size(bdev)-1);
701783d3ff6SMartin Matuska vbio->vbio_offset = zio->io_offset;
702783d3ff6SMartin Matuska vbio->vbio_bio = NULL;
703783d3ff6SMartin Matuska vbio->vbio_flags = flags;
704783d3ff6SMartin Matuska
705783d3ff6SMartin Matuska return (vbio);
706783d3ff6SMartin Matuska }
707783d3ff6SMartin Matuska
708783d3ff6SMartin Matuska BIO_END_IO_PROTO(vbio_completion, bio, error);
709783d3ff6SMartin Matuska
710783d3ff6SMartin Matuska static int
vbio_add_page(vbio_t * vbio,struct page * page,uint_t size,uint_t offset)711783d3ff6SMartin Matuska vbio_add_page(vbio_t *vbio, struct page *page, uint_t size, uint_t offset)
712783d3ff6SMartin Matuska {
713783d3ff6SMartin Matuska struct bio *bio = vbio->vbio_bio;
714783d3ff6SMartin Matuska uint_t ssize;
715783d3ff6SMartin Matuska
716783d3ff6SMartin Matuska while (size > 0) {
717783d3ff6SMartin Matuska if (bio == NULL) {
718783d3ff6SMartin Matuska /* New BIO, allocate and set up */
719783d3ff6SMartin Matuska bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO,
720783d3ff6SMartin Matuska vbio->vbio_max_segs);
721783d3ff6SMartin Matuska VERIFY(bio);
722783d3ff6SMartin Matuska
723783d3ff6SMartin Matuska BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9;
724783d3ff6SMartin Matuska bio_set_op_attrs(bio,
725783d3ff6SMartin Matuska vbio->vbio_zio->io_type == ZIO_TYPE_WRITE ?
726783d3ff6SMartin Matuska WRITE : READ, vbio->vbio_flags);
727783d3ff6SMartin Matuska
728783d3ff6SMartin Matuska if (vbio->vbio_bio) {
729783d3ff6SMartin Matuska bio_chain(vbio->vbio_bio, bio);
730783d3ff6SMartin Matuska vdev_submit_bio(vbio->vbio_bio);
731783d3ff6SMartin Matuska }
732783d3ff6SMartin Matuska vbio->vbio_bio = bio;
733783d3ff6SMartin Matuska }
734783d3ff6SMartin Matuska
735783d3ff6SMartin Matuska /*
736783d3ff6SMartin Matuska * Only load as much of the current page data as will fit in
737783d3ff6SMartin Matuska * the space left in the BIO, respecting lbs alignment. Older
738783d3ff6SMartin Matuska * kernels will error if we try to overfill the BIO, while
739783d3ff6SMartin Matuska * newer ones will accept it and split the BIO. This ensures
740783d3ff6SMartin Matuska * everything works on older kernels, and avoids an additional
741783d3ff6SMartin Matuska * overhead on the new.
742783d3ff6SMartin Matuska */
743783d3ff6SMartin Matuska ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) &
744783d3ff6SMartin Matuska vbio->vbio_lbs_mask);
745783d3ff6SMartin Matuska if (ssize > 0 &&
746783d3ff6SMartin Matuska bio_add_page(bio, page, ssize, offset) == ssize) {
747783d3ff6SMartin Matuska /* Accepted, adjust and load any remaining. */
748783d3ff6SMartin Matuska size -= ssize;
749783d3ff6SMartin Matuska offset += ssize;
750783d3ff6SMartin Matuska continue;
751783d3ff6SMartin Matuska }
752783d3ff6SMartin Matuska
753783d3ff6SMartin Matuska /* No room, set up for a new BIO and loop */
754783d3ff6SMartin Matuska vbio->vbio_offset += BIO_BI_SIZE(bio);
755783d3ff6SMartin Matuska
756783d3ff6SMartin Matuska /* Signal new BIO allocation wanted */
757783d3ff6SMartin Matuska bio = NULL;
758783d3ff6SMartin Matuska }
759783d3ff6SMartin Matuska
760783d3ff6SMartin Matuska return (0);
761783d3ff6SMartin Matuska }
762783d3ff6SMartin Matuska
763783d3ff6SMartin Matuska /* Iterator callback to submit ABD pages to the vbio. */
764783d3ff6SMartin Matuska static int
vbio_fill_cb(struct page * page,size_t off,size_t len,void * priv)765783d3ff6SMartin Matuska vbio_fill_cb(struct page *page, size_t off, size_t len, void *priv)
766783d3ff6SMartin Matuska {
767783d3ff6SMartin Matuska vbio_t *vbio = priv;
768783d3ff6SMartin Matuska return (vbio_add_page(vbio, page, len, off));
769783d3ff6SMartin Matuska }
770783d3ff6SMartin Matuska
771783d3ff6SMartin Matuska /* Create some BIOs, fill them with data and submit them */
772783d3ff6SMartin Matuska static void
vbio_submit(vbio_t * vbio,abd_t * abd,uint64_t size)773783d3ff6SMartin Matuska vbio_submit(vbio_t *vbio, abd_t *abd, uint64_t size)
774783d3ff6SMartin Matuska {
775783d3ff6SMartin Matuska /*
776783d3ff6SMartin Matuska * We plug so we can submit the BIOs as we go and only unplug them when
777783d3ff6SMartin Matuska * they are fully created and submitted. This is important; if we don't
778783d3ff6SMartin Matuska * plug, then the kernel may start executing earlier BIOs while we're
779783d3ff6SMartin Matuska * still creating and executing later ones, and if the device goes
780783d3ff6SMartin Matuska * away while that's happening, older kernels can get confused and
781783d3ff6SMartin Matuska * trample memory.
782783d3ff6SMartin Matuska */
783783d3ff6SMartin Matuska struct blk_plug plug;
784783d3ff6SMartin Matuska blk_start_plug(&plug);
785783d3ff6SMartin Matuska
786783d3ff6SMartin Matuska (void) abd_iterate_page_func(abd, 0, size, vbio_fill_cb, vbio);
787783d3ff6SMartin Matuska ASSERT(vbio->vbio_bio);
788783d3ff6SMartin Matuska
789783d3ff6SMartin Matuska vbio->vbio_bio->bi_end_io = vbio_completion;
790783d3ff6SMartin Matuska vbio->vbio_bio->bi_private = vbio;
791783d3ff6SMartin Matuska
7921719886fSMartin Matuska /*
7931719886fSMartin Matuska * Once submitted, vbio_bio now owns vbio (through bi_private) and we
7941719886fSMartin Matuska * can't touch it again. The bio may complete and vbio_completion() be
7951719886fSMartin Matuska * called and free the vbio before this task is run again, so we must
7961719886fSMartin Matuska * consider it invalid from this point.
7971719886fSMartin Matuska */
798783d3ff6SMartin Matuska vdev_submit_bio(vbio->vbio_bio);
799783d3ff6SMartin Matuska
800783d3ff6SMartin Matuska blk_finish_plug(&plug);
801783d3ff6SMartin Matuska }
802783d3ff6SMartin Matuska
803783d3ff6SMartin Matuska /* IO completion callback */
BIO_END_IO_PROTO(vbio_completion,bio,error)804783d3ff6SMartin Matuska BIO_END_IO_PROTO(vbio_completion, bio, error)
805783d3ff6SMartin Matuska {
806783d3ff6SMartin Matuska vbio_t *vbio = bio->bi_private;
807783d3ff6SMartin Matuska zio_t *zio = vbio->vbio_zio;
808783d3ff6SMartin Matuska
809783d3ff6SMartin Matuska ASSERT(zio);
810783d3ff6SMartin Matuska
811783d3ff6SMartin Matuska /* Capture and log any errors */
812783d3ff6SMartin Matuska #ifdef HAVE_1ARG_BIO_END_IO_T
813783d3ff6SMartin Matuska zio->io_error = BIO_END_IO_ERROR(bio);
814783d3ff6SMartin Matuska #else
815783d3ff6SMartin Matuska zio->io_error = 0;
816783d3ff6SMartin Matuska if (error)
817783d3ff6SMartin Matuska zio->io_error = -(error);
818783d3ff6SMartin Matuska else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
819783d3ff6SMartin Matuska zio->io_error = EIO;
820783d3ff6SMartin Matuska #endif
821783d3ff6SMartin Matuska ASSERT3U(zio->io_error, >=, 0);
822783d3ff6SMartin Matuska
823783d3ff6SMartin Matuska if (zio->io_error)
824783d3ff6SMartin Matuska vdev_disk_error(zio);
825783d3ff6SMartin Matuska
826783d3ff6SMartin Matuska /* Return the BIO to the kernel */
827783d3ff6SMartin Matuska bio_put(bio);
828783d3ff6SMartin Matuska
829783d3ff6SMartin Matuska /*
830783d3ff6SMartin Matuska * If we copied the ABD before issuing it, clean up and return the copy
831783d3ff6SMartin Matuska * to the ADB, with changes if appropriate.
832783d3ff6SMartin Matuska */
833783d3ff6SMartin Matuska if (vbio->vbio_abd != NULL) {
834783d3ff6SMartin Matuska void *buf = abd_to_buf(vbio->vbio_abd);
835783d3ff6SMartin Matuska abd_free(vbio->vbio_abd);
836783d3ff6SMartin Matuska vbio->vbio_abd = NULL;
837783d3ff6SMartin Matuska
838783d3ff6SMartin Matuska if (zio->io_type == ZIO_TYPE_READ)
839783d3ff6SMartin Matuska abd_return_buf_copy(zio->io_abd, buf, zio->io_size);
840783d3ff6SMartin Matuska else
841783d3ff6SMartin Matuska abd_return_buf(zio->io_abd, buf, zio->io_size);
842783d3ff6SMartin Matuska }
843783d3ff6SMartin Matuska
844783d3ff6SMartin Matuska /* Final cleanup */
845783d3ff6SMartin Matuska kmem_free(vbio, sizeof (vbio_t));
846783d3ff6SMartin Matuska
847783d3ff6SMartin Matuska /* All done, submit for processing */
848783d3ff6SMartin Matuska zio_delay_interrupt(zio);
849783d3ff6SMartin Matuska }
850783d3ff6SMartin Matuska
851783d3ff6SMartin Matuska /*
852783d3ff6SMartin Matuska * Iterator callback to count ABD pages and check their size & alignment.
853783d3ff6SMartin Matuska *
854783d3ff6SMartin Matuska * On Linux, each BIO segment can take a page pointer, and an offset+length of
855783d3ff6SMartin Matuska * the data within that page. A page can be arbitrarily large ("compound"
856783d3ff6SMartin Matuska * pages) but we still have to ensure the data portion is correctly sized and
857783d3ff6SMartin Matuska * aligned to the logical block size, to ensure that if the kernel wants to
858783d3ff6SMartin Matuska * split the BIO, the two halves will still be properly aligned.
8591719886fSMartin Matuska *
8601719886fSMartin Matuska * NOTE: if you change this function, change the copy in
8611719886fSMartin Matuska * tests/zfs-tests/tests/functional/vdev_disk/page_alignment.c, and add test
8621719886fSMartin Matuska * data there to validate the change you're making.
8631719886fSMartin Matuska *
864783d3ff6SMartin Matuska */
865783d3ff6SMartin Matuska typedef struct {
866783d3ff6SMartin Matuska uint_t bmask;
867783d3ff6SMartin Matuska uint_t npages;
868783d3ff6SMartin Matuska uint_t end;
869783d3ff6SMartin Matuska } vdev_disk_check_pages_t;
870783d3ff6SMartin Matuska
871783d3ff6SMartin Matuska static int
vdev_disk_check_pages_cb(struct page * page,size_t off,size_t len,void * priv)872783d3ff6SMartin Matuska vdev_disk_check_pages_cb(struct page *page, size_t off, size_t len, void *priv)
873783d3ff6SMartin Matuska {
8741719886fSMartin Matuska (void) page;
875783d3ff6SMartin Matuska vdev_disk_check_pages_t *s = priv;
876783d3ff6SMartin Matuska
877783d3ff6SMartin Matuska /*
878783d3ff6SMartin Matuska * If we didn't finish on a block size boundary last time, then there
879783d3ff6SMartin Matuska * would be a gap if we tried to use this ABD as-is, so abort.
880783d3ff6SMartin Matuska */
881783d3ff6SMartin Matuska if (s->end != 0)
882783d3ff6SMartin Matuska return (1);
883783d3ff6SMartin Matuska
884783d3ff6SMartin Matuska /*
885783d3ff6SMartin Matuska * Note if we're taking less than a full block, so we can check it
886783d3ff6SMartin Matuska * above on the next call.
887783d3ff6SMartin Matuska */
8881719886fSMartin Matuska s->end = (off+len) & s->bmask;
889783d3ff6SMartin Matuska
890783d3ff6SMartin Matuska /* All blocks after the first must start on a block size boundary. */
891783d3ff6SMartin Matuska if (s->npages != 0 && (off & s->bmask) != 0)
892783d3ff6SMartin Matuska return (1);
893783d3ff6SMartin Matuska
894783d3ff6SMartin Matuska s->npages++;
895783d3ff6SMartin Matuska return (0);
896783d3ff6SMartin Matuska }
897783d3ff6SMartin Matuska
898783d3ff6SMartin Matuska /*
899783d3ff6SMartin Matuska * Check if we can submit the pages in this ABD to the kernel as-is. Returns
900783d3ff6SMartin Matuska * the number of pages, or 0 if it can't be submitted like this.
901783d3ff6SMartin Matuska */
902783d3ff6SMartin Matuska static boolean_t
vdev_disk_check_pages(abd_t * abd,uint64_t size,struct block_device * bdev)903783d3ff6SMartin Matuska vdev_disk_check_pages(abd_t *abd, uint64_t size, struct block_device *bdev)
904783d3ff6SMartin Matuska {
905783d3ff6SMartin Matuska vdev_disk_check_pages_t s = {
906783d3ff6SMartin Matuska .bmask = bdev_logical_block_size(bdev)-1,
907783d3ff6SMartin Matuska .npages = 0,
908783d3ff6SMartin Matuska .end = 0,
909783d3ff6SMartin Matuska };
910783d3ff6SMartin Matuska
911783d3ff6SMartin Matuska if (abd_iterate_page_func(abd, 0, size, vdev_disk_check_pages_cb, &s))
912783d3ff6SMartin Matuska return (B_FALSE);
913783d3ff6SMartin Matuska
914783d3ff6SMartin Matuska return (B_TRUE);
915783d3ff6SMartin Matuska }
916783d3ff6SMartin Matuska
917783d3ff6SMartin Matuska static int
vdev_disk_io_rw(zio_t * zio)918783d3ff6SMartin Matuska vdev_disk_io_rw(zio_t *zio)
919783d3ff6SMartin Matuska {
920783d3ff6SMartin Matuska vdev_t *v = zio->io_vd;
921783d3ff6SMartin Matuska vdev_disk_t *vd = v->vdev_tsd;
922783d3ff6SMartin Matuska struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
923783d3ff6SMartin Matuska int flags = 0;
924783d3ff6SMartin Matuska
925783d3ff6SMartin Matuska /*
926783d3ff6SMartin Matuska * Accessing outside the block device is never allowed.
927783d3ff6SMartin Matuska */
928783d3ff6SMartin Matuska if (zio->io_offset + zio->io_size > bdev->bd_inode->i_size) {
929783d3ff6SMartin Matuska vdev_dbgmsg(zio->io_vd,
930783d3ff6SMartin Matuska "Illegal access %llu size %llu, device size %llu",
931783d3ff6SMartin Matuska (u_longlong_t)zio->io_offset,
932783d3ff6SMartin Matuska (u_longlong_t)zio->io_size,
933783d3ff6SMartin Matuska (u_longlong_t)i_size_read(bdev->bd_inode));
934783d3ff6SMartin Matuska return (SET_ERROR(EIO));
935783d3ff6SMartin Matuska }
936783d3ff6SMartin Matuska
937783d3ff6SMartin Matuska if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) &&
938783d3ff6SMartin Matuska v->vdev_failfast == B_TRUE) {
939783d3ff6SMartin Matuska bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
940783d3ff6SMartin Matuska zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
941783d3ff6SMartin Matuska }
942783d3ff6SMartin Matuska
943783d3ff6SMartin Matuska /*
944783d3ff6SMartin Matuska * Check alignment of the incoming ABD. If any part of it would require
945783d3ff6SMartin Matuska * submitting a page that is not aligned to the logical block size,
946783d3ff6SMartin Matuska * then we take a copy into a linear buffer and submit that instead.
947783d3ff6SMartin Matuska * This should be impossible on a 512b LBS, and fairly rare on 4K,
948783d3ff6SMartin Matuska * usually requiring abnormally-small data blocks (eg gang blocks)
949783d3ff6SMartin Matuska * mixed into the same ABD as larger ones (eg aggregated).
950783d3ff6SMartin Matuska */
951783d3ff6SMartin Matuska abd_t *abd = zio->io_abd;
952783d3ff6SMartin Matuska if (!vdev_disk_check_pages(abd, zio->io_size, bdev)) {
953783d3ff6SMartin Matuska void *buf;
954783d3ff6SMartin Matuska if (zio->io_type == ZIO_TYPE_READ)
955783d3ff6SMartin Matuska buf = abd_borrow_buf(zio->io_abd, zio->io_size);
956783d3ff6SMartin Matuska else
957783d3ff6SMartin Matuska buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size);
958783d3ff6SMartin Matuska
959783d3ff6SMartin Matuska /*
960783d3ff6SMartin Matuska * Wrap the copy in an abd_t, so we can use the same iterators
961783d3ff6SMartin Matuska * to count and fill the vbio later.
962783d3ff6SMartin Matuska */
963783d3ff6SMartin Matuska abd = abd_get_from_buf(buf, zio->io_size);
964783d3ff6SMartin Matuska
965783d3ff6SMartin Matuska /*
966783d3ff6SMartin Matuska * False here would mean the borrowed copy has an invalid
967783d3ff6SMartin Matuska * alignment too, which would mean we've somehow been passed a
968783d3ff6SMartin Matuska * linear ABD with an interior page that has a non-zero offset
969783d3ff6SMartin Matuska * or a size not a multiple of PAGE_SIZE. This is not possible.
970783d3ff6SMartin Matuska * It would mean either zio_buf_alloc() or its underlying
971783d3ff6SMartin Matuska * allocators have done something extremely strange, or our
972783d3ff6SMartin Matuska * math in vdev_disk_check_pages() is wrong. In either case,
973783d3ff6SMartin Matuska * something in seriously wrong and its not safe to continue.
974783d3ff6SMartin Matuska */
975783d3ff6SMartin Matuska VERIFY(vdev_disk_check_pages(abd, zio->io_size, bdev));
976783d3ff6SMartin Matuska }
977783d3ff6SMartin Matuska
978783d3ff6SMartin Matuska /* Allocate vbio, with a pointer to the borrowed ABD if necessary */
979783d3ff6SMartin Matuska vbio_t *vbio = vbio_alloc(zio, bdev, flags);
980783d3ff6SMartin Matuska if (abd != zio->io_abd)
981783d3ff6SMartin Matuska vbio->vbio_abd = abd;
982783d3ff6SMartin Matuska
983783d3ff6SMartin Matuska /* Fill it with data pages and submit it to the kernel */
984783d3ff6SMartin Matuska vbio_submit(vbio, abd, zio->io_size);
985783d3ff6SMartin Matuska return (0);
986783d3ff6SMartin Matuska }
987783d3ff6SMartin Matuska
988783d3ff6SMartin Matuska /* ========== */
989783d3ff6SMartin Matuska
990783d3ff6SMartin Matuska /*
991783d3ff6SMartin Matuska * This is the classic, battle-tested BIO submission code. Until we're totally
992783d3ff6SMartin Matuska * sure that the new code is safe and correct in all cases, this will remain
993783d3ff6SMartin Matuska * available and can be enabled by setting zfs_vdev_disk_classic=1 at module
994783d3ff6SMartin Matuska * load time.
995783d3ff6SMartin Matuska *
996783d3ff6SMartin Matuska * These functions have been renamed to vdev_classic_* to make it clear what
997783d3ff6SMartin Matuska * they belong to, but their implementations are unchanged.
998783d3ff6SMartin Matuska */
999783d3ff6SMartin Matuska
1000783d3ff6SMartin Matuska /*
1001783d3ff6SMartin Matuska * Virtual device vector for disks.
1002783d3ff6SMartin Matuska */
1003783d3ff6SMartin Matuska typedef struct dio_request {
1004783d3ff6SMartin Matuska zio_t *dr_zio; /* Parent ZIO */
1005783d3ff6SMartin Matuska atomic_t dr_ref; /* References */
1006783d3ff6SMartin Matuska int dr_error; /* Bio error */
1007783d3ff6SMartin Matuska int dr_bio_count; /* Count of bio's */
1008783d3ff6SMartin Matuska struct bio *dr_bio[]; /* Attached bio's */
1009783d3ff6SMartin Matuska } dio_request_t;
1010783d3ff6SMartin Matuska
1011783d3ff6SMartin Matuska static dio_request_t *
vdev_classic_dio_alloc(int bio_count)1012783d3ff6SMartin Matuska vdev_classic_dio_alloc(int bio_count)
1013783d3ff6SMartin Matuska {
1014783d3ff6SMartin Matuska dio_request_t *dr = kmem_zalloc(sizeof (dio_request_t) +
1015783d3ff6SMartin Matuska sizeof (struct bio *) * bio_count, KM_SLEEP);
1016783d3ff6SMartin Matuska atomic_set(&dr->dr_ref, 0);
1017783d3ff6SMartin Matuska dr->dr_bio_count = bio_count;
1018783d3ff6SMartin Matuska dr->dr_error = 0;
1019783d3ff6SMartin Matuska
1020783d3ff6SMartin Matuska for (int i = 0; i < dr->dr_bio_count; i++)
1021783d3ff6SMartin Matuska dr->dr_bio[i] = NULL;
1022783d3ff6SMartin Matuska
1023783d3ff6SMartin Matuska return (dr);
1024783d3ff6SMartin Matuska }
1025783d3ff6SMartin Matuska
1026783d3ff6SMartin Matuska static void
vdev_classic_dio_free(dio_request_t * dr)1027783d3ff6SMartin Matuska vdev_classic_dio_free(dio_request_t *dr)
1028783d3ff6SMartin Matuska {
1029783d3ff6SMartin Matuska int i;
1030783d3ff6SMartin Matuska
1031783d3ff6SMartin Matuska for (i = 0; i < dr->dr_bio_count; i++)
1032783d3ff6SMartin Matuska if (dr->dr_bio[i])
1033783d3ff6SMartin Matuska bio_put(dr->dr_bio[i]);
1034783d3ff6SMartin Matuska
1035783d3ff6SMartin Matuska kmem_free(dr, sizeof (dio_request_t) +
1036783d3ff6SMartin Matuska sizeof (struct bio *) * dr->dr_bio_count);
1037783d3ff6SMartin Matuska }
1038783d3ff6SMartin Matuska
1039783d3ff6SMartin Matuska static void
vdev_classic_dio_get(dio_request_t * dr)1040783d3ff6SMartin Matuska vdev_classic_dio_get(dio_request_t *dr)
1041783d3ff6SMartin Matuska {
1042783d3ff6SMartin Matuska atomic_inc(&dr->dr_ref);
1043783d3ff6SMartin Matuska }
1044783d3ff6SMartin Matuska
1045783d3ff6SMartin Matuska static void
vdev_classic_dio_put(dio_request_t * dr)1046783d3ff6SMartin Matuska vdev_classic_dio_put(dio_request_t *dr)
1047783d3ff6SMartin Matuska {
1048783d3ff6SMartin Matuska int rc = atomic_dec_return(&dr->dr_ref);
1049783d3ff6SMartin Matuska
1050783d3ff6SMartin Matuska /*
1051783d3ff6SMartin Matuska * Free the dio_request when the last reference is dropped and
1052783d3ff6SMartin Matuska * ensure zio_interpret is called only once with the correct zio
1053783d3ff6SMartin Matuska */
1054783d3ff6SMartin Matuska if (rc == 0) {
1055783d3ff6SMartin Matuska zio_t *zio = dr->dr_zio;
1056783d3ff6SMartin Matuska int error = dr->dr_error;
1057783d3ff6SMartin Matuska
1058783d3ff6SMartin Matuska vdev_classic_dio_free(dr);
1059783d3ff6SMartin Matuska
1060783d3ff6SMartin Matuska if (zio) {
1061783d3ff6SMartin Matuska zio->io_error = error;
1062783d3ff6SMartin Matuska ASSERT3S(zio->io_error, >=, 0);
1063783d3ff6SMartin Matuska if (zio->io_error)
1064783d3ff6SMartin Matuska vdev_disk_error(zio);
1065783d3ff6SMartin Matuska
1066783d3ff6SMartin Matuska zio_delay_interrupt(zio);
1067783d3ff6SMartin Matuska }
1068783d3ff6SMartin Matuska }
1069783d3ff6SMartin Matuska }
1070783d3ff6SMartin Matuska
BIO_END_IO_PROTO(vdev_classic_physio_completion,bio,error)1071783d3ff6SMartin Matuska BIO_END_IO_PROTO(vdev_classic_physio_completion, bio, error)
1072783d3ff6SMartin Matuska {
1073783d3ff6SMartin Matuska dio_request_t *dr = bio->bi_private;
1074783d3ff6SMartin Matuska
1075783d3ff6SMartin Matuska if (dr->dr_error == 0) {
1076783d3ff6SMartin Matuska #ifdef HAVE_1ARG_BIO_END_IO_T
1077783d3ff6SMartin Matuska dr->dr_error = BIO_END_IO_ERROR(bio);
1078783d3ff6SMartin Matuska #else
1079783d3ff6SMartin Matuska if (error)
1080783d3ff6SMartin Matuska dr->dr_error = -(error);
1081783d3ff6SMartin Matuska else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1082783d3ff6SMartin Matuska dr->dr_error = EIO;
1083783d3ff6SMartin Matuska #endif
1084783d3ff6SMartin Matuska }
1085783d3ff6SMartin Matuska
1086783d3ff6SMartin Matuska /* Drop reference acquired by vdev_classic_physio */
1087783d3ff6SMartin Matuska vdev_classic_dio_put(dr);
1088783d3ff6SMartin Matuska }
1089783d3ff6SMartin Matuska
1090e3aa18adSMartin Matuska static inline unsigned int
vdev_classic_bio_max_segs(zio_t * zio,int bio_size,uint64_t abd_offset)1091783d3ff6SMartin Matuska vdev_classic_bio_max_segs(zio_t *zio, int bio_size, uint64_t abd_offset)
1092e3aa18adSMartin Matuska {
1093e3aa18adSMartin Matuska unsigned long nr_segs = abd_nr_pages_off(zio->io_abd,
1094e3aa18adSMartin Matuska bio_size, abd_offset);
1095e3aa18adSMartin Matuska
1096e3aa18adSMartin Matuska #ifdef HAVE_BIO_MAX_SEGS
1097e3aa18adSMartin Matuska return (bio_max_segs(nr_segs));
1098e3aa18adSMartin Matuska #else
1099e3aa18adSMartin Matuska return (MIN(nr_segs, BIO_MAX_PAGES));
1100e3aa18adSMartin Matuska #endif
1101e3aa18adSMartin Matuska }
1102e3aa18adSMartin Matuska
1103eda14cbcSMatt Macy static int
vdev_classic_physio(zio_t * zio)1104783d3ff6SMartin Matuska vdev_classic_physio(zio_t *zio)
1105eda14cbcSMatt Macy {
1106783d3ff6SMartin Matuska vdev_t *v = zio->io_vd;
1107783d3ff6SMartin Matuska vdev_disk_t *vd = v->vdev_tsd;
1108783d3ff6SMartin Matuska struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
1109783d3ff6SMartin Matuska size_t io_size = zio->io_size;
1110783d3ff6SMartin Matuska uint64_t io_offset = zio->io_offset;
1111783d3ff6SMartin Matuska int rw = zio->io_type == ZIO_TYPE_READ ? READ : WRITE;
1112783d3ff6SMartin Matuska int flags = 0;
1113783d3ff6SMartin Matuska
1114eda14cbcSMatt Macy dio_request_t *dr;
1115eda14cbcSMatt Macy uint64_t abd_offset;
1116eda14cbcSMatt Macy uint64_t bio_offset;
1117184c1b94SMartin Matuska int bio_size;
1118184c1b94SMartin Matuska int bio_count = 16;
1119184c1b94SMartin Matuska int error = 0;
1120eda14cbcSMatt Macy struct blk_plug plug;
1121e3aa18adSMartin Matuska unsigned short nr_vecs;
1122eda14cbcSMatt Macy
1123eda14cbcSMatt Macy /*
1124eda14cbcSMatt Macy * Accessing outside the block device is never allowed.
1125eda14cbcSMatt Macy */
1126eda14cbcSMatt Macy if (io_offset + io_size > bdev->bd_inode->i_size) {
1127eda14cbcSMatt Macy vdev_dbgmsg(zio->io_vd,
1128eda14cbcSMatt Macy "Illegal access %llu size %llu, device size %llu",
11291f88aa09SMartin Matuska (u_longlong_t)io_offset,
11301f88aa09SMartin Matuska (u_longlong_t)io_size,
11311f88aa09SMartin Matuska (u_longlong_t)i_size_read(bdev->bd_inode));
1132eda14cbcSMatt Macy return (SET_ERROR(EIO));
1133eda14cbcSMatt Macy }
1134eda14cbcSMatt Macy
1135eda14cbcSMatt Macy retry:
1136783d3ff6SMartin Matuska dr = vdev_classic_dio_alloc(bio_count);
1137eda14cbcSMatt Macy
113815f0b8c3SMartin Matuska if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) &&
1139dbd5678dSMartin Matuska zio->io_vd->vdev_failfast == B_TRUE) {
1140dbd5678dSMartin Matuska bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
1141dbd5678dSMartin Matuska zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
1142dbd5678dSMartin Matuska }
1143eda14cbcSMatt Macy
1144eda14cbcSMatt Macy dr->dr_zio = zio;
1145eda14cbcSMatt Macy
1146eda14cbcSMatt Macy /*
1147184c1b94SMartin Matuska * Since bio's can have up to BIO_MAX_PAGES=256 iovec's, each of which
1148184c1b94SMartin Matuska * is at least 512 bytes and at most PAGESIZE (typically 4K), one bio
1149184c1b94SMartin Matuska * can cover at least 128KB and at most 1MB. When the required number
1150184c1b94SMartin Matuska * of iovec's exceeds this, we are forced to break the IO in multiple
1151184c1b94SMartin Matuska * bio's and wait for them all to complete. This is likely if the
1152184c1b94SMartin Matuska * recordsize property is increased beyond 1MB. The default
1153184c1b94SMartin Matuska * bio_count=16 should typically accommodate the maximum-size zio of
1154184c1b94SMartin Matuska * 16MB.
1155eda14cbcSMatt Macy */
1156eda14cbcSMatt Macy
1157eda14cbcSMatt Macy abd_offset = 0;
1158eda14cbcSMatt Macy bio_offset = io_offset;
1159eda14cbcSMatt Macy bio_size = io_size;
1160184c1b94SMartin Matuska for (int i = 0; i <= dr->dr_bio_count; i++) {
1161eda14cbcSMatt Macy
1162eda14cbcSMatt Macy /* Finished constructing bio's for given buffer */
1163eda14cbcSMatt Macy if (bio_size <= 0)
1164eda14cbcSMatt Macy break;
1165eda14cbcSMatt Macy
1166eda14cbcSMatt Macy /*
1167184c1b94SMartin Matuska * If additional bio's are required, we have to retry, but
1168184c1b94SMartin Matuska * this should be rare - see the comment above.
1169eda14cbcSMatt Macy */
1170eda14cbcSMatt Macy if (dr->dr_bio_count == i) {
1171783d3ff6SMartin Matuska vdev_classic_dio_free(dr);
1172eda14cbcSMatt Macy bio_count *= 2;
1173eda14cbcSMatt Macy goto retry;
1174eda14cbcSMatt Macy }
1175eda14cbcSMatt Macy
1176783d3ff6SMartin Matuska nr_vecs = vdev_classic_bio_max_segs(zio, bio_size, abd_offset);
1177e3aa18adSMartin Matuska dr->dr_bio[i] = vdev_bio_alloc(bdev, GFP_NOIO, nr_vecs);
1178eda14cbcSMatt Macy if (unlikely(dr->dr_bio[i] == NULL)) {
1179783d3ff6SMartin Matuska vdev_classic_dio_free(dr);
1180eda14cbcSMatt Macy return (SET_ERROR(ENOMEM));
1181eda14cbcSMatt Macy }
1182eda14cbcSMatt Macy
1183783d3ff6SMartin Matuska /* Matching put called by vdev_classic_physio_completion */
1184783d3ff6SMartin Matuska vdev_classic_dio_get(dr);
1185eda14cbcSMatt Macy
1186eda14cbcSMatt Macy BIO_BI_SECTOR(dr->dr_bio[i]) = bio_offset >> 9;
1187783d3ff6SMartin Matuska dr->dr_bio[i]->bi_end_io = vdev_classic_physio_completion;
1188eda14cbcSMatt Macy dr->dr_bio[i]->bi_private = dr;
1189eda14cbcSMatt Macy bio_set_op_attrs(dr->dr_bio[i], rw, flags);
1190eda14cbcSMatt Macy
1191eda14cbcSMatt Macy /* Remaining size is returned to become the new size */
1192eda14cbcSMatt Macy bio_size = abd_bio_map_off(dr->dr_bio[i], zio->io_abd,
1193eda14cbcSMatt Macy bio_size, abd_offset);
1194eda14cbcSMatt Macy
1195eda14cbcSMatt Macy /* Advance in buffer and construct another bio if needed */
1196eda14cbcSMatt Macy abd_offset += BIO_BI_SIZE(dr->dr_bio[i]);
1197eda14cbcSMatt Macy bio_offset += BIO_BI_SIZE(dr->dr_bio[i]);
1198eda14cbcSMatt Macy }
1199eda14cbcSMatt Macy
1200eda14cbcSMatt Macy /* Extra reference to protect dio_request during vdev_submit_bio */
1201783d3ff6SMartin Matuska vdev_classic_dio_get(dr);
1202eda14cbcSMatt Macy
1203eda14cbcSMatt Macy if (dr->dr_bio_count > 1)
1204eda14cbcSMatt Macy blk_start_plug(&plug);
1205eda14cbcSMatt Macy
1206eda14cbcSMatt Macy /* Submit all bio's associated with this dio */
1207184c1b94SMartin Matuska for (int i = 0; i < dr->dr_bio_count; i++) {
1208eda14cbcSMatt Macy if (dr->dr_bio[i])
1209eda14cbcSMatt Macy vdev_submit_bio(dr->dr_bio[i]);
1210184c1b94SMartin Matuska }
1211eda14cbcSMatt Macy
1212eda14cbcSMatt Macy if (dr->dr_bio_count > 1)
1213eda14cbcSMatt Macy blk_finish_plug(&plug);
1214eda14cbcSMatt Macy
1215783d3ff6SMartin Matuska vdev_classic_dio_put(dr);
1216eda14cbcSMatt Macy
1217eda14cbcSMatt Macy return (error);
1218eda14cbcSMatt Macy }
1219eda14cbcSMatt Macy
1220783d3ff6SMartin Matuska /* ========== */
1221783d3ff6SMartin Matuska
BIO_END_IO_PROTO(vdev_disk_io_flush_completion,bio,error)1222eda14cbcSMatt Macy BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, error)
1223eda14cbcSMatt Macy {
1224eda14cbcSMatt Macy zio_t *zio = bio->bi_private;
1225eda14cbcSMatt Macy #ifdef HAVE_1ARG_BIO_END_IO_T
1226eda14cbcSMatt Macy zio->io_error = BIO_END_IO_ERROR(bio);
1227eda14cbcSMatt Macy #else
1228eda14cbcSMatt Macy zio->io_error = -error;
1229eda14cbcSMatt Macy #endif
1230eda14cbcSMatt Macy
1231eda14cbcSMatt Macy if (zio->io_error && (zio->io_error == EOPNOTSUPP))
1232eda14cbcSMatt Macy zio->io_vd->vdev_nowritecache = B_TRUE;
1233eda14cbcSMatt Macy
1234eda14cbcSMatt Macy bio_put(bio);
1235eda14cbcSMatt Macy ASSERT3S(zio->io_error, >=, 0);
1236eda14cbcSMatt Macy if (zio->io_error)
1237eda14cbcSMatt Macy vdev_disk_error(zio);
1238eda14cbcSMatt Macy zio_interrupt(zio);
1239eda14cbcSMatt Macy }
1240eda14cbcSMatt Macy
1241eda14cbcSMatt Macy static int
vdev_disk_io_flush(struct block_device * bdev,zio_t * zio)1242eda14cbcSMatt Macy vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
1243eda14cbcSMatt Macy {
1244eda14cbcSMatt Macy struct request_queue *q;
1245eda14cbcSMatt Macy struct bio *bio;
1246eda14cbcSMatt Macy
1247eda14cbcSMatt Macy q = bdev_get_queue(bdev);
1248eda14cbcSMatt Macy if (!q)
1249eda14cbcSMatt Macy return (SET_ERROR(ENXIO));
1250eda14cbcSMatt Macy
1251e3aa18adSMartin Matuska bio = vdev_bio_alloc(bdev, GFP_NOIO, 0);
1252eda14cbcSMatt Macy if (unlikely(bio == NULL))
1253eda14cbcSMatt Macy return (SET_ERROR(ENOMEM));
1254eda14cbcSMatt Macy
1255eda14cbcSMatt Macy bio->bi_end_io = vdev_disk_io_flush_completion;
1256eda14cbcSMatt Macy bio->bi_private = zio;
1257eda14cbcSMatt Macy bio_set_flush(bio);
1258eda14cbcSMatt Macy vdev_submit_bio(bio);
1259eda14cbcSMatt Macy invalidate_bdev(bdev);
1260eda14cbcSMatt Macy
1261eda14cbcSMatt Macy return (0);
1262eda14cbcSMatt Macy }
1263eda14cbcSMatt Macy
BIO_END_IO_PROTO(vdev_disk_discard_end_io,bio,error)1264a4e5e010SMartin Matuska BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
1265a4e5e010SMartin Matuska {
1266a4e5e010SMartin Matuska zio_t *zio = bio->bi_private;
1267a4e5e010SMartin Matuska #ifdef HAVE_1ARG_BIO_END_IO_T
1268a4e5e010SMartin Matuska zio->io_error = BIO_END_IO_ERROR(bio);
1269a4e5e010SMartin Matuska #else
1270a4e5e010SMartin Matuska zio->io_error = -error;
1271a4e5e010SMartin Matuska #endif
1272a4e5e010SMartin Matuska bio_put(bio);
1273a4e5e010SMartin Matuska if (zio->io_error)
1274a4e5e010SMartin Matuska vdev_disk_error(zio);
1275a4e5e010SMartin Matuska zio_interrupt(zio);
1276a4e5e010SMartin Matuska }
1277a4e5e010SMartin Matuska
12781719886fSMartin Matuska /*
12791719886fSMartin Matuska * Wrappers for the different secure erase and discard APIs. We use async
12801719886fSMartin Matuska * when available; in this case, *biop is set to the last bio in the chain.
12811719886fSMartin Matuska */
1282a4e5e010SMartin Matuska static int
vdev_bdev_issue_secure_erase(zfs_bdev_handle_t * bdh,sector_t sector,sector_t nsect,struct bio ** biop)12831719886fSMartin Matuska vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
12841719886fSMartin Matuska sector_t nsect, struct bio **biop)
1285a4e5e010SMartin Matuska {
12861719886fSMartin Matuska *biop = NULL;
12871719886fSMartin Matuska int error;
1288a4e5e010SMartin Matuska
12891719886fSMartin Matuska #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
12901719886fSMartin Matuska error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
12911719886fSMartin Matuska sector, nsect, GFP_NOFS);
12921719886fSMartin Matuska #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
12931719886fSMartin Matuska error = __blkdev_issue_discard(BDH_BDEV(bdh),
12941719886fSMartin Matuska sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
12951719886fSMartin Matuska #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
12961719886fSMartin Matuska error = blkdev_issue_discard(BDH_BDEV(bdh),
12971719886fSMartin Matuska sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
1298a4e5e010SMartin Matuska #else
12991719886fSMartin Matuska #error "unsupported kernel"
1300a4e5e010SMartin Matuska #endif
13011719886fSMartin Matuska
13021719886fSMartin Matuska return (error);
13031719886fSMartin Matuska }
13041719886fSMartin Matuska
13051719886fSMartin Matuska static int
vdev_bdev_issue_discard(zfs_bdev_handle_t * bdh,sector_t sector,sector_t nsect,struct bio ** biop)13061719886fSMartin Matuska vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
13071719886fSMartin Matuska sector_t nsect, struct bio **biop)
13081719886fSMartin Matuska {
13091719886fSMartin Matuska *biop = NULL;
13101719886fSMartin Matuska int error;
13111719886fSMartin Matuska
13121719886fSMartin Matuska #if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
13131719886fSMartin Matuska error = __blkdev_issue_discard(BDH_BDEV(bdh),
13141719886fSMartin Matuska sector, nsect, GFP_NOFS, 0, biop);
13151719886fSMartin Matuska #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
13161719886fSMartin Matuska error = __blkdev_issue_discard(BDH_BDEV(bdh),
13171719886fSMartin Matuska sector, nsect, GFP_NOFS, biop);
13181719886fSMartin Matuska #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
13191719886fSMartin Matuska error = blkdev_issue_discard(BDH_BDEV(bdh),
13201719886fSMartin Matuska sector, nsect, GFP_NOFS, 0);
13211719886fSMartin Matuska #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
13221719886fSMartin Matuska error = blkdev_issue_discard(BDH_BDEV(bdh),
13231719886fSMartin Matuska sector, nsect, GFP_NOFS);
13241719886fSMartin Matuska #else
13251719886fSMartin Matuska #error "unsupported kernel"
13261719886fSMartin Matuska #endif
13271719886fSMartin Matuska
13281719886fSMartin Matuska return (error);
13291719886fSMartin Matuska }
13301719886fSMartin Matuska
13311719886fSMartin Matuska /*
13321719886fSMartin Matuska * Entry point for TRIM ops. This calls the right wrapper for secure erase or
13331719886fSMartin Matuska * discard, and then does the appropriate finishing work for error vs success
13341719886fSMartin Matuska * and async vs sync.
13351719886fSMartin Matuska */
13361719886fSMartin Matuska static int
vdev_disk_io_trim(zio_t * zio)13371719886fSMartin Matuska vdev_disk_io_trim(zio_t *zio)
13381719886fSMartin Matuska {
13391719886fSMartin Matuska int error;
13401719886fSMartin Matuska struct bio *bio;
13411719886fSMartin Matuska
13421719886fSMartin Matuska zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh;
13431719886fSMartin Matuska sector_t sector = zio->io_offset >> 9;
13441719886fSMartin Matuska sector_t nsects = zio->io_size >> 9;
13451719886fSMartin Matuska
13461719886fSMartin Matuska if (zio->io_trim_flags & ZIO_TRIM_SECURE)
13471719886fSMartin Matuska error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
13481719886fSMartin Matuska else
13491719886fSMartin Matuska error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
13501719886fSMartin Matuska
13511719886fSMartin Matuska if (error != 0)
13521719886fSMartin Matuska return (SET_ERROR(-error));
13531719886fSMartin Matuska
13541719886fSMartin Matuska if (bio == NULL) {
13551719886fSMartin Matuska /*
13561719886fSMartin Matuska * This was a synchronous op that completed successfully, so
13571719886fSMartin Matuska * return it to ZFS immediately.
13581719886fSMartin Matuska */
13591719886fSMartin Matuska zio_interrupt(zio);
13601719886fSMartin Matuska } else {
13611719886fSMartin Matuska /*
13621719886fSMartin Matuska * This was an asynchronous op; set up completion callback and
13631719886fSMartin Matuska * issue it.
13641719886fSMartin Matuska */
1365a4e5e010SMartin Matuska bio->bi_private = zio;
1366a4e5e010SMartin Matuska bio->bi_end_io = vdev_disk_discard_end_io;
1367a4e5e010SMartin Matuska vdev_submit_bio(bio);
1368a4e5e010SMartin Matuska }
1369a4e5e010SMartin Matuska
13701719886fSMartin Matuska return (0);
1371e3aa18adSMartin Matuska }
1372e3aa18adSMartin Matuska
1373783d3ff6SMartin Matuska int (*vdev_disk_io_rw_fn)(zio_t *zio) = NULL;
1374783d3ff6SMartin Matuska
1375eda14cbcSMatt Macy static void
vdev_disk_io_start(zio_t * zio)1376eda14cbcSMatt Macy vdev_disk_io_start(zio_t *zio)
1377eda14cbcSMatt Macy {
1378eda14cbcSMatt Macy vdev_t *v = zio->io_vd;
1379eda14cbcSMatt Macy vdev_disk_t *vd = v->vdev_tsd;
1380783d3ff6SMartin Matuska int error;
1381eda14cbcSMatt Macy
1382eda14cbcSMatt Macy /*
1383eda14cbcSMatt Macy * If the vdev is closed, it's likely in the REMOVED or FAULTED state.
1384eda14cbcSMatt Macy * Nothing to be done here but return failure.
1385eda14cbcSMatt Macy */
1386eda14cbcSMatt Macy if (vd == NULL) {
1387eda14cbcSMatt Macy zio->io_error = ENXIO;
1388eda14cbcSMatt Macy zio_interrupt(zio);
1389eda14cbcSMatt Macy return;
1390eda14cbcSMatt Macy }
1391eda14cbcSMatt Macy
1392eda14cbcSMatt Macy rw_enter(&vd->vd_lock, RW_READER);
1393eda14cbcSMatt Macy
1394eda14cbcSMatt Macy /*
1395eda14cbcSMatt Macy * If the vdev is closed, it's likely due to a failed reopen and is
1396eda14cbcSMatt Macy * in the UNAVAIL state. Nothing to be done here but return failure.
1397eda14cbcSMatt Macy */
1398fd45b686SMartin Matuska if (vd->vd_bdh == NULL) {
1399eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
1400eda14cbcSMatt Macy zio->io_error = ENXIO;
1401eda14cbcSMatt Macy zio_interrupt(zio);
1402eda14cbcSMatt Macy return;
1403eda14cbcSMatt Macy }
1404eda14cbcSMatt Macy
1405eda14cbcSMatt Macy switch (zio->io_type) {
14061719886fSMartin Matuska case ZIO_TYPE_FLUSH:
1407eda14cbcSMatt Macy
1408eda14cbcSMatt Macy if (!vdev_readable(v)) {
14091719886fSMartin Matuska /* Drive not there, can't flush */
14101719886fSMartin Matuska error = SET_ERROR(ENXIO);
14111719886fSMartin Matuska } else if (zfs_nocacheflush) {
14121719886fSMartin Matuska /* Flushing disabled by operator, declare success */
14131719886fSMartin Matuska error = 0;
14141719886fSMartin Matuska } else if (v->vdev_nowritecache) {
14151719886fSMartin Matuska /* This vdev not capable of flushing */
14161719886fSMartin Matuska error = SET_ERROR(ENOTSUP);
14171719886fSMartin Matuska } else {
14181719886fSMartin Matuska /*
14191719886fSMartin Matuska * Issue the flush. If successful, the response will
14201719886fSMartin Matuska * be handled in the completion callback, so we're done.
14211719886fSMartin Matuska */
1422fd45b686SMartin Matuska error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio);
1423eda14cbcSMatt Macy if (error == 0) {
1424eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
1425eda14cbcSMatt Macy return;
1426eda14cbcSMatt Macy }
1427eda14cbcSMatt Macy }
1428eda14cbcSMatt Macy
14291719886fSMartin Matuska /* Couldn't issue the flush, so set the error and return it */
1430eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
14311719886fSMartin Matuska zio->io_error = error;
1432eda14cbcSMatt Macy zio_execute(zio);
1433eda14cbcSMatt Macy return;
1434eda14cbcSMatt Macy
1435eda14cbcSMatt Macy case ZIO_TYPE_TRIM:
14361719886fSMartin Matuska error = vdev_disk_io_trim(zio);
1437eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
14381719886fSMartin Matuska if (error) {
14391719886fSMartin Matuska zio->io_error = error;
14401719886fSMartin Matuska zio_execute(zio);
14411719886fSMartin Matuska }
1442eda14cbcSMatt Macy return;
1443eda14cbcSMatt Macy
1444783d3ff6SMartin Matuska case ZIO_TYPE_READ:
1445783d3ff6SMartin Matuska case ZIO_TYPE_WRITE:
1446783d3ff6SMartin Matuska zio->io_target_timestamp = zio_handle_io_delay(zio);
1447783d3ff6SMartin Matuska error = vdev_disk_io_rw_fn(zio);
1448783d3ff6SMartin Matuska rw_exit(&vd->vd_lock);
1449783d3ff6SMartin Matuska if (error) {
1450783d3ff6SMartin Matuska zio->io_error = error;
1451783d3ff6SMartin Matuska zio_interrupt(zio);
1452783d3ff6SMartin Matuska }
1453783d3ff6SMartin Matuska return;
1454783d3ff6SMartin Matuska
1455eda14cbcSMatt Macy default:
1456783d3ff6SMartin Matuska /*
1457783d3ff6SMartin Matuska * Getting here means our parent vdev has made a very strange
1458783d3ff6SMartin Matuska * request of us, and shouldn't happen. Assert here to force a
1459783d3ff6SMartin Matuska * crash in dev builds, but in production return the IO
1460783d3ff6SMartin Matuska * unhandled. The pool will likely suspend anyway but that's
1461783d3ff6SMartin Matuska * nicer than crashing the kernel.
1462783d3ff6SMartin Matuska */
1463783d3ff6SMartin Matuska ASSERT3S(zio->io_type, ==, -1);
1464783d3ff6SMartin Matuska
1465eda14cbcSMatt Macy rw_exit(&vd->vd_lock);
1466eda14cbcSMatt Macy zio->io_error = SET_ERROR(ENOTSUP);
1467eda14cbcSMatt Macy zio_interrupt(zio);
1468eda14cbcSMatt Macy return;
1469eda14cbcSMatt Macy }
1470eda14cbcSMatt Macy
1471783d3ff6SMartin Matuska __builtin_unreachable();
1472eda14cbcSMatt Macy }
1473eda14cbcSMatt Macy
1474eda14cbcSMatt Macy static void
vdev_disk_io_done(zio_t * zio)1475eda14cbcSMatt Macy vdev_disk_io_done(zio_t *zio)
1476eda14cbcSMatt Macy {
1477eda14cbcSMatt Macy /*
1478eda14cbcSMatt Macy * If the device returned EIO, we revalidate the media. If it is
1479eda14cbcSMatt Macy * determined the media has changed this triggers the asynchronous
1480eda14cbcSMatt Macy * removal of the device from the configuration.
1481eda14cbcSMatt Macy */
1482eda14cbcSMatt Macy if (zio->io_error == EIO) {
1483eda14cbcSMatt Macy vdev_t *v = zio->io_vd;
1484eda14cbcSMatt Macy vdev_disk_t *vd = v->vdev_tsd;
1485eda14cbcSMatt Macy
1486fd45b686SMartin Matuska if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) {
1487fd45b686SMartin Matuska invalidate_bdev(BDH_BDEV(vd->vd_bdh));
1488eda14cbcSMatt Macy v->vdev_remove_wanted = B_TRUE;
1489eda14cbcSMatt Macy spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
1490eda14cbcSMatt Macy }
1491eda14cbcSMatt Macy }
1492eda14cbcSMatt Macy }
1493eda14cbcSMatt Macy
1494eda14cbcSMatt Macy static void
vdev_disk_hold(vdev_t * vd)1495eda14cbcSMatt Macy vdev_disk_hold(vdev_t *vd)
1496eda14cbcSMatt Macy {
1497eda14cbcSMatt Macy ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
1498eda14cbcSMatt Macy
1499eda14cbcSMatt Macy /* We must have a pathname, and it must be absolute. */
1500eda14cbcSMatt Macy if (vd->vdev_path == NULL || vd->vdev_path[0] != '/')
1501eda14cbcSMatt Macy return;
1502eda14cbcSMatt Macy
1503eda14cbcSMatt Macy /*
1504eda14cbcSMatt Macy * Only prefetch path and devid info if the device has
1505eda14cbcSMatt Macy * never been opened.
1506eda14cbcSMatt Macy */
1507eda14cbcSMatt Macy if (vd->vdev_tsd != NULL)
1508eda14cbcSMatt Macy return;
1509eda14cbcSMatt Macy
1510eda14cbcSMatt Macy }
1511eda14cbcSMatt Macy
1512eda14cbcSMatt Macy static void
vdev_disk_rele(vdev_t * vd)1513eda14cbcSMatt Macy vdev_disk_rele(vdev_t *vd)
1514eda14cbcSMatt Macy {
1515eda14cbcSMatt Macy ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
1516eda14cbcSMatt Macy
1517eda14cbcSMatt Macy /* XXX: Implement me as a vnode rele for the device */
1518eda14cbcSMatt Macy }
1519eda14cbcSMatt Macy
1520783d3ff6SMartin Matuska /*
1521783d3ff6SMartin Matuska * BIO submission method. See comment above about vdev_classic.
1522783d3ff6SMartin Matuska * Set zfs_vdev_disk_classic=0 for new, =1 for classic
1523783d3ff6SMartin Matuska */
1524783d3ff6SMartin Matuska static uint_t zfs_vdev_disk_classic = 0; /* default new */
1525783d3ff6SMartin Matuska
1526783d3ff6SMartin Matuska /* Set submission function from module parameter */
1527783d3ff6SMartin Matuska static int
vdev_disk_param_set_classic(const char * buf,zfs_kernel_param_t * kp)1528783d3ff6SMartin Matuska vdev_disk_param_set_classic(const char *buf, zfs_kernel_param_t *kp)
1529783d3ff6SMartin Matuska {
1530783d3ff6SMartin Matuska int err = param_set_uint(buf, kp);
1531783d3ff6SMartin Matuska if (err < 0)
1532783d3ff6SMartin Matuska return (SET_ERROR(err));
1533783d3ff6SMartin Matuska
1534783d3ff6SMartin Matuska vdev_disk_io_rw_fn =
1535783d3ff6SMartin Matuska zfs_vdev_disk_classic ? vdev_classic_physio : vdev_disk_io_rw;
1536783d3ff6SMartin Matuska
1537783d3ff6SMartin Matuska printk(KERN_INFO "ZFS: forcing %s BIO submission\n",
1538783d3ff6SMartin Matuska zfs_vdev_disk_classic ? "classic" : "new");
1539783d3ff6SMartin Matuska
1540783d3ff6SMartin Matuska return (0);
1541783d3ff6SMartin Matuska }
1542783d3ff6SMartin Matuska
1543783d3ff6SMartin Matuska /*
1544783d3ff6SMartin Matuska * At first use vdev use, set the submission function from the default value if
1545783d3ff6SMartin Matuska * it hasn't been set already.
1546783d3ff6SMartin Matuska */
1547783d3ff6SMartin Matuska static int
vdev_disk_init(spa_t * spa,nvlist_t * nv,void ** tsd)1548783d3ff6SMartin Matuska vdev_disk_init(spa_t *spa, nvlist_t *nv, void **tsd)
1549783d3ff6SMartin Matuska {
1550783d3ff6SMartin Matuska (void) spa;
1551783d3ff6SMartin Matuska (void) nv;
1552783d3ff6SMartin Matuska (void) tsd;
1553783d3ff6SMartin Matuska
1554783d3ff6SMartin Matuska if (vdev_disk_io_rw_fn == NULL)
1555783d3ff6SMartin Matuska vdev_disk_io_rw_fn = zfs_vdev_disk_classic ?
1556783d3ff6SMartin Matuska vdev_classic_physio : vdev_disk_io_rw;
1557783d3ff6SMartin Matuska
1558783d3ff6SMartin Matuska return (0);
1559783d3ff6SMartin Matuska }
1560783d3ff6SMartin Matuska
1561eda14cbcSMatt Macy vdev_ops_t vdev_disk_ops = {
1562783d3ff6SMartin Matuska .vdev_op_init = vdev_disk_init,
15637877fdebSMatt Macy .vdev_op_fini = NULL,
1564eda14cbcSMatt Macy .vdev_op_open = vdev_disk_open,
1565eda14cbcSMatt Macy .vdev_op_close = vdev_disk_close,
1566eda14cbcSMatt Macy .vdev_op_asize = vdev_default_asize,
15677877fdebSMatt Macy .vdev_op_min_asize = vdev_default_min_asize,
15687877fdebSMatt Macy .vdev_op_min_alloc = NULL,
1569eda14cbcSMatt Macy .vdev_op_io_start = vdev_disk_io_start,
1570eda14cbcSMatt Macy .vdev_op_io_done = vdev_disk_io_done,
1571eda14cbcSMatt Macy .vdev_op_state_change = NULL,
1572eda14cbcSMatt Macy .vdev_op_need_resilver = NULL,
1573eda14cbcSMatt Macy .vdev_op_hold = vdev_disk_hold,
1574eda14cbcSMatt Macy .vdev_op_rele = vdev_disk_rele,
1575eda14cbcSMatt Macy .vdev_op_remap = NULL,
1576eda14cbcSMatt Macy .vdev_op_xlate = vdev_default_xlate,
15777877fdebSMatt Macy .vdev_op_rebuild_asize = NULL,
15787877fdebSMatt Macy .vdev_op_metaslab_init = NULL,
15797877fdebSMatt Macy .vdev_op_config_generate = NULL,
15807877fdebSMatt Macy .vdev_op_nparity = NULL,
15817877fdebSMatt Macy .vdev_op_ndisks = NULL,
1582eda14cbcSMatt Macy .vdev_op_type = VDEV_TYPE_DISK, /* name of this vdev type */
1583be181ee2SMartin Matuska .vdev_op_leaf = B_TRUE, /* leaf vdev */
1584be181ee2SMartin Matuska .vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post
1585eda14cbcSMatt Macy };
1586eda14cbcSMatt Macy
1587eda14cbcSMatt Macy /*
1588eda14cbcSMatt Macy * The zfs_vdev_scheduler module option has been deprecated. Setting this
1589eda14cbcSMatt Macy * value no longer has any effect. It has not yet been entirely removed
1590eda14cbcSMatt Macy * to allow the module to be loaded if this option is specified in the
1591eda14cbcSMatt Macy * /etc/modprobe.d/zfs.conf file. The following warning will be logged.
1592eda14cbcSMatt Macy */
1593eda14cbcSMatt Macy static int
param_set_vdev_scheduler(const char * val,zfs_kernel_param_t * kp)1594eda14cbcSMatt Macy param_set_vdev_scheduler(const char *val, zfs_kernel_param_t *kp)
1595eda14cbcSMatt Macy {
1596eda14cbcSMatt Macy int error = param_set_charp(val, kp);
1597eda14cbcSMatt Macy if (error == 0) {
1598eda14cbcSMatt Macy printk(KERN_INFO "The 'zfs_vdev_scheduler' module option "
1599eda14cbcSMatt Macy "is not supported.\n");
1600eda14cbcSMatt Macy }
1601eda14cbcSMatt Macy
1602eda14cbcSMatt Macy return (error);
1603eda14cbcSMatt Macy }
1604eda14cbcSMatt Macy
1605e92ffd9bSMartin Matuska static const char *zfs_vdev_scheduler = "unused";
1606eda14cbcSMatt Macy module_param_call(zfs_vdev_scheduler, param_set_vdev_scheduler,
1607eda14cbcSMatt Macy param_get_charp, &zfs_vdev_scheduler, 0644);
1608eda14cbcSMatt Macy MODULE_PARM_DESC(zfs_vdev_scheduler, "I/O scheduler");
1609eda14cbcSMatt Macy
1610eda14cbcSMatt Macy int
param_set_min_auto_ashift(const char * buf,zfs_kernel_param_t * kp)1611eda14cbcSMatt Macy param_set_min_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
1612eda14cbcSMatt Macy {
1613dbd5678dSMartin Matuska uint_t val;
1614eda14cbcSMatt Macy int error;
1615eda14cbcSMatt Macy
1616dbd5678dSMartin Matuska error = kstrtouint(buf, 0, &val);
1617eda14cbcSMatt Macy if (error < 0)
1618eda14cbcSMatt Macy return (SET_ERROR(error));
1619eda14cbcSMatt Macy
1620eda14cbcSMatt Macy if (val < ASHIFT_MIN || val > zfs_vdev_max_auto_ashift)
1621eda14cbcSMatt Macy return (SET_ERROR(-EINVAL));
1622eda14cbcSMatt Macy
1623dbd5678dSMartin Matuska error = param_set_uint(buf, kp);
1624eda14cbcSMatt Macy if (error < 0)
1625eda14cbcSMatt Macy return (SET_ERROR(error));
1626eda14cbcSMatt Macy
1627eda14cbcSMatt Macy return (0);
1628eda14cbcSMatt Macy }
1629eda14cbcSMatt Macy
1630eda14cbcSMatt Macy int
param_set_max_auto_ashift(const char * buf,zfs_kernel_param_t * kp)1631eda14cbcSMatt Macy param_set_max_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
1632eda14cbcSMatt Macy {
1633dbd5678dSMartin Matuska uint_t val;
1634eda14cbcSMatt Macy int error;
1635eda14cbcSMatt Macy
1636dbd5678dSMartin Matuska error = kstrtouint(buf, 0, &val);
1637eda14cbcSMatt Macy if (error < 0)
1638eda14cbcSMatt Macy return (SET_ERROR(error));
1639eda14cbcSMatt Macy
1640eda14cbcSMatt Macy if (val > ASHIFT_MAX || val < zfs_vdev_min_auto_ashift)
1641eda14cbcSMatt Macy return (SET_ERROR(-EINVAL));
1642eda14cbcSMatt Macy
1643dbd5678dSMartin Matuska error = param_set_uint(buf, kp);
1644eda14cbcSMatt Macy if (error < 0)
1645eda14cbcSMatt Macy return (SET_ERROR(error));
1646eda14cbcSMatt Macy
1647eda14cbcSMatt Macy return (0);
1648eda14cbcSMatt Macy }
1649dbd5678dSMartin Matuska
1650dbd5678dSMartin Matuska ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, open_timeout_ms, UINT, ZMOD_RW,
1651dbd5678dSMartin Matuska "Timeout before determining that a device is missing");
1652dbd5678dSMartin Matuska
1653dbd5678dSMartin Matuska ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, failfast_mask, UINT, ZMOD_RW,
1654dbd5678dSMartin Matuska "Defines failfast mask: 1 - device, 2 - transport, 4 - driver");
1655783d3ff6SMartin Matuska
1656783d3ff6SMartin Matuska ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, max_segs, UINT, ZMOD_RW,
1657783d3ff6SMartin Matuska "Maximum number of data segments to add to an IO request (min 4)");
1658783d3ff6SMartin Matuska
1659783d3ff6SMartin Matuska ZFS_MODULE_PARAM_CALL(zfs_vdev_disk, zfs_vdev_disk_, classic,
1660783d3ff6SMartin Matuska vdev_disk_param_set_classic, param_get_uint, ZMOD_RD,
1661783d3ff6SMartin Matuska "Use classic BIO submission method");
1662