1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2010 Red Hat, Inc.
4 * Copyright (C) 2016-2023 Christoph Hellwig.
5 */
6 #include <linux/module.h>
7 #include <linux/compiler.h>
8 #include <linux/fs.h>
9 #include <linux/iomap.h>
10 #include <linux/pagemap.h>
11 #include <linux/uio.h>
12 #include <linux/buffer_head.h>
13 #include <linux/dax.h>
14 #include <linux/writeback.h>
15 #include <linux/list_sort.h>
16 #include <linux/swap.h>
17 #include <linux/bio.h>
18 #include <linux/sched/signal.h>
19 #include <linux/migrate.h>
20 #include "trace.h"
21
22 #include "../internal.h"
23
24 #define IOEND_BATCH_SIZE 4096
25
26 typedef int (*iomap_punch_t)(struct inode *inode, loff_t offset, loff_t length);
27 /*
28 * Structure allocated for each folio to track per-block uptodate, dirty state
29 * and I/O completions.
30 */
31 struct iomap_folio_state {
32 spinlock_t state_lock;
33 unsigned int read_bytes_pending;
34 atomic_t write_bytes_pending;
35
36 /*
37 * Each block has two bits in this bitmap:
38 * Bits [0..blocks_per_folio) has the uptodate status.
39 * Bits [b_p_f...(2*b_p_f)) has the dirty status.
40 */
41 unsigned long state[];
42 };
43
44 static struct bio_set iomap_ioend_bioset;
45
ifs_is_fully_uptodate(struct folio * folio,struct iomap_folio_state * ifs)46 static inline bool ifs_is_fully_uptodate(struct folio *folio,
47 struct iomap_folio_state *ifs)
48 {
49 struct inode *inode = folio->mapping->host;
50
51 return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio));
52 }
53
ifs_block_is_uptodate(struct iomap_folio_state * ifs,unsigned int block)54 static inline bool ifs_block_is_uptodate(struct iomap_folio_state *ifs,
55 unsigned int block)
56 {
57 return test_bit(block, ifs->state);
58 }
59
ifs_set_range_uptodate(struct folio * folio,struct iomap_folio_state * ifs,size_t off,size_t len)60 static bool ifs_set_range_uptodate(struct folio *folio,
61 struct iomap_folio_state *ifs, size_t off, size_t len)
62 {
63 struct inode *inode = folio->mapping->host;
64 unsigned int first_blk = off >> inode->i_blkbits;
65 unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
66 unsigned int nr_blks = last_blk - first_blk + 1;
67
68 bitmap_set(ifs->state, first_blk, nr_blks);
69 return ifs_is_fully_uptodate(folio, ifs);
70 }
71
iomap_set_range_uptodate(struct folio * folio,size_t off,size_t len)72 static void iomap_set_range_uptodate(struct folio *folio, size_t off,
73 size_t len)
74 {
75 struct iomap_folio_state *ifs = folio->private;
76 unsigned long flags;
77 bool uptodate = true;
78
79 if (ifs) {
80 spin_lock_irqsave(&ifs->state_lock, flags);
81 uptodate = ifs_set_range_uptodate(folio, ifs, off, len);
82 spin_unlock_irqrestore(&ifs->state_lock, flags);
83 }
84
85 if (uptodate)
86 folio_mark_uptodate(folio);
87 }
88
ifs_block_is_dirty(struct folio * folio,struct iomap_folio_state * ifs,int block)89 static inline bool ifs_block_is_dirty(struct folio *folio,
90 struct iomap_folio_state *ifs, int block)
91 {
92 struct inode *inode = folio->mapping->host;
93 unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
94
95 return test_bit(block + blks_per_folio, ifs->state);
96 }
97
ifs_find_dirty_range(struct folio * folio,struct iomap_folio_state * ifs,u64 * range_start,u64 range_end)98 static unsigned ifs_find_dirty_range(struct folio *folio,
99 struct iomap_folio_state *ifs, u64 *range_start, u64 range_end)
100 {
101 struct inode *inode = folio->mapping->host;
102 unsigned start_blk =
103 offset_in_folio(folio, *range_start) >> inode->i_blkbits;
104 unsigned end_blk = min_not_zero(
105 offset_in_folio(folio, range_end) >> inode->i_blkbits,
106 i_blocks_per_folio(inode, folio));
107 unsigned nblks = 1;
108
109 while (!ifs_block_is_dirty(folio, ifs, start_blk))
110 if (++start_blk == end_blk)
111 return 0;
112
113 while (start_blk + nblks < end_blk) {
114 if (!ifs_block_is_dirty(folio, ifs, start_blk + nblks))
115 break;
116 nblks++;
117 }
118
119 *range_start = folio_pos(folio) + (start_blk << inode->i_blkbits);
120 return nblks << inode->i_blkbits;
121 }
122
iomap_find_dirty_range(struct folio * folio,u64 * range_start,u64 range_end)123 static unsigned iomap_find_dirty_range(struct folio *folio, u64 *range_start,
124 u64 range_end)
125 {
126 struct iomap_folio_state *ifs = folio->private;
127
128 if (*range_start >= range_end)
129 return 0;
130
131 if (ifs)
132 return ifs_find_dirty_range(folio, ifs, range_start, range_end);
133 return range_end - *range_start;
134 }
135
ifs_clear_range_dirty(struct folio * folio,struct iomap_folio_state * ifs,size_t off,size_t len)136 static void ifs_clear_range_dirty(struct folio *folio,
137 struct iomap_folio_state *ifs, size_t off, size_t len)
138 {
139 struct inode *inode = folio->mapping->host;
140 unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
141 unsigned int first_blk = (off >> inode->i_blkbits);
142 unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
143 unsigned int nr_blks = last_blk - first_blk + 1;
144 unsigned long flags;
145
146 spin_lock_irqsave(&ifs->state_lock, flags);
147 bitmap_clear(ifs->state, first_blk + blks_per_folio, nr_blks);
148 spin_unlock_irqrestore(&ifs->state_lock, flags);
149 }
150
iomap_clear_range_dirty(struct folio * folio,size_t off,size_t len)151 static void iomap_clear_range_dirty(struct folio *folio, size_t off, size_t len)
152 {
153 struct iomap_folio_state *ifs = folio->private;
154
155 if (ifs)
156 ifs_clear_range_dirty(folio, ifs, off, len);
157 }
158
ifs_set_range_dirty(struct folio * folio,struct iomap_folio_state * ifs,size_t off,size_t len)159 static void ifs_set_range_dirty(struct folio *folio,
160 struct iomap_folio_state *ifs, size_t off, size_t len)
161 {
162 struct inode *inode = folio->mapping->host;
163 unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
164 unsigned int first_blk = (off >> inode->i_blkbits);
165 unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
166 unsigned int nr_blks = last_blk - first_blk + 1;
167 unsigned long flags;
168
169 spin_lock_irqsave(&ifs->state_lock, flags);
170 bitmap_set(ifs->state, first_blk + blks_per_folio, nr_blks);
171 spin_unlock_irqrestore(&ifs->state_lock, flags);
172 }
173
iomap_set_range_dirty(struct folio * folio,size_t off,size_t len)174 static void iomap_set_range_dirty(struct folio *folio, size_t off, size_t len)
175 {
176 struct iomap_folio_state *ifs = folio->private;
177
178 if (ifs)
179 ifs_set_range_dirty(folio, ifs, off, len);
180 }
181
ifs_alloc(struct inode * inode,struct folio * folio,unsigned int flags)182 static struct iomap_folio_state *ifs_alloc(struct inode *inode,
183 struct folio *folio, unsigned int flags)
184 {
185 struct iomap_folio_state *ifs = folio->private;
186 unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
187 gfp_t gfp;
188
189 if (ifs || nr_blocks <= 1)
190 return ifs;
191
192 if (flags & IOMAP_NOWAIT)
193 gfp = GFP_NOWAIT;
194 else
195 gfp = GFP_NOFS | __GFP_NOFAIL;
196
197 /*
198 * ifs->state tracks two sets of state flags when the
199 * filesystem block size is smaller than the folio size.
200 * The first state tracks per-block uptodate and the
201 * second tracks per-block dirty state.
202 */
203 ifs = kzalloc(struct_size(ifs, state,
204 BITS_TO_LONGS(2 * nr_blocks)), gfp);
205 if (!ifs)
206 return ifs;
207
208 spin_lock_init(&ifs->state_lock);
209 if (folio_test_uptodate(folio))
210 bitmap_set(ifs->state, 0, nr_blocks);
211 if (folio_test_dirty(folio))
212 bitmap_set(ifs->state, nr_blocks, nr_blocks);
213 folio_attach_private(folio, ifs);
214
215 return ifs;
216 }
217
ifs_free(struct folio * folio)218 static void ifs_free(struct folio *folio)
219 {
220 struct iomap_folio_state *ifs = folio_detach_private(folio);
221
222 if (!ifs)
223 return;
224 WARN_ON_ONCE(ifs->read_bytes_pending != 0);
225 WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending));
226 WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) !=
227 folio_test_uptodate(folio));
228 kfree(ifs);
229 }
230
231 /*
232 * Calculate the range inside the folio that we actually need to read.
233 */
iomap_adjust_read_range(struct inode * inode,struct folio * folio,loff_t * pos,loff_t length,size_t * offp,size_t * lenp)234 static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,
235 loff_t *pos, loff_t length, size_t *offp, size_t *lenp)
236 {
237 struct iomap_folio_state *ifs = folio->private;
238 loff_t orig_pos = *pos;
239 loff_t isize = i_size_read(inode);
240 unsigned block_bits = inode->i_blkbits;
241 unsigned block_size = (1 << block_bits);
242 size_t poff = offset_in_folio(folio, *pos);
243 size_t plen = min_t(loff_t, folio_size(folio) - poff, length);
244 size_t orig_plen = plen;
245 unsigned first = poff >> block_bits;
246 unsigned last = (poff + plen - 1) >> block_bits;
247
248 /*
249 * If the block size is smaller than the page size, we need to check the
250 * per-block uptodate status and adjust the offset and length if needed
251 * to avoid reading in already uptodate ranges.
252 */
253 if (ifs) {
254 unsigned int i;
255
256 /* move forward for each leading block marked uptodate */
257 for (i = first; i <= last; i++) {
258 if (!ifs_block_is_uptodate(ifs, i))
259 break;
260 *pos += block_size;
261 poff += block_size;
262 plen -= block_size;
263 first++;
264 }
265
266 /* truncate len if we find any trailing uptodate block(s) */
267 for ( ; i <= last; i++) {
268 if (ifs_block_is_uptodate(ifs, i)) {
269 plen -= (last - i + 1) * block_size;
270 last = i - 1;
271 break;
272 }
273 }
274 }
275
276 /*
277 * If the extent spans the block that contains the i_size, we need to
278 * handle both halves separately so that we properly zero data in the
279 * page cache for blocks that are entirely outside of i_size.
280 */
281 if (orig_pos <= isize && orig_pos + orig_plen > isize) {
282 unsigned end = offset_in_folio(folio, isize - 1) >> block_bits;
283
284 if (first <= end && last > end)
285 plen -= (last - end) * block_size;
286 }
287
288 *offp = poff;
289 *lenp = plen;
290 }
291
iomap_finish_folio_read(struct folio * folio,size_t off,size_t len,int error)292 static void iomap_finish_folio_read(struct folio *folio, size_t off,
293 size_t len, int error)
294 {
295 struct iomap_folio_state *ifs = folio->private;
296 bool uptodate = !error;
297 bool finished = true;
298
299 if (ifs) {
300 unsigned long flags;
301
302 spin_lock_irqsave(&ifs->state_lock, flags);
303 if (!error)
304 uptodate = ifs_set_range_uptodate(folio, ifs, off, len);
305 ifs->read_bytes_pending -= len;
306 finished = !ifs->read_bytes_pending;
307 spin_unlock_irqrestore(&ifs->state_lock, flags);
308 }
309
310 if (error)
311 folio_set_error(folio);
312 if (finished)
313 folio_end_read(folio, uptodate);
314 }
315
iomap_read_end_io(struct bio * bio)316 static void iomap_read_end_io(struct bio *bio)
317 {
318 int error = blk_status_to_errno(bio->bi_status);
319 struct folio_iter fi;
320
321 bio_for_each_folio_all(fi, bio)
322 iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
323 bio_put(bio);
324 }
325
326 struct iomap_readpage_ctx {
327 struct folio *cur_folio;
328 bool cur_folio_in_bio;
329 struct bio *bio;
330 struct readahead_control *rac;
331 };
332
333 /**
334 * iomap_read_inline_data - copy inline data into the page cache
335 * @iter: iteration structure
336 * @folio: folio to copy to
337 *
338 * Copy the inline data in @iter into @folio and zero out the rest of the folio.
339 * Only a single IOMAP_INLINE extent is allowed at the end of each file.
340 * Returns zero for success to complete the read, or the usual negative errno.
341 */
iomap_read_inline_data(const struct iomap_iter * iter,struct folio * folio)342 static int iomap_read_inline_data(const struct iomap_iter *iter,
343 struct folio *folio)
344 {
345 const struct iomap *iomap = iomap_iter_srcmap(iter);
346 size_t size = i_size_read(iter->inode) - iomap->offset;
347 size_t offset = offset_in_folio(folio, iomap->offset);
348
349 if (folio_test_uptodate(folio))
350 return 0;
351
352 if (WARN_ON_ONCE(size > iomap->length))
353 return -EIO;
354 if (offset > 0)
355 ifs_alloc(iter->inode, folio, iter->flags);
356
357 folio_fill_tail(folio, offset, iomap->inline_data, size);
358 iomap_set_range_uptodate(folio, offset, folio_size(folio) - offset);
359 return 0;
360 }
361
iomap_block_needs_zeroing(const struct iomap_iter * iter,loff_t pos)362 static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter,
363 loff_t pos)
364 {
365 const struct iomap *srcmap = iomap_iter_srcmap(iter);
366
367 return srcmap->type != IOMAP_MAPPED ||
368 (srcmap->flags & IOMAP_F_NEW) ||
369 pos >= i_size_read(iter->inode);
370 }
371
iomap_readpage_iter(const struct iomap_iter * iter,struct iomap_readpage_ctx * ctx,loff_t offset)372 static loff_t iomap_readpage_iter(const struct iomap_iter *iter,
373 struct iomap_readpage_ctx *ctx, loff_t offset)
374 {
375 const struct iomap *iomap = &iter->iomap;
376 loff_t pos = iter->pos + offset;
377 loff_t length = iomap_length(iter) - offset;
378 struct folio *folio = ctx->cur_folio;
379 struct iomap_folio_state *ifs;
380 loff_t orig_pos = pos;
381 size_t poff, plen;
382 sector_t sector;
383
384 if (iomap->type == IOMAP_INLINE)
385 return iomap_read_inline_data(iter, folio);
386
387 /* zero post-eof blocks as the page may be mapped */
388 ifs = ifs_alloc(iter->inode, folio, iter->flags);
389 iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen);
390 if (plen == 0)
391 goto done;
392
393 if (iomap_block_needs_zeroing(iter, pos)) {
394 folio_zero_range(folio, poff, plen);
395 iomap_set_range_uptodate(folio, poff, plen);
396 goto done;
397 }
398
399 ctx->cur_folio_in_bio = true;
400 if (ifs) {
401 spin_lock_irq(&ifs->state_lock);
402 ifs->read_bytes_pending += plen;
403 spin_unlock_irq(&ifs->state_lock);
404 }
405
406 sector = iomap_sector(iomap, pos);
407 if (!ctx->bio ||
408 bio_end_sector(ctx->bio) != sector ||
409 !bio_add_folio(ctx->bio, folio, plen, poff)) {
410 gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
411 gfp_t orig_gfp = gfp;
412 unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
413
414 if (ctx->bio)
415 submit_bio(ctx->bio);
416
417 if (ctx->rac) /* same as readahead_gfp_mask */
418 gfp |= __GFP_NORETRY | __GFP_NOWARN;
419 ctx->bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs),
420 REQ_OP_READ, gfp);
421 /*
422 * If the bio_alloc fails, try it again for a single page to
423 * avoid having to deal with partial page reads. This emulates
424 * what do_mpage_read_folio does.
425 */
426 if (!ctx->bio) {
427 ctx->bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ,
428 orig_gfp);
429 }
430 if (ctx->rac)
431 ctx->bio->bi_opf |= REQ_RAHEAD;
432 ctx->bio->bi_iter.bi_sector = sector;
433 ctx->bio->bi_end_io = iomap_read_end_io;
434 bio_add_folio_nofail(ctx->bio, folio, plen, poff);
435 }
436
437 done:
438 /*
439 * Move the caller beyond our range so that it keeps making progress.
440 * For that, we have to include any leading non-uptodate ranges, but
441 * we can skip trailing ones as they will be handled in the next
442 * iteration.
443 */
444 return pos - orig_pos + plen;
445 }
446
iomap_read_folio(struct folio * folio,const struct iomap_ops * ops)447 int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
448 {
449 struct iomap_iter iter = {
450 .inode = folio->mapping->host,
451 .pos = folio_pos(folio),
452 .len = folio_size(folio),
453 };
454 struct iomap_readpage_ctx ctx = {
455 .cur_folio = folio,
456 };
457 int ret;
458
459 trace_iomap_readpage(iter.inode, 1);
460
461 while ((ret = iomap_iter(&iter, ops)) > 0)
462 iter.processed = iomap_readpage_iter(&iter, &ctx, 0);
463
464 if (ret < 0)
465 folio_set_error(folio);
466
467 if (ctx.bio) {
468 submit_bio(ctx.bio);
469 WARN_ON_ONCE(!ctx.cur_folio_in_bio);
470 } else {
471 WARN_ON_ONCE(ctx.cur_folio_in_bio);
472 folio_unlock(folio);
473 }
474
475 /*
476 * Just like mpage_readahead and block_read_full_folio, we always
477 * return 0 and just set the folio error flag on errors. This
478 * should be cleaned up throughout the stack eventually.
479 */
480 return 0;
481 }
482 EXPORT_SYMBOL_GPL(iomap_read_folio);
483
iomap_readahead_iter(const struct iomap_iter * iter,struct iomap_readpage_ctx * ctx)484 static loff_t iomap_readahead_iter(const struct iomap_iter *iter,
485 struct iomap_readpage_ctx *ctx)
486 {
487 loff_t length = iomap_length(iter);
488 loff_t done, ret;
489
490 for (done = 0; done < length; done += ret) {
491 if (ctx->cur_folio &&
492 offset_in_folio(ctx->cur_folio, iter->pos + done) == 0) {
493 if (!ctx->cur_folio_in_bio)
494 folio_unlock(ctx->cur_folio);
495 ctx->cur_folio = NULL;
496 }
497 if (!ctx->cur_folio) {
498 ctx->cur_folio = readahead_folio(ctx->rac);
499 ctx->cur_folio_in_bio = false;
500 }
501 ret = iomap_readpage_iter(iter, ctx, done);
502 if (ret <= 0)
503 return ret;
504 }
505
506 return done;
507 }
508
509 /**
510 * iomap_readahead - Attempt to read pages from a file.
511 * @rac: Describes the pages to be read.
512 * @ops: The operations vector for the filesystem.
513 *
514 * This function is for filesystems to call to implement their readahead
515 * address_space operation.
516 *
517 * Context: The @ops callbacks may submit I/O (eg to read the addresses of
518 * blocks from disc), and may wait for it. The caller may be trying to
519 * access a different page, and so sleeping excessively should be avoided.
520 * It may allocate memory, but should avoid costly allocations. This
521 * function is called with memalloc_nofs set, so allocations will not cause
522 * the filesystem to be reentered.
523 */
iomap_readahead(struct readahead_control * rac,const struct iomap_ops * ops)524 void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops)
525 {
526 struct iomap_iter iter = {
527 .inode = rac->mapping->host,
528 .pos = readahead_pos(rac),
529 .len = readahead_length(rac),
530 };
531 struct iomap_readpage_ctx ctx = {
532 .rac = rac,
533 };
534
535 trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
536
537 while (iomap_iter(&iter, ops) > 0)
538 iter.processed = iomap_readahead_iter(&iter, &ctx);
539
540 if (ctx.bio)
541 submit_bio(ctx.bio);
542 if (ctx.cur_folio) {
543 if (!ctx.cur_folio_in_bio)
544 folio_unlock(ctx.cur_folio);
545 }
546 }
547 EXPORT_SYMBOL_GPL(iomap_readahead);
548
549 /*
550 * iomap_is_partially_uptodate checks whether blocks within a folio are
551 * uptodate or not.
552 *
553 * Returns true if all blocks which correspond to the specified part
554 * of the folio are uptodate.
555 */
iomap_is_partially_uptodate(struct folio * folio,size_t from,size_t count)556 bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
557 {
558 struct iomap_folio_state *ifs = folio->private;
559 struct inode *inode = folio->mapping->host;
560 unsigned first, last, i;
561
562 if (!ifs)
563 return false;
564
565 /* Caller's range may extend past the end of this folio */
566 count = min(folio_size(folio) - from, count);
567
568 /* First and last blocks in range within folio */
569 first = from >> inode->i_blkbits;
570 last = (from + count - 1) >> inode->i_blkbits;
571
572 for (i = first; i <= last; i++)
573 if (!ifs_block_is_uptodate(ifs, i))
574 return false;
575 return true;
576 }
577 EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
578
579 /**
580 * iomap_get_folio - get a folio reference for writing
581 * @iter: iteration structure
582 * @pos: start offset of write
583 * @len: Suggested size of folio to create.
584 *
585 * Returns a locked reference to the folio at @pos, or an error pointer if the
586 * folio could not be obtained.
587 */
iomap_get_folio(struct iomap_iter * iter,loff_t pos,size_t len)588 struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)
589 {
590 fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS;
591
592 if (iter->flags & IOMAP_NOWAIT)
593 fgp |= FGP_NOWAIT;
594 fgp |= fgf_set_order(len);
595
596 return __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT,
597 fgp, mapping_gfp_mask(iter->inode->i_mapping));
598 }
599 EXPORT_SYMBOL_GPL(iomap_get_folio);
600
iomap_release_folio(struct folio * folio,gfp_t gfp_flags)601 bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags)
602 {
603 trace_iomap_release_folio(folio->mapping->host, folio_pos(folio),
604 folio_size(folio));
605
606 /*
607 * If the folio is dirty, we refuse to release our metadata because
608 * it may be partially dirty. Once we track per-block dirty state,
609 * we can release the metadata if every block is dirty.
610 */
611 if (folio_test_dirty(folio))
612 return false;
613 ifs_free(folio);
614 return true;
615 }
616 EXPORT_SYMBOL_GPL(iomap_release_folio);
617
iomap_invalidate_folio(struct folio * folio,size_t offset,size_t len)618 void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len)
619 {
620 trace_iomap_invalidate_folio(folio->mapping->host,
621 folio_pos(folio) + offset, len);
622
623 /*
624 * If we're invalidating the entire folio, clear the dirty state
625 * from it and release it to avoid unnecessary buildup of the LRU.
626 */
627 if (offset == 0 && len == folio_size(folio)) {
628 WARN_ON_ONCE(folio_test_writeback(folio));
629 folio_cancel_dirty(folio);
630 ifs_free(folio);
631 }
632 }
633 EXPORT_SYMBOL_GPL(iomap_invalidate_folio);
634
iomap_dirty_folio(struct address_space * mapping,struct folio * folio)635 bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio)
636 {
637 struct inode *inode = mapping->host;
638 size_t len = folio_size(folio);
639
640 ifs_alloc(inode, folio, 0);
641 iomap_set_range_dirty(folio, 0, len);
642 return filemap_dirty_folio(mapping, folio);
643 }
644 EXPORT_SYMBOL_GPL(iomap_dirty_folio);
645
646 static void
iomap_write_failed(struct inode * inode,loff_t pos,unsigned len)647 iomap_write_failed(struct inode *inode, loff_t pos, unsigned len)
648 {
649 loff_t i_size = i_size_read(inode);
650
651 /*
652 * Only truncate newly allocated pages beyoned EOF, even if the
653 * write started inside the existing inode size.
654 */
655 if (pos + len > i_size)
656 truncate_pagecache_range(inode, max(pos, i_size),
657 pos + len - 1);
658 }
659
iomap_read_folio_sync(loff_t block_start,struct folio * folio,size_t poff,size_t plen,const struct iomap * iomap)660 static int iomap_read_folio_sync(loff_t block_start, struct folio *folio,
661 size_t poff, size_t plen, const struct iomap *iomap)
662 {
663 struct bio_vec bvec;
664 struct bio bio;
665
666 bio_init(&bio, iomap->bdev, &bvec, 1, REQ_OP_READ);
667 bio.bi_iter.bi_sector = iomap_sector(iomap, block_start);
668 bio_add_folio_nofail(&bio, folio, plen, poff);
669 return submit_bio_wait(&bio);
670 }
671
__iomap_write_begin(const struct iomap_iter * iter,loff_t pos,size_t len,struct folio * folio)672 static int __iomap_write_begin(const struct iomap_iter *iter, loff_t pos,
673 size_t len, struct folio *folio)
674 {
675 const struct iomap *srcmap = iomap_iter_srcmap(iter);
676 struct iomap_folio_state *ifs;
677 loff_t block_size = i_blocksize(iter->inode);
678 loff_t block_start = round_down(pos, block_size);
679 loff_t block_end = round_up(pos + len, block_size);
680 unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio);
681 size_t from = offset_in_folio(folio, pos), to = from + len;
682 size_t poff, plen;
683
684 /*
685 * If the write or zeroing completely overlaps the current folio, then
686 * entire folio will be dirtied so there is no need for
687 * per-block state tracking structures to be attached to this folio.
688 * For the unshare case, we must read in the ondisk contents because we
689 * are not changing pagecache contents.
690 */
691 if (!(iter->flags & IOMAP_UNSHARE) && pos <= folio_pos(folio) &&
692 pos + len >= folio_pos(folio) + folio_size(folio))
693 return 0;
694
695 ifs = ifs_alloc(iter->inode, folio, iter->flags);
696 if ((iter->flags & IOMAP_NOWAIT) && !ifs && nr_blocks > 1)
697 return -EAGAIN;
698
699 if (folio_test_uptodate(folio))
700 return 0;
701 folio_clear_error(folio);
702
703 do {
704 iomap_adjust_read_range(iter->inode, folio, &block_start,
705 block_end - block_start, &poff, &plen);
706 if (plen == 0)
707 break;
708
709 if (!(iter->flags & IOMAP_UNSHARE) &&
710 (from <= poff || from >= poff + plen) &&
711 (to <= poff || to >= poff + plen))
712 continue;
713
714 if (iomap_block_needs_zeroing(iter, block_start)) {
715 if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE))
716 return -EIO;
717 folio_zero_segments(folio, poff, from, to, poff + plen);
718 } else {
719 int status;
720
721 if (iter->flags & IOMAP_NOWAIT)
722 return -EAGAIN;
723
724 status = iomap_read_folio_sync(block_start, folio,
725 poff, plen, srcmap);
726 if (status)
727 return status;
728 }
729 iomap_set_range_uptodate(folio, poff, plen);
730 } while ((block_start += plen) < block_end);
731
732 return 0;
733 }
734
__iomap_get_folio(struct iomap_iter * iter,loff_t pos,size_t len)735 static struct folio *__iomap_get_folio(struct iomap_iter *iter, loff_t pos,
736 size_t len)
737 {
738 const struct iomap_folio_ops *folio_ops = iter->iomap.folio_ops;
739
740 if (folio_ops && folio_ops->get_folio)
741 return folio_ops->get_folio(iter, pos, len);
742 else
743 return iomap_get_folio(iter, pos, len);
744 }
745
__iomap_put_folio(struct iomap_iter * iter,loff_t pos,size_t ret,struct folio * folio)746 static void __iomap_put_folio(struct iomap_iter *iter, loff_t pos, size_t ret,
747 struct folio *folio)
748 {
749 const struct iomap_folio_ops *folio_ops = iter->iomap.folio_ops;
750
751 if (folio_ops && folio_ops->put_folio) {
752 folio_ops->put_folio(iter->inode, pos, ret, folio);
753 } else {
754 folio_unlock(folio);
755 folio_put(folio);
756 }
757 }
758
iomap_write_begin_inline(const struct iomap_iter * iter,struct folio * folio)759 static int iomap_write_begin_inline(const struct iomap_iter *iter,
760 struct folio *folio)
761 {
762 /* needs more work for the tailpacking case; disable for now */
763 if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0))
764 return -EIO;
765 return iomap_read_inline_data(iter, folio);
766 }
767
iomap_write_begin(struct iomap_iter * iter,loff_t pos,size_t len,struct folio ** foliop)768 static int iomap_write_begin(struct iomap_iter *iter, loff_t pos,
769 size_t len, struct folio **foliop)
770 {
771 const struct iomap_folio_ops *folio_ops = iter->iomap.folio_ops;
772 const struct iomap *srcmap = iomap_iter_srcmap(iter);
773 struct folio *folio;
774 int status = 0;
775
776 BUG_ON(pos + len > iter->iomap.offset + iter->iomap.length);
777 if (srcmap != &iter->iomap)
778 BUG_ON(pos + len > srcmap->offset + srcmap->length);
779
780 if (fatal_signal_pending(current))
781 return -EINTR;
782
783 if (!mapping_large_folio_support(iter->inode->i_mapping))
784 len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos));
785
786 folio = __iomap_get_folio(iter, pos, len);
787 if (IS_ERR(folio))
788 return PTR_ERR(folio);
789
790 /*
791 * Now we have a locked folio, before we do anything with it we need to
792 * check that the iomap we have cached is not stale. The inode extent
793 * mapping can change due to concurrent IO in flight (e.g.
794 * IOMAP_UNWRITTEN state can change and memory reclaim could have
795 * reclaimed a previously partially written page at this index after IO
796 * completion before this write reaches this file offset) and hence we
797 * could do the wrong thing here (zero a page range incorrectly or fail
798 * to zero) and corrupt data.
799 */
800 if (folio_ops && folio_ops->iomap_valid) {
801 bool iomap_valid = folio_ops->iomap_valid(iter->inode,
802 &iter->iomap);
803 if (!iomap_valid) {
804 iter->iomap.flags |= IOMAP_F_STALE;
805 status = 0;
806 goto out_unlock;
807 }
808 }
809
810 if (pos + len > folio_pos(folio) + folio_size(folio))
811 len = folio_pos(folio) + folio_size(folio) - pos;
812
813 if (srcmap->type == IOMAP_INLINE)
814 status = iomap_write_begin_inline(iter, folio);
815 else if (srcmap->flags & IOMAP_F_BUFFER_HEAD)
816 status = __block_write_begin_int(folio, pos, len, NULL, srcmap);
817 else
818 status = __iomap_write_begin(iter, pos, len, folio);
819
820 if (unlikely(status))
821 goto out_unlock;
822
823 *foliop = folio;
824 return 0;
825
826 out_unlock:
827 __iomap_put_folio(iter, pos, 0, folio);
828
829 return status;
830 }
831
__iomap_write_end(struct inode * inode,loff_t pos,size_t len,size_t copied,struct folio * folio)832 static bool __iomap_write_end(struct inode *inode, loff_t pos, size_t len,
833 size_t copied, struct folio *folio)
834 {
835 flush_dcache_folio(folio);
836
837 /*
838 * The blocks that were entirely written will now be uptodate, so we
839 * don't have to worry about a read_folio reading them and overwriting a
840 * partial write. However, if we've encountered a short write and only
841 * partially written into a block, it will not be marked uptodate, so a
842 * read_folio might come in and destroy our partial write.
843 *
844 * Do the simplest thing and just treat any short write to a
845 * non-uptodate page as a zero-length write, and force the caller to
846 * redo the whole thing.
847 */
848 if (unlikely(copied < len && !folio_test_uptodate(folio)))
849 return false;
850 iomap_set_range_uptodate(folio, offset_in_folio(folio, pos), len);
851 iomap_set_range_dirty(folio, offset_in_folio(folio, pos), copied);
852 filemap_dirty_folio(inode->i_mapping, folio);
853 return true;
854 }
855
iomap_write_end_inline(const struct iomap_iter * iter,struct folio * folio,loff_t pos,size_t copied)856 static void iomap_write_end_inline(const struct iomap_iter *iter,
857 struct folio *folio, loff_t pos, size_t copied)
858 {
859 const struct iomap *iomap = &iter->iomap;
860 void *addr;
861
862 WARN_ON_ONCE(!folio_test_uptodate(folio));
863 BUG_ON(!iomap_inline_data_valid(iomap));
864
865 flush_dcache_folio(folio);
866 addr = kmap_local_folio(folio, pos);
867 memcpy(iomap_inline_data(iomap, pos), addr, copied);
868 kunmap_local(addr);
869
870 mark_inode_dirty(iter->inode);
871 }
872
873 /*
874 * Returns true if all copied bytes have been written to the pagecache,
875 * otherwise return false.
876 */
iomap_write_end(struct iomap_iter * iter,loff_t pos,size_t len,size_t copied,struct folio * folio)877 static bool iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len,
878 size_t copied, struct folio *folio)
879 {
880 const struct iomap *srcmap = iomap_iter_srcmap(iter);
881 loff_t old_size = iter->inode->i_size;
882 size_t written;
883
884 if (srcmap->type == IOMAP_INLINE) {
885 iomap_write_end_inline(iter, folio, pos, copied);
886 written = copied;
887 } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) {
888 written = block_write_end(NULL, iter->inode->i_mapping, pos,
889 len, copied, &folio->page, NULL);
890 WARN_ON_ONCE(written != copied && written != 0);
891 } else {
892 written = __iomap_write_end(iter->inode, pos, len, copied,
893 folio) ? copied : 0;
894 }
895
896 /*
897 * Update the in-memory inode size after copying the data into the page
898 * cache. It's up to the file system to write the updated size to disk,
899 * preferably after I/O completion so that no stale data is exposed.
900 * Only once that's done can we unlock and release the folio.
901 */
902 if (pos + written > old_size) {
903 i_size_write(iter->inode, pos + written);
904 iter->iomap.flags |= IOMAP_F_SIZE_CHANGED;
905 }
906 __iomap_put_folio(iter, pos, written, folio);
907
908 if (old_size < pos)
909 pagecache_isize_extended(iter->inode, old_size, pos);
910
911 return written == copied;
912 }
913
iomap_write_iter(struct iomap_iter * iter,struct iov_iter * i)914 static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
915 {
916 loff_t length = iomap_length(iter);
917 loff_t pos = iter->pos;
918 ssize_t total_written = 0;
919 long status = 0;
920 struct address_space *mapping = iter->inode->i_mapping;
921 size_t chunk = mapping_max_folio_size(mapping);
922 unsigned int bdp_flags = (iter->flags & IOMAP_NOWAIT) ? BDP_ASYNC : 0;
923
924 do {
925 struct folio *folio;
926 size_t offset; /* Offset into folio */
927 size_t bytes; /* Bytes to write to folio */
928 size_t copied; /* Bytes copied from user */
929 size_t written; /* Bytes have been written */
930
931 bytes = iov_iter_count(i);
932 retry:
933 offset = pos & (chunk - 1);
934 bytes = min(chunk - offset, bytes);
935 status = balance_dirty_pages_ratelimited_flags(mapping,
936 bdp_flags);
937 if (unlikely(status))
938 break;
939
940 if (bytes > length)
941 bytes = length;
942
943 /*
944 * Bring in the user page that we'll copy from _first_.
945 * Otherwise there's a nasty deadlock on copying from the
946 * same page as we're writing to, without it being marked
947 * up-to-date.
948 *
949 * For async buffered writes the assumption is that the user
950 * page has already been faulted in. This can be optimized by
951 * faulting the user page.
952 */
953 if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) {
954 status = -EFAULT;
955 break;
956 }
957
958 status = iomap_write_begin(iter, pos, bytes, &folio);
959 if (unlikely(status)) {
960 iomap_write_failed(iter->inode, pos, bytes);
961 break;
962 }
963 if (iter->iomap.flags & IOMAP_F_STALE)
964 break;
965
966 offset = offset_in_folio(folio, pos);
967 if (bytes > folio_size(folio) - offset)
968 bytes = folio_size(folio) - offset;
969
970 if (mapping_writably_mapped(mapping))
971 flush_dcache_folio(folio);
972
973 copied = copy_folio_from_iter_atomic(folio, offset, bytes, i);
974 written = iomap_write_end(iter, pos, bytes, copied, folio) ?
975 copied : 0;
976
977 cond_resched();
978 if (unlikely(written == 0)) {
979 /*
980 * A short copy made iomap_write_end() reject the
981 * thing entirely. Might be memory poisoning
982 * halfway through, might be a race with munmap,
983 * might be severe memory pressure.
984 */
985 iomap_write_failed(iter->inode, pos, bytes);
986 iov_iter_revert(i, copied);
987
988 if (chunk > PAGE_SIZE)
989 chunk /= 2;
990 if (copied) {
991 bytes = copied;
992 goto retry;
993 }
994 } else {
995 pos += written;
996 total_written += written;
997 length -= written;
998 }
999 } while (iov_iter_count(i) && length);
1000
1001 if (status == -EAGAIN) {
1002 iov_iter_revert(i, total_written);
1003 return -EAGAIN;
1004 }
1005 return total_written ? total_written : status;
1006 }
1007
1008 ssize_t
iomap_file_buffered_write(struct kiocb * iocb,struct iov_iter * i,const struct iomap_ops * ops)1009 iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
1010 const struct iomap_ops *ops)
1011 {
1012 struct iomap_iter iter = {
1013 .inode = iocb->ki_filp->f_mapping->host,
1014 .pos = iocb->ki_pos,
1015 .len = iov_iter_count(i),
1016 .flags = IOMAP_WRITE,
1017 };
1018 ssize_t ret;
1019
1020 if (iocb->ki_flags & IOCB_NOWAIT)
1021 iter.flags |= IOMAP_NOWAIT;
1022
1023 while ((ret = iomap_iter(&iter, ops)) > 0)
1024 iter.processed = iomap_write_iter(&iter, i);
1025
1026 if (unlikely(iter.pos == iocb->ki_pos))
1027 return ret;
1028 ret = iter.pos - iocb->ki_pos;
1029 iocb->ki_pos = iter.pos;
1030 return ret;
1031 }
1032 EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
1033
iomap_write_delalloc_ifs_punch(struct inode * inode,struct folio * folio,loff_t start_byte,loff_t end_byte,iomap_punch_t punch)1034 static int iomap_write_delalloc_ifs_punch(struct inode *inode,
1035 struct folio *folio, loff_t start_byte, loff_t end_byte,
1036 iomap_punch_t punch)
1037 {
1038 unsigned int first_blk, last_blk, i;
1039 loff_t last_byte;
1040 u8 blkbits = inode->i_blkbits;
1041 struct iomap_folio_state *ifs;
1042 int ret = 0;
1043
1044 /*
1045 * When we have per-block dirty tracking, there can be
1046 * blocks within a folio which are marked uptodate
1047 * but not dirty. In that case it is necessary to punch
1048 * out such blocks to avoid leaking any delalloc blocks.
1049 */
1050 ifs = folio->private;
1051 if (!ifs)
1052 return ret;
1053
1054 last_byte = min_t(loff_t, end_byte - 1,
1055 folio_pos(folio) + folio_size(folio) - 1);
1056 first_blk = offset_in_folio(folio, start_byte) >> blkbits;
1057 last_blk = offset_in_folio(folio, last_byte) >> blkbits;
1058 for (i = first_blk; i <= last_blk; i++) {
1059 if (!ifs_block_is_dirty(folio, ifs, i)) {
1060 ret = punch(inode, folio_pos(folio) + (i << blkbits),
1061 1 << blkbits);
1062 if (ret)
1063 return ret;
1064 }
1065 }
1066
1067 return ret;
1068 }
1069
1070
iomap_write_delalloc_punch(struct inode * inode,struct folio * folio,loff_t * punch_start_byte,loff_t start_byte,loff_t end_byte,iomap_punch_t punch)1071 static int iomap_write_delalloc_punch(struct inode *inode, struct folio *folio,
1072 loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte,
1073 iomap_punch_t punch)
1074 {
1075 int ret = 0;
1076
1077 if (!folio_test_dirty(folio))
1078 return ret;
1079
1080 /* if dirty, punch up to offset */
1081 if (start_byte > *punch_start_byte) {
1082 ret = punch(inode, *punch_start_byte,
1083 start_byte - *punch_start_byte);
1084 if (ret)
1085 return ret;
1086 }
1087
1088 /* Punch non-dirty blocks within folio */
1089 ret = iomap_write_delalloc_ifs_punch(inode, folio, start_byte,
1090 end_byte, punch);
1091 if (ret)
1092 return ret;
1093
1094 /*
1095 * Make sure the next punch start is correctly bound to
1096 * the end of this data range, not the end of the folio.
1097 */
1098 *punch_start_byte = min_t(loff_t, end_byte,
1099 folio_pos(folio) + folio_size(folio));
1100
1101 return ret;
1102 }
1103
1104 /*
1105 * Scan the data range passed to us for dirty page cache folios. If we find a
1106 * dirty folio, punch out the preceding range and update the offset from which
1107 * the next punch will start from.
1108 *
1109 * We can punch out storage reservations under clean pages because they either
1110 * contain data that has been written back - in which case the delalloc punch
1111 * over that range is a no-op - or they have been read faults in which case they
1112 * contain zeroes and we can remove the delalloc backing range and any new
1113 * writes to those pages will do the normal hole filling operation...
1114 *
1115 * This makes the logic simple: we only need to keep the delalloc extents only
1116 * over the dirty ranges of the page cache.
1117 *
1118 * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1119 * simplify range iterations.
1120 */
iomap_write_delalloc_scan(struct inode * inode,loff_t * punch_start_byte,loff_t start_byte,loff_t end_byte,iomap_punch_t punch)1121 static int iomap_write_delalloc_scan(struct inode *inode,
1122 loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte,
1123 iomap_punch_t punch)
1124 {
1125 while (start_byte < end_byte) {
1126 struct folio *folio;
1127 int ret;
1128
1129 /* grab locked page */
1130 folio = filemap_lock_folio(inode->i_mapping,
1131 start_byte >> PAGE_SHIFT);
1132 if (IS_ERR(folio)) {
1133 start_byte = ALIGN_DOWN(start_byte, PAGE_SIZE) +
1134 PAGE_SIZE;
1135 continue;
1136 }
1137
1138 ret = iomap_write_delalloc_punch(inode, folio, punch_start_byte,
1139 start_byte, end_byte, punch);
1140 if (ret) {
1141 folio_unlock(folio);
1142 folio_put(folio);
1143 return ret;
1144 }
1145
1146 /* move offset to start of next folio in range */
1147 start_byte = folio_next_index(folio) << PAGE_SHIFT;
1148 folio_unlock(folio);
1149 folio_put(folio);
1150 }
1151 return 0;
1152 }
1153
1154 /*
1155 * Punch out all the delalloc blocks in the range given except for those that
1156 * have dirty data still pending in the page cache - those are going to be
1157 * written and so must still retain the delalloc backing for writeback.
1158 *
1159 * As we are scanning the page cache for data, we don't need to reimplement the
1160 * wheel - mapping_seek_hole_data() does exactly what we need to identify the
1161 * start and end of data ranges correctly even for sub-folio block sizes. This
1162 * byte range based iteration is especially convenient because it means we
1163 * don't have to care about variable size folios, nor where the start or end of
1164 * the data range lies within a folio, if they lie within the same folio or even
1165 * if there are multiple discontiguous data ranges within the folio.
1166 *
1167 * It should be noted that mapping_seek_hole_data() is not aware of EOF, and so
1168 * can return data ranges that exist in the cache beyond EOF. e.g. a page fault
1169 * spanning EOF will initialise the post-EOF data to zeroes and mark it up to
1170 * date. A write page fault can then mark it dirty. If we then fail a write()
1171 * beyond EOF into that up to date cached range, we allocate a delalloc block
1172 * beyond EOF and then have to punch it out. Because the range is up to date,
1173 * mapping_seek_hole_data() will return it, and we will skip the punch because
1174 * the folio is dirty. THis is incorrect - we always need to punch out delalloc
1175 * beyond EOF in this case as writeback will never write back and covert that
1176 * delalloc block beyond EOF. Hence we limit the cached data scan range to EOF,
1177 * resulting in always punching out the range from the EOF to the end of the
1178 * range the iomap spans.
1179 *
1180 * Intervals are of the form [start_byte, end_byte) (i.e. open ended) because it
1181 * matches the intervals returned by mapping_seek_hole_data(). i.e. SEEK_DATA
1182 * returns the start of a data range (start_byte), and SEEK_HOLE(start_byte)
1183 * returns the end of the data range (data_end). Using closed intervals would
1184 * require sprinkling this code with magic "+ 1" and "- 1" arithmetic and expose
1185 * the code to subtle off-by-one bugs....
1186 */
iomap_write_delalloc_release(struct inode * inode,loff_t start_byte,loff_t end_byte,iomap_punch_t punch)1187 static int iomap_write_delalloc_release(struct inode *inode,
1188 loff_t start_byte, loff_t end_byte, iomap_punch_t punch)
1189 {
1190 loff_t punch_start_byte = start_byte;
1191 loff_t scan_end_byte = min(i_size_read(inode), end_byte);
1192 int error = 0;
1193
1194 /*
1195 * Lock the mapping to avoid races with page faults re-instantiating
1196 * folios and dirtying them via ->page_mkwrite whilst we walk the
1197 * cache and perform delalloc extent removal. Failing to do this can
1198 * leave dirty pages with no space reservation in the cache.
1199 */
1200 filemap_invalidate_lock(inode->i_mapping);
1201 while (start_byte < scan_end_byte) {
1202 loff_t data_end;
1203
1204 start_byte = mapping_seek_hole_data(inode->i_mapping,
1205 start_byte, scan_end_byte, SEEK_DATA);
1206 /*
1207 * If there is no more data to scan, all that is left is to
1208 * punch out the remaining range.
1209 */
1210 if (start_byte == -ENXIO || start_byte == scan_end_byte)
1211 break;
1212 if (start_byte < 0) {
1213 error = start_byte;
1214 goto out_unlock;
1215 }
1216 WARN_ON_ONCE(start_byte < punch_start_byte);
1217 WARN_ON_ONCE(start_byte > scan_end_byte);
1218
1219 /*
1220 * We find the end of this contiguous cached data range by
1221 * seeking from start_byte to the beginning of the next hole.
1222 */
1223 data_end = mapping_seek_hole_data(inode->i_mapping, start_byte,
1224 scan_end_byte, SEEK_HOLE);
1225 if (data_end < 0) {
1226 error = data_end;
1227 goto out_unlock;
1228 }
1229 WARN_ON_ONCE(data_end <= start_byte);
1230 WARN_ON_ONCE(data_end > scan_end_byte);
1231
1232 error = iomap_write_delalloc_scan(inode, &punch_start_byte,
1233 start_byte, data_end, punch);
1234 if (error)
1235 goto out_unlock;
1236
1237 /* The next data search starts at the end of this one. */
1238 start_byte = data_end;
1239 }
1240
1241 if (punch_start_byte < end_byte)
1242 error = punch(inode, punch_start_byte,
1243 end_byte - punch_start_byte);
1244 out_unlock:
1245 filemap_invalidate_unlock(inode->i_mapping);
1246 return error;
1247 }
1248
1249 /*
1250 * When a short write occurs, the filesystem may need to remove reserved space
1251 * that was allocated in ->iomap_begin from it's ->iomap_end method. For
1252 * filesystems that use delayed allocation, we need to punch out delalloc
1253 * extents from the range that are not dirty in the page cache. As the write can
1254 * race with page faults, there can be dirty pages over the delalloc extent
1255 * outside the range of a short write but still within the delalloc extent
1256 * allocated for this iomap.
1257 *
1258 * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1259 * simplify range iterations.
1260 *
1261 * The punch() callback *must* only punch delalloc extents in the range passed
1262 * to it. It must skip over all other types of extents in the range and leave
1263 * them completely unchanged. It must do this punch atomically with respect to
1264 * other extent modifications.
1265 *
1266 * The punch() callback may be called with a folio locked to prevent writeback
1267 * extent allocation racing at the edge of the range we are currently punching.
1268 * The locked folio may or may not cover the range being punched, so it is not
1269 * safe for the punch() callback to lock folios itself.
1270 *
1271 * Lock order is:
1272 *
1273 * inode->i_rwsem (shared or exclusive)
1274 * inode->i_mapping->invalidate_lock (exclusive)
1275 * folio_lock()
1276 * ->punch
1277 * internal filesystem allocation lock
1278 */
iomap_file_buffered_write_punch_delalloc(struct inode * inode,struct iomap * iomap,loff_t pos,loff_t length,ssize_t written,iomap_punch_t punch)1279 int iomap_file_buffered_write_punch_delalloc(struct inode *inode,
1280 struct iomap *iomap, loff_t pos, loff_t length,
1281 ssize_t written, iomap_punch_t punch)
1282 {
1283 loff_t start_byte;
1284 loff_t end_byte;
1285 unsigned int blocksize = i_blocksize(inode);
1286
1287 if (iomap->type != IOMAP_DELALLOC)
1288 return 0;
1289
1290 /* If we didn't reserve the blocks, we're not allowed to punch them. */
1291 if (!(iomap->flags & IOMAP_F_NEW))
1292 return 0;
1293
1294 /*
1295 * start_byte refers to the first unused block after a short write. If
1296 * nothing was written, round offset down to point at the first block in
1297 * the range.
1298 */
1299 if (unlikely(!written))
1300 start_byte = round_down(pos, blocksize);
1301 else
1302 start_byte = round_up(pos + written, blocksize);
1303 end_byte = round_up(pos + length, blocksize);
1304
1305 /* Nothing to do if we've written the entire delalloc extent */
1306 if (start_byte >= end_byte)
1307 return 0;
1308
1309 return iomap_write_delalloc_release(inode, start_byte, end_byte,
1310 punch);
1311 }
1312 EXPORT_SYMBOL_GPL(iomap_file_buffered_write_punch_delalloc);
1313
iomap_unshare_iter(struct iomap_iter * iter)1314 static loff_t iomap_unshare_iter(struct iomap_iter *iter)
1315 {
1316 struct iomap *iomap = &iter->iomap;
1317 const struct iomap *srcmap = iomap_iter_srcmap(iter);
1318 loff_t pos = iter->pos;
1319 loff_t length = iomap_length(iter);
1320 loff_t written = 0;
1321
1322 /* don't bother with blocks that are not shared to start with */
1323 if (!(iomap->flags & IOMAP_F_SHARED))
1324 return length;
1325 /* don't bother with holes or unwritten extents */
1326 if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
1327 return length;
1328
1329 do {
1330 struct folio *folio;
1331 int status;
1332 size_t offset;
1333 size_t bytes = min_t(u64, SIZE_MAX, length);
1334 bool ret;
1335
1336 status = iomap_write_begin(iter, pos, bytes, &folio);
1337 if (unlikely(status))
1338 return status;
1339 if (iomap->flags & IOMAP_F_STALE)
1340 break;
1341
1342 offset = offset_in_folio(folio, pos);
1343 if (bytes > folio_size(folio) - offset)
1344 bytes = folio_size(folio) - offset;
1345
1346 ret = iomap_write_end(iter, pos, bytes, bytes, folio);
1347 if (WARN_ON_ONCE(!ret))
1348 return -EIO;
1349
1350 cond_resched();
1351
1352 pos += bytes;
1353 written += bytes;
1354 length -= bytes;
1355
1356 balance_dirty_pages_ratelimited(iter->inode->i_mapping);
1357 } while (length > 0);
1358
1359 return written;
1360 }
1361
1362 int
iomap_file_unshare(struct inode * inode,loff_t pos,loff_t len,const struct iomap_ops * ops)1363 iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
1364 const struct iomap_ops *ops)
1365 {
1366 struct iomap_iter iter = {
1367 .inode = inode,
1368 .pos = pos,
1369 .len = len,
1370 .flags = IOMAP_WRITE | IOMAP_UNSHARE,
1371 };
1372 int ret;
1373
1374 while ((ret = iomap_iter(&iter, ops)) > 0)
1375 iter.processed = iomap_unshare_iter(&iter);
1376 return ret;
1377 }
1378 EXPORT_SYMBOL_GPL(iomap_file_unshare);
1379
iomap_zero_iter(struct iomap_iter * iter,bool * did_zero)1380 static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
1381 {
1382 const struct iomap *srcmap = iomap_iter_srcmap(iter);
1383 loff_t pos = iter->pos;
1384 loff_t length = iomap_length(iter);
1385 loff_t written = 0;
1386
1387 /* already zeroed? we're done. */
1388 if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
1389 return length;
1390
1391 do {
1392 struct folio *folio;
1393 int status;
1394 size_t offset;
1395 size_t bytes = min_t(u64, SIZE_MAX, length);
1396 bool ret;
1397
1398 status = iomap_write_begin(iter, pos, bytes, &folio);
1399 if (status)
1400 return status;
1401 if (iter->iomap.flags & IOMAP_F_STALE)
1402 break;
1403
1404 offset = offset_in_folio(folio, pos);
1405 if (bytes > folio_size(folio) - offset)
1406 bytes = folio_size(folio) - offset;
1407
1408 folio_zero_range(folio, offset, bytes);
1409 folio_mark_accessed(folio);
1410
1411 ret = iomap_write_end(iter, pos, bytes, bytes, folio);
1412 if (WARN_ON_ONCE(!ret))
1413 return -EIO;
1414
1415 pos += bytes;
1416 length -= bytes;
1417 written += bytes;
1418 } while (length > 0);
1419
1420 if (did_zero)
1421 *did_zero = true;
1422 return written;
1423 }
1424
1425 int
iomap_zero_range(struct inode * inode,loff_t pos,loff_t len,bool * did_zero,const struct iomap_ops * ops)1426 iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
1427 const struct iomap_ops *ops)
1428 {
1429 struct iomap_iter iter = {
1430 .inode = inode,
1431 .pos = pos,
1432 .len = len,
1433 .flags = IOMAP_ZERO,
1434 };
1435 int ret;
1436
1437 while ((ret = iomap_iter(&iter, ops)) > 0)
1438 iter.processed = iomap_zero_iter(&iter, did_zero);
1439 return ret;
1440 }
1441 EXPORT_SYMBOL_GPL(iomap_zero_range);
1442
1443 int
iomap_truncate_page(struct inode * inode,loff_t pos,bool * did_zero,const struct iomap_ops * ops)1444 iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
1445 const struct iomap_ops *ops)
1446 {
1447 unsigned int blocksize = i_blocksize(inode);
1448 unsigned int off = pos & (blocksize - 1);
1449
1450 /* Block boundary? Nothing to do */
1451 if (!off)
1452 return 0;
1453 return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops);
1454 }
1455 EXPORT_SYMBOL_GPL(iomap_truncate_page);
1456
iomap_folio_mkwrite_iter(struct iomap_iter * iter,struct folio * folio)1457 static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter,
1458 struct folio *folio)
1459 {
1460 loff_t length = iomap_length(iter);
1461 int ret;
1462
1463 if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) {
1464 ret = __block_write_begin_int(folio, iter->pos, length, NULL,
1465 &iter->iomap);
1466 if (ret)
1467 return ret;
1468 block_commit_write(&folio->page, 0, length);
1469 } else {
1470 WARN_ON_ONCE(!folio_test_uptodate(folio));
1471 folio_mark_dirty(folio);
1472 }
1473
1474 return length;
1475 }
1476
iomap_page_mkwrite(struct vm_fault * vmf,const struct iomap_ops * ops)1477 vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
1478 {
1479 struct iomap_iter iter = {
1480 .inode = file_inode(vmf->vma->vm_file),
1481 .flags = IOMAP_WRITE | IOMAP_FAULT,
1482 };
1483 struct folio *folio = page_folio(vmf->page);
1484 ssize_t ret;
1485
1486 folio_lock(folio);
1487 ret = folio_mkwrite_check_truncate(folio, iter.inode);
1488 if (ret < 0)
1489 goto out_unlock;
1490 iter.pos = folio_pos(folio);
1491 iter.len = ret;
1492 while ((ret = iomap_iter(&iter, ops)) > 0)
1493 iter.processed = iomap_folio_mkwrite_iter(&iter, folio);
1494
1495 if (ret < 0)
1496 goto out_unlock;
1497 folio_wait_stable(folio);
1498 return VM_FAULT_LOCKED;
1499 out_unlock:
1500 folio_unlock(folio);
1501 return vmf_fs_error(ret);
1502 }
1503 EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1504
iomap_finish_folio_write(struct inode * inode,struct folio * folio,size_t len)1505 static void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
1506 size_t len)
1507 {
1508 struct iomap_folio_state *ifs = folio->private;
1509
1510 WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);
1511 WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) <= 0);
1512
1513 if (!ifs || atomic_sub_and_test(len, &ifs->write_bytes_pending))
1514 folio_end_writeback(folio);
1515 }
1516
1517 /*
1518 * We're now finished for good with this ioend structure. Update the page
1519 * state, release holds on bios, and finally free up memory. Do not use the
1520 * ioend after this.
1521 */
1522 static u32
iomap_finish_ioend(struct iomap_ioend * ioend,int error)1523 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
1524 {
1525 struct inode *inode = ioend->io_inode;
1526 struct bio *bio = &ioend->io_bio;
1527 struct folio_iter fi;
1528 u32 folio_count = 0;
1529
1530 if (error) {
1531 mapping_set_error(inode->i_mapping, error);
1532 if (!bio_flagged(bio, BIO_QUIET)) {
1533 pr_err_ratelimited(
1534 "%s: writeback error on inode %lu, offset %lld, sector %llu",
1535 inode->i_sb->s_id, inode->i_ino,
1536 ioend->io_offset, ioend->io_sector);
1537 }
1538 }
1539
1540 /* walk all folios in bio, ending page IO on them */
1541 bio_for_each_folio_all(fi, bio) {
1542 if (error)
1543 folio_set_error(fi.folio);
1544 iomap_finish_folio_write(inode, fi.folio, fi.length);
1545 folio_count++;
1546 }
1547
1548 bio_put(bio); /* frees the ioend */
1549 return folio_count;
1550 }
1551
1552 /*
1553 * Ioend completion routine for merged bios. This can only be called from task
1554 * contexts as merged ioends can be of unbound length. Hence we have to break up
1555 * the writeback completions into manageable chunks to avoid long scheduler
1556 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
1557 * good batch processing throughput without creating adverse scheduler latency
1558 * conditions.
1559 */
1560 void
iomap_finish_ioends(struct iomap_ioend * ioend,int error)1561 iomap_finish_ioends(struct iomap_ioend *ioend, int error)
1562 {
1563 struct list_head tmp;
1564 u32 completions;
1565
1566 might_sleep();
1567
1568 list_replace_init(&ioend->io_list, &tmp);
1569 completions = iomap_finish_ioend(ioend, error);
1570
1571 while (!list_empty(&tmp)) {
1572 if (completions > IOEND_BATCH_SIZE * 8) {
1573 cond_resched();
1574 completions = 0;
1575 }
1576 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
1577 list_del_init(&ioend->io_list);
1578 completions += iomap_finish_ioend(ioend, error);
1579 }
1580 }
1581 EXPORT_SYMBOL_GPL(iomap_finish_ioends);
1582
1583 /*
1584 * We can merge two adjacent ioends if they have the same set of work to do.
1585 */
1586 static bool
iomap_ioend_can_merge(struct iomap_ioend * ioend,struct iomap_ioend * next)1587 iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next)
1588 {
1589 if (ioend->io_bio.bi_status != next->io_bio.bi_status)
1590 return false;
1591 if ((ioend->io_flags & IOMAP_F_SHARED) ^
1592 (next->io_flags & IOMAP_F_SHARED))
1593 return false;
1594 if ((ioend->io_type == IOMAP_UNWRITTEN) ^
1595 (next->io_type == IOMAP_UNWRITTEN))
1596 return false;
1597 if (ioend->io_offset + ioend->io_size != next->io_offset)
1598 return false;
1599 /*
1600 * Do not merge physically discontiguous ioends. The filesystem
1601 * completion functions will have to iterate the physical
1602 * discontiguities even if we merge the ioends at a logical level, so
1603 * we don't gain anything by merging physical discontiguities here.
1604 *
1605 * We cannot use bio->bi_iter.bi_sector here as it is modified during
1606 * submission so does not point to the start sector of the bio at
1607 * completion.
1608 */
1609 if (ioend->io_sector + (ioend->io_size >> 9) != next->io_sector)
1610 return false;
1611 return true;
1612 }
1613
1614 void
iomap_ioend_try_merge(struct iomap_ioend * ioend,struct list_head * more_ioends)1615 iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends)
1616 {
1617 struct iomap_ioend *next;
1618
1619 INIT_LIST_HEAD(&ioend->io_list);
1620
1621 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
1622 io_list))) {
1623 if (!iomap_ioend_can_merge(ioend, next))
1624 break;
1625 list_move_tail(&next->io_list, &ioend->io_list);
1626 ioend->io_size += next->io_size;
1627 }
1628 }
1629 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
1630
1631 static int
iomap_ioend_compare(void * priv,const struct list_head * a,const struct list_head * b)1632 iomap_ioend_compare(void *priv, const struct list_head *a,
1633 const struct list_head *b)
1634 {
1635 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
1636 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
1637
1638 if (ia->io_offset < ib->io_offset)
1639 return -1;
1640 if (ia->io_offset > ib->io_offset)
1641 return 1;
1642 return 0;
1643 }
1644
1645 void
iomap_sort_ioends(struct list_head * ioend_list)1646 iomap_sort_ioends(struct list_head *ioend_list)
1647 {
1648 list_sort(NULL, ioend_list, iomap_ioend_compare);
1649 }
1650 EXPORT_SYMBOL_GPL(iomap_sort_ioends);
1651
iomap_writepage_end_bio(struct bio * bio)1652 static void iomap_writepage_end_bio(struct bio *bio)
1653 {
1654 iomap_finish_ioend(iomap_ioend_from_bio(bio),
1655 blk_status_to_errno(bio->bi_status));
1656 }
1657
1658 /*
1659 * Submit the final bio for an ioend.
1660 *
1661 * If @error is non-zero, it means that we have a situation where some part of
1662 * the submission process has failed after we've marked pages for writeback.
1663 * We cannot cancel ioend directly in that case, so call the bio end I/O handler
1664 * with the error status here to run the normal I/O completion handler to clear
1665 * the writeback bit and let the file system proess the errors.
1666 */
iomap_submit_ioend(struct iomap_writepage_ctx * wpc,int error)1667 static int iomap_submit_ioend(struct iomap_writepage_ctx *wpc, int error)
1668 {
1669 if (!wpc->ioend)
1670 return error;
1671
1672 /*
1673 * Let the file systems prepare the I/O submission and hook in an I/O
1674 * comletion handler. This also needs to happen in case after a
1675 * failure happened so that the file system end I/O handler gets called
1676 * to clean up.
1677 */
1678 if (wpc->ops->prepare_ioend)
1679 error = wpc->ops->prepare_ioend(wpc->ioend, error);
1680
1681 if (error) {
1682 wpc->ioend->io_bio.bi_status = errno_to_blk_status(error);
1683 bio_endio(&wpc->ioend->io_bio);
1684 } else {
1685 submit_bio(&wpc->ioend->io_bio);
1686 }
1687
1688 wpc->ioend = NULL;
1689 return error;
1690 }
1691
iomap_alloc_ioend(struct iomap_writepage_ctx * wpc,struct writeback_control * wbc,struct inode * inode,loff_t pos)1692 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
1693 struct writeback_control *wbc, struct inode *inode, loff_t pos)
1694 {
1695 struct iomap_ioend *ioend;
1696 struct bio *bio;
1697
1698 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
1699 REQ_OP_WRITE | wbc_to_write_flags(wbc),
1700 GFP_NOFS, &iomap_ioend_bioset);
1701 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
1702 bio->bi_end_io = iomap_writepage_end_bio;
1703 wbc_init_bio(wbc, bio);
1704 bio->bi_write_hint = inode->i_write_hint;
1705
1706 ioend = iomap_ioend_from_bio(bio);
1707 INIT_LIST_HEAD(&ioend->io_list);
1708 ioend->io_type = wpc->iomap.type;
1709 ioend->io_flags = wpc->iomap.flags;
1710 ioend->io_inode = inode;
1711 ioend->io_size = 0;
1712 ioend->io_offset = pos;
1713 ioend->io_sector = bio->bi_iter.bi_sector;
1714
1715 wpc->nr_folios = 0;
1716 return ioend;
1717 }
1718
iomap_can_add_to_ioend(struct iomap_writepage_ctx * wpc,loff_t pos)1719 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos)
1720 {
1721 if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
1722 (wpc->ioend->io_flags & IOMAP_F_SHARED))
1723 return false;
1724 if (wpc->iomap.type != wpc->ioend->io_type)
1725 return false;
1726 if (pos != wpc->ioend->io_offset + wpc->ioend->io_size)
1727 return false;
1728 if (iomap_sector(&wpc->iomap, pos) !=
1729 bio_end_sector(&wpc->ioend->io_bio))
1730 return false;
1731 /*
1732 * Limit ioend bio chain lengths to minimise IO completion latency. This
1733 * also prevents long tight loops ending page writeback on all the
1734 * folios in the ioend.
1735 */
1736 if (wpc->nr_folios >= IOEND_BATCH_SIZE)
1737 return false;
1738 return true;
1739 }
1740
1741 /*
1742 * Test to see if we have an existing ioend structure that we could append to
1743 * first; otherwise finish off the current ioend and start another.
1744 *
1745 * If a new ioend is created and cached, the old ioend is submitted to the block
1746 * layer instantly. Batching optimisations are provided by higher level block
1747 * plugging.
1748 *
1749 * At the end of a writeback pass, there will be a cached ioend remaining on the
1750 * writepage context that the caller will need to submit.
1751 */
iomap_add_to_ioend(struct iomap_writepage_ctx * wpc,struct writeback_control * wbc,struct folio * folio,struct inode * inode,loff_t pos,unsigned len)1752 static int iomap_add_to_ioend(struct iomap_writepage_ctx *wpc,
1753 struct writeback_control *wbc, struct folio *folio,
1754 struct inode *inode, loff_t pos, unsigned len)
1755 {
1756 struct iomap_folio_state *ifs = folio->private;
1757 size_t poff = offset_in_folio(folio, pos);
1758 int error;
1759
1760 if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos)) {
1761 new_ioend:
1762 error = iomap_submit_ioend(wpc, 0);
1763 if (error)
1764 return error;
1765 wpc->ioend = iomap_alloc_ioend(wpc, wbc, inode, pos);
1766 }
1767
1768 if (!bio_add_folio(&wpc->ioend->io_bio, folio, len, poff))
1769 goto new_ioend;
1770
1771 if (ifs)
1772 atomic_add(len, &ifs->write_bytes_pending);
1773 wpc->ioend->io_size += len;
1774 wbc_account_cgroup_owner(wbc, &folio->page, len);
1775 return 0;
1776 }
1777
iomap_writepage_map_blocks(struct iomap_writepage_ctx * wpc,struct writeback_control * wbc,struct folio * folio,struct inode * inode,u64 pos,unsigned dirty_len,unsigned * count)1778 static int iomap_writepage_map_blocks(struct iomap_writepage_ctx *wpc,
1779 struct writeback_control *wbc, struct folio *folio,
1780 struct inode *inode, u64 pos, unsigned dirty_len,
1781 unsigned *count)
1782 {
1783 int error;
1784
1785 do {
1786 unsigned map_len;
1787
1788 error = wpc->ops->map_blocks(wpc, inode, pos, dirty_len);
1789 if (error)
1790 break;
1791 trace_iomap_writepage_map(inode, pos, dirty_len, &wpc->iomap);
1792
1793 map_len = min_t(u64, dirty_len,
1794 wpc->iomap.offset + wpc->iomap.length - pos);
1795 WARN_ON_ONCE(!folio->private && map_len < dirty_len);
1796
1797 switch (wpc->iomap.type) {
1798 case IOMAP_INLINE:
1799 WARN_ON_ONCE(1);
1800 error = -EIO;
1801 break;
1802 case IOMAP_HOLE:
1803 break;
1804 default:
1805 error = iomap_add_to_ioend(wpc, wbc, folio, inode, pos,
1806 map_len);
1807 if (!error)
1808 (*count)++;
1809 break;
1810 }
1811 dirty_len -= map_len;
1812 pos += map_len;
1813 } while (dirty_len && !error);
1814
1815 /*
1816 * We cannot cancel the ioend directly here on error. We may have
1817 * already set other pages under writeback and hence we have to run I/O
1818 * completion to mark the error state of the pages under writeback
1819 * appropriately.
1820 *
1821 * Just let the file system know what portion of the folio failed to
1822 * map.
1823 */
1824 if (error && wpc->ops->discard_folio)
1825 wpc->ops->discard_folio(folio, pos);
1826 return error;
1827 }
1828
1829 /*
1830 * Check interaction of the folio with the file end.
1831 *
1832 * If the folio is entirely beyond i_size, return false. If it straddles
1833 * i_size, adjust end_pos and zero all data beyond i_size.
1834 */
iomap_writepage_handle_eof(struct folio * folio,struct inode * inode,u64 * end_pos)1835 static bool iomap_writepage_handle_eof(struct folio *folio, struct inode *inode,
1836 u64 *end_pos)
1837 {
1838 u64 isize = i_size_read(inode);
1839
1840 if (*end_pos > isize) {
1841 size_t poff = offset_in_folio(folio, isize);
1842 pgoff_t end_index = isize >> PAGE_SHIFT;
1843
1844 /*
1845 * If the folio is entirely ouside of i_size, skip it.
1846 *
1847 * This can happen due to a truncate operation that is in
1848 * progress and in that case truncate will finish it off once
1849 * we've dropped the folio lock.
1850 *
1851 * Note that the pgoff_t used for end_index is an unsigned long.
1852 * If the given offset is greater than 16TB on a 32-bit system,
1853 * then if we checked if the folio is fully outside i_size with
1854 * "if (folio->index >= end_index + 1)", "end_index + 1" would
1855 * overflow and evaluate to 0. Hence this folio would be
1856 * redirtied and written out repeatedly, which would result in
1857 * an infinite loop; the user program performing this operation
1858 * would hang. Instead, we can detect this situation by
1859 * checking if the folio is totally beyond i_size or if its
1860 * offset is just equal to the EOF.
1861 */
1862 if (folio->index > end_index ||
1863 (folio->index == end_index && poff == 0))
1864 return false;
1865
1866 /*
1867 * The folio straddles i_size.
1868 *
1869 * It must be zeroed out on each and every writepage invocation
1870 * because it may be mmapped:
1871 *
1872 * A file is mapped in multiples of the page size. For a
1873 * file that is not a multiple of the page size, the
1874 * remaining memory is zeroed when mapped, and writes to that
1875 * region are not written out to the file.
1876 *
1877 * Also adjust the writeback range to skip all blocks entirely
1878 * beyond i_size.
1879 */
1880 folio_zero_segment(folio, poff, folio_size(folio));
1881 *end_pos = round_up(isize, i_blocksize(inode));
1882 }
1883
1884 return true;
1885 }
1886
iomap_writepage_map(struct iomap_writepage_ctx * wpc,struct writeback_control * wbc,struct folio * folio)1887 static int iomap_writepage_map(struct iomap_writepage_ctx *wpc,
1888 struct writeback_control *wbc, struct folio *folio)
1889 {
1890 struct iomap_folio_state *ifs = folio->private;
1891 struct inode *inode = folio->mapping->host;
1892 u64 pos = folio_pos(folio);
1893 u64 end_pos = pos + folio_size(folio);
1894 unsigned count = 0;
1895 int error = 0;
1896 u32 rlen;
1897
1898 WARN_ON_ONCE(!folio_test_locked(folio));
1899 WARN_ON_ONCE(folio_test_dirty(folio));
1900 WARN_ON_ONCE(folio_test_writeback(folio));
1901
1902 trace_iomap_writepage(inode, pos, folio_size(folio));
1903
1904 if (!iomap_writepage_handle_eof(folio, inode, &end_pos)) {
1905 folio_unlock(folio);
1906 return 0;
1907 }
1908 WARN_ON_ONCE(end_pos <= pos);
1909
1910 if (i_blocks_per_folio(inode, folio) > 1) {
1911 if (!ifs) {
1912 ifs = ifs_alloc(inode, folio, 0);
1913 iomap_set_range_dirty(folio, 0, end_pos - pos);
1914 }
1915
1916 /*
1917 * Keep the I/O completion handler from clearing the writeback
1918 * bit until we have submitted all blocks by adding a bias to
1919 * ifs->write_bytes_pending, which is dropped after submitting
1920 * all blocks.
1921 */
1922 WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending) != 0);
1923 atomic_inc(&ifs->write_bytes_pending);
1924 }
1925
1926 /*
1927 * Set the writeback bit ASAP, as the I/O completion for the single
1928 * block per folio case happen hit as soon as we're submitting the bio.
1929 */
1930 folio_start_writeback(folio);
1931
1932 /*
1933 * Walk through the folio to find dirty areas to write back.
1934 */
1935 while ((rlen = iomap_find_dirty_range(folio, &pos, end_pos))) {
1936 error = iomap_writepage_map_blocks(wpc, wbc, folio, inode,
1937 pos, rlen, &count);
1938 if (error)
1939 break;
1940 pos += rlen;
1941 }
1942
1943 if (count)
1944 wpc->nr_folios++;
1945
1946 /*
1947 * We can have dirty bits set past end of file in page_mkwrite path
1948 * while mapping the last partial folio. Hence it's better to clear
1949 * all the dirty bits in the folio here.
1950 */
1951 iomap_clear_range_dirty(folio, 0, folio_size(folio));
1952
1953 /*
1954 * Usually the writeback bit is cleared by the I/O completion handler.
1955 * But we may end up either not actually writing any blocks, or (when
1956 * there are multiple blocks in a folio) all I/O might have finished
1957 * already at this point. In that case we need to clear the writeback
1958 * bit ourselves right after unlocking the page.
1959 */
1960 folio_unlock(folio);
1961 if (ifs) {
1962 if (atomic_dec_and_test(&ifs->write_bytes_pending))
1963 folio_end_writeback(folio);
1964 } else {
1965 if (!count)
1966 folio_end_writeback(folio);
1967 }
1968 mapping_set_error(inode->i_mapping, error);
1969 return error;
1970 }
1971
1972 int
iomap_writepages(struct address_space * mapping,struct writeback_control * wbc,struct iomap_writepage_ctx * wpc,const struct iomap_writeback_ops * ops)1973 iomap_writepages(struct address_space *mapping, struct writeback_control *wbc,
1974 struct iomap_writepage_ctx *wpc,
1975 const struct iomap_writeback_ops *ops)
1976 {
1977 struct folio *folio = NULL;
1978 int error;
1979
1980 /*
1981 * Writeback from reclaim context should never happen except in the case
1982 * of a VM regression so warn about it and refuse to write the data.
1983 */
1984 if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC | PF_KSWAPD)) ==
1985 PF_MEMALLOC))
1986 return -EIO;
1987
1988 wpc->ops = ops;
1989 while ((folio = writeback_iter(mapping, wbc, folio, &error)))
1990 error = iomap_writepage_map(wpc, wbc, folio);
1991 return iomap_submit_ioend(wpc, error);
1992 }
1993 EXPORT_SYMBOL_GPL(iomap_writepages);
1994
iomap_init(void)1995 static int __init iomap_init(void)
1996 {
1997 return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
1998 offsetof(struct iomap_ioend, io_bio),
1999 BIOSET_NEED_BVECS);
2000 }
2001 fs_initcall(iomap_init);
2002