1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * mm/readahead.c - address_space-level file readahead. 4 * 5 * Copyright (C) 2002, Linus Torvalds 6 * 7 * 09Apr2002 Andrew Morton 8 * Initial version. 9 */ 10 11 /** 12 * DOC: Readahead Overview 13 * 14 * Readahead is used to read content into the page cache before it is 15 * explicitly requested by the application. Readahead only ever 16 * attempts to read folios that are not yet in the page cache. If a 17 * folio is present but not up-to-date, readahead will not try to read 18 * it. In that case a simple ->readpage() will be requested. 19 * 20 * Readahead is triggered when an application read request (whether a 21 * system call or a page fault) finds that the requested folio is not in 22 * the page cache, or that it is in the page cache and has the 23 * readahead flag set. This flag indicates that the folio was read 24 * as part of a previous readahead request and now that it has been 25 * accessed, it is time for the next readahead. 26 * 27 * Each readahead request is partly synchronous read, and partly async 28 * readahead. This is reflected in the struct file_ra_state which 29 * contains ->size being the total number of pages, and ->async_size 30 * which is the number of pages in the async section. The readahead 31 * flag will be set on the first folio in this async section to trigger 32 * a subsequent readahead. Once a series of sequential reads has been 33 * established, there should be no need for a synchronous component and 34 * all readahead request will be fully asynchronous. 35 * 36 * When either of the triggers causes a readahead, three numbers need 37 * to be determined: the start of the region to read, the size of the 38 * region, and the size of the async tail. 39 * 40 * The start of the region is simply the first page address at or after 41 * the accessed address, which is not currently populated in the page 42 * cache. This is found with a simple search in the page cache. 43 * 44 * The size of the async tail is determined by subtracting the size that 45 * was explicitly requested from the determined request size, unless 46 * this would be less than zero - then zero is used. NOTE THIS 47 * CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED 48 * PAGE. ALSO THIS CALCULATION IS NOT USED CONSISTENTLY. 49 * 50 * The size of the region is normally determined from the size of the 51 * previous readahead which loaded the preceding pages. This may be 52 * discovered from the struct file_ra_state for simple sequential reads, 53 * or from examining the state of the page cache when multiple 54 * sequential reads are interleaved. Specifically: where the readahead 55 * was triggered by the readahead flag, the size of the previous 56 * readahead is assumed to be the number of pages from the triggering 57 * page to the start of the new readahead. In these cases, the size of 58 * the previous readahead is scaled, often doubled, for the new 59 * readahead, though see get_next_ra_size() for details. 60 * 61 * If the size of the previous read cannot be determined, the number of 62 * preceding pages in the page cache is used to estimate the size of 63 * a previous read. This estimate could easily be misled by random 64 * reads being coincidentally adjacent, so it is ignored unless it is 65 * larger than the current request, and it is not scaled up, unless it 66 * is at the start of file. 67 * 68 * In general readahead is accelerated at the start of the file, as 69 * reads from there are often sequential. There are other minor 70 * adjustments to the readahead size in various special cases and these 71 * are best discovered by reading the code. 72 * 73 * The above calculation, based on the previous readahead size, 74 * determines the size of the readahead, to which any requested read 75 * size may be added. 76 * 77 * Readahead requests are sent to the filesystem using the ->readahead() 78 * address space operation, for which mpage_readahead() is a canonical 79 * implementation. ->readahead() should normally initiate reads on all 80 * folios, but may fail to read any or all folios without causing an I/O 81 * error. The page cache reading code will issue a ->readpage() request 82 * for any folio which ->readahead() did not read, and only an error 83 * from this will be final. 84 * 85 * ->readahead() will generally call readahead_folio() repeatedly to get 86 * each folio from those prepared for readahead. It may fail to read a 87 * folio by: 88 * 89 * * not calling readahead_folio() sufficiently many times, effectively 90 * ignoring some folios, as might be appropriate if the path to 91 * storage is congested. 92 * 93 * * failing to actually submit a read request for a given folio, 94 * possibly due to insufficient resources, or 95 * 96 * * getting an error during subsequent processing of a request. 97 * 98 * In the last two cases, the folio should be unlocked by the filesystem 99 * to indicate that the read attempt has failed. In the first case the 100 * folio will be unlocked by the VFS. 101 * 102 * Those folios not in the final ``async_size`` of the request should be 103 * considered to be important and ->readahead() should not fail them due 104 * to congestion or temporary resource unavailability, but should wait 105 * for necessary resources (e.g. memory or indexing information) to 106 * become available. Folios in the final ``async_size`` may be 107 * considered less urgent and failure to read them is more acceptable. 108 * In this case it is best to use filemap_remove_folio() to remove the 109 * folios from the page cache as is automatically done for folios that 110 * were not fetched with readahead_folio(). This will allow a 111 * subsequent synchronous readahead request to try them again. If they 112 * are left in the page cache, then they will be read individually using 113 * ->readpage() which may be less efficient. 114 */ 115 116 #include <linux/blkdev.h> 117 #include <linux/kernel.h> 118 #include <linux/dax.h> 119 #include <linux/gfp.h> 120 #include <linux/export.h> 121 #include <linux/backing-dev.h> 122 #include <linux/task_io_accounting_ops.h> 123 #include <linux/pagevec.h> 124 #include <linux/pagemap.h> 125 #include <linux/syscalls.h> 126 #include <linux/file.h> 127 #include <linux/mm_inline.h> 128 #include <linux/blk-cgroup.h> 129 #include <linux/fadvise.h> 130 #include <linux/sched/mm.h> 131 132 #include "internal.h" 133 134 /* 135 * Initialise a struct file's readahead state. Assumes that the caller has 136 * memset *ra to zero. 137 */ 138 void 139 file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping) 140 { 141 ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages; 142 ra->prev_pos = -1; 143 } 144 EXPORT_SYMBOL_GPL(file_ra_state_init); 145 146 static void read_pages(struct readahead_control *rac) 147 { 148 const struct address_space_operations *aops = rac->mapping->a_ops; 149 struct page *page; 150 struct blk_plug plug; 151 152 if (!readahead_count(rac)) 153 return; 154 155 blk_start_plug(&plug); 156 157 if (aops->readahead) { 158 aops->readahead(rac); 159 /* 160 * Clean up the remaining pages. The sizes in ->ra 161 * may be used to size the next readahead, so make sure 162 * they accurately reflect what happened. 163 */ 164 while ((page = readahead_page(rac))) { 165 rac->ra->size -= 1; 166 if (rac->ra->async_size > 0) { 167 rac->ra->async_size -= 1; 168 delete_from_page_cache(page); 169 } 170 unlock_page(page); 171 put_page(page); 172 } 173 } else { 174 while ((page = readahead_page(rac))) { 175 aops->readpage(rac->file, page); 176 put_page(page); 177 } 178 } 179 180 blk_finish_plug(&plug); 181 182 BUG_ON(readahead_count(rac)); 183 } 184 185 /** 186 * page_cache_ra_unbounded - Start unchecked readahead. 187 * @ractl: Readahead control. 188 * @nr_to_read: The number of pages to read. 189 * @lookahead_size: Where to start the next readahead. 190 * 191 * This function is for filesystems to call when they want to start 192 * readahead beyond a file's stated i_size. This is almost certainly 193 * not the function you want to call. Use page_cache_async_readahead() 194 * or page_cache_sync_readahead() instead. 195 * 196 * Context: File is referenced by caller. Mutexes may be held by caller. 197 * May sleep, but will not reenter filesystem to reclaim memory. 198 */ 199 void page_cache_ra_unbounded(struct readahead_control *ractl, 200 unsigned long nr_to_read, unsigned long lookahead_size) 201 { 202 struct address_space *mapping = ractl->mapping; 203 unsigned long index = readahead_index(ractl); 204 gfp_t gfp_mask = readahead_gfp_mask(mapping); 205 unsigned long i; 206 207 /* 208 * Partway through the readahead operation, we will have added 209 * locked pages to the page cache, but will not yet have submitted 210 * them for I/O. Adding another page may need to allocate memory, 211 * which can trigger memory reclaim. Telling the VM we're in 212 * the middle of a filesystem operation will cause it to not 213 * touch file-backed pages, preventing a deadlock. Most (all?) 214 * filesystems already specify __GFP_NOFS in their mapping's 215 * gfp_mask, but let's be explicit here. 216 */ 217 unsigned int nofs = memalloc_nofs_save(); 218 219 filemap_invalidate_lock_shared(mapping); 220 /* 221 * Preallocate as many pages as we will need. 222 */ 223 for (i = 0; i < nr_to_read; i++) { 224 struct folio *folio = xa_load(&mapping->i_pages, index + i); 225 226 if (folio && !xa_is_value(folio)) { 227 /* 228 * Page already present? Kick off the current batch 229 * of contiguous pages before continuing with the 230 * next batch. This page may be the one we would 231 * have intended to mark as Readahead, but we don't 232 * have a stable reference to this page, and it's 233 * not worth getting one just for that. 234 */ 235 read_pages(ractl); 236 ractl->_index++; 237 i = ractl->_index + ractl->_nr_pages - index - 1; 238 continue; 239 } 240 241 folio = filemap_alloc_folio(gfp_mask, 0); 242 if (!folio) 243 break; 244 if (filemap_add_folio(mapping, folio, index + i, 245 gfp_mask) < 0) { 246 folio_put(folio); 247 read_pages(ractl); 248 ractl->_index++; 249 i = ractl->_index + ractl->_nr_pages - index - 1; 250 continue; 251 } 252 if (i == nr_to_read - lookahead_size) 253 folio_set_readahead(folio); 254 ractl->_nr_pages++; 255 } 256 257 /* 258 * Now start the IO. We ignore I/O errors - if the page is not 259 * uptodate then the caller will launch readpage again, and 260 * will then handle the error. 261 */ 262 read_pages(ractl); 263 filemap_invalidate_unlock_shared(mapping); 264 memalloc_nofs_restore(nofs); 265 } 266 EXPORT_SYMBOL_GPL(page_cache_ra_unbounded); 267 268 /* 269 * do_page_cache_ra() actually reads a chunk of disk. It allocates 270 * the pages first, then submits them for I/O. This avoids the very bad 271 * behaviour which would occur if page allocations are causing VM writeback. 272 * We really don't want to intermingle reads and writes like that. 273 */ 274 static void do_page_cache_ra(struct readahead_control *ractl, 275 unsigned long nr_to_read, unsigned long lookahead_size) 276 { 277 struct inode *inode = ractl->mapping->host; 278 unsigned long index = readahead_index(ractl); 279 loff_t isize = i_size_read(inode); 280 pgoff_t end_index; /* The last page we want to read */ 281 282 if (isize == 0) 283 return; 284 285 end_index = (isize - 1) >> PAGE_SHIFT; 286 if (index > end_index) 287 return; 288 /* Don't read past the page containing the last byte of the file */ 289 if (nr_to_read > end_index - index) 290 nr_to_read = end_index - index + 1; 291 292 page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size); 293 } 294 295 /* 296 * Chunk the readahead into 2 megabyte units, so that we don't pin too much 297 * memory at once. 298 */ 299 void force_page_cache_ra(struct readahead_control *ractl, 300 unsigned long nr_to_read) 301 { 302 struct address_space *mapping = ractl->mapping; 303 struct file_ra_state *ra = ractl->ra; 304 struct backing_dev_info *bdi = inode_to_bdi(mapping->host); 305 unsigned long max_pages, index; 306 307 if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readahead)) 308 return; 309 310 /* 311 * If the request exceeds the readahead window, allow the read to 312 * be up to the optimal hardware IO size 313 */ 314 index = readahead_index(ractl); 315 max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages); 316 nr_to_read = min_t(unsigned long, nr_to_read, max_pages); 317 while (nr_to_read) { 318 unsigned long this_chunk = (2 * 1024 * 1024) / PAGE_SIZE; 319 320 if (this_chunk > nr_to_read) 321 this_chunk = nr_to_read; 322 ractl->_index = index; 323 do_page_cache_ra(ractl, this_chunk, 0); 324 325 index += this_chunk; 326 nr_to_read -= this_chunk; 327 } 328 } 329 330 /* 331 * Set the initial window size, round to next power of 2 and square 332 * for small size, x 4 for medium, and x 2 for large 333 * for 128k (32 page) max ra 334 * 1-2 page = 16k, 3-4 page 32k, 5-8 page = 64k, > 8 page = 128k initial 335 */ 336 static unsigned long get_init_ra_size(unsigned long size, unsigned long max) 337 { 338 unsigned long newsize = roundup_pow_of_two(size); 339 340 if (newsize <= max / 32) 341 newsize = newsize * 4; 342 else if (newsize <= max / 4) 343 newsize = newsize * 2; 344 else 345 newsize = max; 346 347 return newsize; 348 } 349 350 /* 351 * Get the previous window size, ramp it up, and 352 * return it as the new window size. 353 */ 354 static unsigned long get_next_ra_size(struct file_ra_state *ra, 355 unsigned long max) 356 { 357 unsigned long cur = ra->size; 358 359 if (cur < max / 16) 360 return 4 * cur; 361 if (cur <= max / 2) 362 return 2 * cur; 363 return max; 364 } 365 366 /* 367 * On-demand readahead design. 368 * 369 * The fields in struct file_ra_state represent the most-recently-executed 370 * readahead attempt: 371 * 372 * |<----- async_size ---------| 373 * |------------------- size -------------------->| 374 * |==================#===========================| 375 * ^start ^page marked with PG_readahead 376 * 377 * To overlap application thinking time and disk I/O time, we do 378 * `readahead pipelining': Do not wait until the application consumed all 379 * readahead pages and stalled on the missing page at readahead_index; 380 * Instead, submit an asynchronous readahead I/O as soon as there are 381 * only async_size pages left in the readahead window. Normally async_size 382 * will be equal to size, for maximum pipelining. 383 * 384 * In interleaved sequential reads, concurrent streams on the same fd can 385 * be invalidating each other's readahead state. So we flag the new readahead 386 * page at (start+size-async_size) with PG_readahead, and use it as readahead 387 * indicator. The flag won't be set on already cached pages, to avoid the 388 * readahead-for-nothing fuss, saving pointless page cache lookups. 389 * 390 * prev_pos tracks the last visited byte in the _previous_ read request. 391 * It should be maintained by the caller, and will be used for detecting 392 * small random reads. Note that the readahead algorithm checks loosely 393 * for sequential patterns. Hence interleaved reads might be served as 394 * sequential ones. 395 * 396 * There is a special-case: if the first page which the application tries to 397 * read happens to be the first page of the file, it is assumed that a linear 398 * read is about to happen and the window is immediately set to the initial size 399 * based on I/O request size and the max_readahead. 400 * 401 * The code ramps up the readahead size aggressively at first, but slow down as 402 * it approaches max_readhead. 403 */ 404 405 /* 406 * Count contiguously cached pages from @index-1 to @index-@max, 407 * this count is a conservative estimation of 408 * - length of the sequential read sequence, or 409 * - thrashing threshold in memory tight systems 410 */ 411 static pgoff_t count_history_pages(struct address_space *mapping, 412 pgoff_t index, unsigned long max) 413 { 414 pgoff_t head; 415 416 rcu_read_lock(); 417 head = page_cache_prev_miss(mapping, index - 1, max); 418 rcu_read_unlock(); 419 420 return index - 1 - head; 421 } 422 423 /* 424 * page cache context based readahead 425 */ 426 static int try_context_readahead(struct address_space *mapping, 427 struct file_ra_state *ra, 428 pgoff_t index, 429 unsigned long req_size, 430 unsigned long max) 431 { 432 pgoff_t size; 433 434 size = count_history_pages(mapping, index, max); 435 436 /* 437 * not enough history pages: 438 * it could be a random read 439 */ 440 if (size <= req_size) 441 return 0; 442 443 /* 444 * starts from beginning of file: 445 * it is a strong indication of long-run stream (or whole-file-read) 446 */ 447 if (size >= index) 448 size *= 2; 449 450 ra->start = index; 451 ra->size = min(size + req_size, max); 452 ra->async_size = 1; 453 454 return 1; 455 } 456 457 /* 458 * There are some parts of the kernel which assume that PMD entries 459 * are exactly HPAGE_PMD_ORDER. Those should be fixed, but until then, 460 * limit the maximum allocation order to PMD size. I'm not aware of any 461 * assumptions about maximum order if THP are disabled, but 8 seems like 462 * a good order (that's 1MB if you're using 4kB pages) 463 */ 464 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 465 #define MAX_PAGECACHE_ORDER HPAGE_PMD_ORDER 466 #else 467 #define MAX_PAGECACHE_ORDER 8 468 #endif 469 470 static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index, 471 pgoff_t mark, unsigned int order, gfp_t gfp) 472 { 473 int err; 474 struct folio *folio = filemap_alloc_folio(gfp, order); 475 476 if (!folio) 477 return -ENOMEM; 478 mark = round_up(mark, 1UL << order); 479 if (index == mark) 480 folio_set_readahead(folio); 481 err = filemap_add_folio(ractl->mapping, folio, index, gfp); 482 if (err) 483 folio_put(folio); 484 else 485 ractl->_nr_pages += 1UL << order; 486 return err; 487 } 488 489 void page_cache_ra_order(struct readahead_control *ractl, 490 struct file_ra_state *ra, unsigned int new_order) 491 { 492 struct address_space *mapping = ractl->mapping; 493 pgoff_t index = readahead_index(ractl); 494 pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT; 495 pgoff_t mark = index + ra->size - ra->async_size; 496 int err = 0; 497 gfp_t gfp = readahead_gfp_mask(mapping); 498 499 if (!mapping_large_folio_support(mapping) || ra->size < 4) 500 goto fallback; 501 502 limit = min(limit, index + ra->size - 1); 503 504 if (new_order < MAX_PAGECACHE_ORDER) { 505 new_order += 2; 506 if (new_order > MAX_PAGECACHE_ORDER) 507 new_order = MAX_PAGECACHE_ORDER; 508 while ((1 << new_order) > ra->size) 509 new_order--; 510 } 511 512 while (index <= limit) { 513 unsigned int order = new_order; 514 515 /* Align with smaller pages if needed */ 516 if (index & ((1UL << order) - 1)) { 517 order = __ffs(index); 518 if (order == 1) 519 order = 0; 520 } 521 /* Don't allocate pages past EOF */ 522 while (index + (1UL << order) - 1 > limit) { 523 if (--order == 1) 524 order = 0; 525 } 526 err = ra_alloc_folio(ractl, index, mark, order, gfp); 527 if (err) 528 break; 529 index += 1UL << order; 530 } 531 532 if (index > limit) { 533 ra->size += index - limit - 1; 534 ra->async_size += index - limit - 1; 535 } 536 537 read_pages(ractl); 538 539 /* 540 * If there were already pages in the page cache, then we may have 541 * left some gaps. Let the regular readahead code take care of this 542 * situation. 543 */ 544 if (!err) 545 return; 546 fallback: 547 do_page_cache_ra(ractl, ra->size, ra->async_size); 548 } 549 550 /* 551 * A minimal readahead algorithm for trivial sequential/random reads. 552 */ 553 static void ondemand_readahead(struct readahead_control *ractl, 554 struct folio *folio, unsigned long req_size) 555 { 556 struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host); 557 struct file_ra_state *ra = ractl->ra; 558 unsigned long max_pages = ra->ra_pages; 559 unsigned long add_pages; 560 pgoff_t index = readahead_index(ractl); 561 pgoff_t expected, prev_index; 562 unsigned int order = folio ? folio_order(folio) : 0; 563 564 /* 565 * If the request exceeds the readahead window, allow the read to 566 * be up to the optimal hardware IO size 567 */ 568 if (req_size > max_pages && bdi->io_pages > max_pages) 569 max_pages = min(req_size, bdi->io_pages); 570 571 /* 572 * start of file 573 */ 574 if (!index) 575 goto initial_readahead; 576 577 /* 578 * It's the expected callback index, assume sequential access. 579 * Ramp up sizes, and push forward the readahead window. 580 */ 581 expected = round_up(ra->start + ra->size - ra->async_size, 582 1UL << order); 583 if (index == expected || index == (ra->start + ra->size)) { 584 ra->start += ra->size; 585 ra->size = get_next_ra_size(ra, max_pages); 586 ra->async_size = ra->size; 587 goto readit; 588 } 589 590 /* 591 * Hit a marked folio without valid readahead state. 592 * E.g. interleaved reads. 593 * Query the pagecache for async_size, which normally equals to 594 * readahead size. Ramp it up and use it as the new readahead size. 595 */ 596 if (folio) { 597 pgoff_t start; 598 599 rcu_read_lock(); 600 start = page_cache_next_miss(ractl->mapping, index + 1, 601 max_pages); 602 rcu_read_unlock(); 603 604 if (!start || start - index > max_pages) 605 return; 606 607 ra->start = start; 608 ra->size = start - index; /* old async_size */ 609 ra->size += req_size; 610 ra->size = get_next_ra_size(ra, max_pages); 611 ra->async_size = ra->size; 612 goto readit; 613 } 614 615 /* 616 * oversize read 617 */ 618 if (req_size > max_pages) 619 goto initial_readahead; 620 621 /* 622 * sequential cache miss 623 * trivial case: (index - prev_index) == 1 624 * unaligned reads: (index - prev_index) == 0 625 */ 626 prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT; 627 if (index - prev_index <= 1UL) 628 goto initial_readahead; 629 630 /* 631 * Query the page cache and look for the traces(cached history pages) 632 * that a sequential stream would leave behind. 633 */ 634 if (try_context_readahead(ractl->mapping, ra, index, req_size, 635 max_pages)) 636 goto readit; 637 638 /* 639 * standalone, small random read 640 * Read as is, and do not pollute the readahead state. 641 */ 642 do_page_cache_ra(ractl, req_size, 0); 643 return; 644 645 initial_readahead: 646 ra->start = index; 647 ra->size = get_init_ra_size(req_size, max_pages); 648 ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size; 649 650 readit: 651 /* 652 * Will this read hit the readahead marker made by itself? 653 * If so, trigger the readahead marker hit now, and merge 654 * the resulted next readahead window into the current one. 655 * Take care of maximum IO pages as above. 656 */ 657 if (index == ra->start && ra->size == ra->async_size) { 658 add_pages = get_next_ra_size(ra, max_pages); 659 if (ra->size + add_pages <= max_pages) { 660 ra->async_size = add_pages; 661 ra->size += add_pages; 662 } else { 663 ra->size = max_pages; 664 ra->async_size = max_pages >> 1; 665 } 666 } 667 668 ractl->_index = ra->start; 669 page_cache_ra_order(ractl, ra, order); 670 } 671 672 void page_cache_sync_ra(struct readahead_control *ractl, 673 unsigned long req_count) 674 { 675 bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM); 676 677 /* 678 * Even if readahead is disabled, issue this request as readahead 679 * as we'll need it to satisfy the requested range. The forced 680 * readahead will do the right thing and limit the read to just the 681 * requested range, which we'll set to 1 page for this case. 682 */ 683 if (!ractl->ra->ra_pages || blk_cgroup_congested()) { 684 if (!ractl->file) 685 return; 686 req_count = 1; 687 do_forced_ra = true; 688 } 689 690 /* be dumb */ 691 if (do_forced_ra) { 692 force_page_cache_ra(ractl, req_count); 693 return; 694 } 695 696 ondemand_readahead(ractl, NULL, req_count); 697 } 698 EXPORT_SYMBOL_GPL(page_cache_sync_ra); 699 700 void page_cache_async_ra(struct readahead_control *ractl, 701 struct folio *folio, unsigned long req_count) 702 { 703 /* no readahead */ 704 if (!ractl->ra->ra_pages) 705 return; 706 707 /* 708 * Same bit is used for PG_readahead and PG_reclaim. 709 */ 710 if (folio_test_writeback(folio)) 711 return; 712 713 folio_clear_readahead(folio); 714 715 if (blk_cgroup_congested()) 716 return; 717 718 ondemand_readahead(ractl, folio, req_count); 719 } 720 EXPORT_SYMBOL_GPL(page_cache_async_ra); 721 722 ssize_t ksys_readahead(int fd, loff_t offset, size_t count) 723 { 724 ssize_t ret; 725 struct fd f; 726 727 ret = -EBADF; 728 f = fdget(fd); 729 if (!f.file || !(f.file->f_mode & FMODE_READ)) 730 goto out; 731 732 /* 733 * The readahead() syscall is intended to run only on files 734 * that can execute readahead. If readahead is not possible 735 * on this file, then we must return -EINVAL. 736 */ 737 ret = -EINVAL; 738 if (!f.file->f_mapping || !f.file->f_mapping->a_ops || 739 !S_ISREG(file_inode(f.file)->i_mode)) 740 goto out; 741 742 ret = vfs_fadvise(f.file, offset, count, POSIX_FADV_WILLNEED); 743 out: 744 fdput(f); 745 return ret; 746 } 747 748 SYSCALL_DEFINE3(readahead, int, fd, loff_t, offset, size_t, count) 749 { 750 return ksys_readahead(fd, offset, count); 751 } 752 753 /** 754 * readahead_expand - Expand a readahead request 755 * @ractl: The request to be expanded 756 * @new_start: The revised start 757 * @new_len: The revised size of the request 758 * 759 * Attempt to expand a readahead request outwards from the current size to the 760 * specified size by inserting locked pages before and after the current window 761 * to increase the size to the new window. This may involve the insertion of 762 * THPs, in which case the window may get expanded even beyond what was 763 * requested. 764 * 765 * The algorithm will stop if it encounters a conflicting page already in the 766 * pagecache and leave a smaller expansion than requested. 767 * 768 * The caller must check for this by examining the revised @ractl object for a 769 * different expansion than was requested. 770 */ 771 void readahead_expand(struct readahead_control *ractl, 772 loff_t new_start, size_t new_len) 773 { 774 struct address_space *mapping = ractl->mapping; 775 struct file_ra_state *ra = ractl->ra; 776 pgoff_t new_index, new_nr_pages; 777 gfp_t gfp_mask = readahead_gfp_mask(mapping); 778 779 new_index = new_start / PAGE_SIZE; 780 781 /* Expand the leading edge downwards */ 782 while (ractl->_index > new_index) { 783 unsigned long index = ractl->_index - 1; 784 struct page *page = xa_load(&mapping->i_pages, index); 785 786 if (page && !xa_is_value(page)) 787 return; /* Page apparently present */ 788 789 page = __page_cache_alloc(gfp_mask); 790 if (!page) 791 return; 792 if (add_to_page_cache_lru(page, mapping, index, gfp_mask) < 0) { 793 put_page(page); 794 return; 795 } 796 797 ractl->_nr_pages++; 798 ractl->_index = page->index; 799 } 800 801 new_len += new_start - readahead_pos(ractl); 802 new_nr_pages = DIV_ROUND_UP(new_len, PAGE_SIZE); 803 804 /* Expand the trailing edge upwards */ 805 while (ractl->_nr_pages < new_nr_pages) { 806 unsigned long index = ractl->_index + ractl->_nr_pages; 807 struct page *page = xa_load(&mapping->i_pages, index); 808 809 if (page && !xa_is_value(page)) 810 return; /* Page apparently present */ 811 812 page = __page_cache_alloc(gfp_mask); 813 if (!page) 814 return; 815 if (add_to_page_cache_lru(page, mapping, index, gfp_mask) < 0) { 816 put_page(page); 817 return; 818 } 819 ractl->_nr_pages++; 820 if (ra) { 821 ra->size++; 822 ra->async_size++; 823 } 824 } 825 } 826 EXPORT_SYMBOL(readahead_expand); 827