1288ace2fSDavid Howells // SPDX-License-Identifier: GPL-2.0-only
2288ace2fSDavid Howells /* Network filesystem high-level (buffered) writeback.
3288ace2fSDavid Howells *
4288ace2fSDavid Howells * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5288ace2fSDavid Howells * Written by David Howells (dhowells@redhat.com)
6288ace2fSDavid Howells *
7288ace2fSDavid Howells *
8288ace2fSDavid Howells * To support network filesystems with local caching, we manage a situation
9288ace2fSDavid Howells * that can be envisioned like the following:
10288ace2fSDavid Howells *
11288ace2fSDavid Howells * +---+---+-----+-----+---+----------+
12288ace2fSDavid Howells * Folios: | | | | | | |
13288ace2fSDavid Howells * +---+---+-----+-----+---+----------+
14288ace2fSDavid Howells *
15288ace2fSDavid Howells * +------+------+ +----+----+
16288ace2fSDavid Howells * Upload: | | |.....| | |
17288ace2fSDavid Howells * (Stream 0) +------+------+ +----+----+
18288ace2fSDavid Howells *
19288ace2fSDavid Howells * +------+------+------+------+------+
20288ace2fSDavid Howells * Cache: | | | | | |
21288ace2fSDavid Howells * (Stream 1) +------+------+------+------+------+
22288ace2fSDavid Howells *
23288ace2fSDavid Howells * Where we have a sequence of folios of varying sizes that we need to overlay
24288ace2fSDavid Howells * with multiple parallel streams of I/O requests, where the I/O requests in a
25288ace2fSDavid Howells * stream may also be of various sizes (in cifs, for example, the sizes are
26288ace2fSDavid Howells * negotiated with the server; in something like ceph, they may represent the
27288ace2fSDavid Howells * sizes of storage objects).
28288ace2fSDavid Howells *
29288ace2fSDavid Howells * The sequence in each stream may contain gaps and noncontiguous subrequests
30288ace2fSDavid Howells * may be glued together into single vectored write RPCs.
31288ace2fSDavid Howells */
32288ace2fSDavid Howells
33288ace2fSDavid Howells #include <linux/export.h>
34288ace2fSDavid Howells #include <linux/fs.h>
35288ace2fSDavid Howells #include <linux/mm.h>
36288ace2fSDavid Howells #include <linux/pagemap.h>
37288ace2fSDavid Howells #include "internal.h"
38288ace2fSDavid Howells
39288ace2fSDavid Howells /*
40288ace2fSDavid Howells * Kill all dirty folios in the event of an unrecoverable error, starting with
41288ace2fSDavid Howells * a locked folio we've already obtained from writeback_iter().
42288ace2fSDavid Howells */
netfs_kill_dirty_pages(struct address_space * mapping,struct writeback_control * wbc,struct folio * folio)43288ace2fSDavid Howells static void netfs_kill_dirty_pages(struct address_space *mapping,
44288ace2fSDavid Howells struct writeback_control *wbc,
45288ace2fSDavid Howells struct folio *folio)
46288ace2fSDavid Howells {
47288ace2fSDavid Howells int error = 0;
48288ace2fSDavid Howells
49288ace2fSDavid Howells do {
50288ace2fSDavid Howells enum netfs_folio_trace why = netfs_folio_trace_kill;
51288ace2fSDavid Howells struct netfs_group *group = NULL;
52288ace2fSDavid Howells struct netfs_folio *finfo = NULL;
53288ace2fSDavid Howells void *priv;
54288ace2fSDavid Howells
55288ace2fSDavid Howells priv = folio_detach_private(folio);
56288ace2fSDavid Howells if (priv) {
57288ace2fSDavid Howells finfo = __netfs_folio_info(priv);
58288ace2fSDavid Howells if (finfo) {
59288ace2fSDavid Howells /* Kill folio from streaming write. */
60288ace2fSDavid Howells group = finfo->netfs_group;
61288ace2fSDavid Howells why = netfs_folio_trace_kill_s;
62288ace2fSDavid Howells } else {
63288ace2fSDavid Howells group = priv;
64288ace2fSDavid Howells if (group == NETFS_FOLIO_COPY_TO_CACHE) {
65288ace2fSDavid Howells /* Kill copy-to-cache folio */
66288ace2fSDavid Howells why = netfs_folio_trace_kill_cc;
67288ace2fSDavid Howells group = NULL;
68288ace2fSDavid Howells } else {
69288ace2fSDavid Howells /* Kill folio with group */
70288ace2fSDavid Howells why = netfs_folio_trace_kill_g;
71288ace2fSDavid Howells }
72288ace2fSDavid Howells }
73288ace2fSDavid Howells }
74288ace2fSDavid Howells
75288ace2fSDavid Howells trace_netfs_folio(folio, why);
76288ace2fSDavid Howells
77288ace2fSDavid Howells folio_start_writeback(folio);
78288ace2fSDavid Howells folio_unlock(folio);
79288ace2fSDavid Howells folio_end_writeback(folio);
80288ace2fSDavid Howells
81288ace2fSDavid Howells netfs_put_group(group);
82288ace2fSDavid Howells kfree(finfo);
83288ace2fSDavid Howells
84288ace2fSDavid Howells } while ((folio = writeback_iter(mapping, wbc, folio, &error)));
85288ace2fSDavid Howells }
86288ace2fSDavid Howells
87288ace2fSDavid Howells /*
88288ace2fSDavid Howells * Create a write request and set it up appropriately for the origin type.
89288ace2fSDavid Howells */
netfs_create_write_req(struct address_space * mapping,struct file * file,loff_t start,enum netfs_io_origin origin)90288ace2fSDavid Howells struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
91288ace2fSDavid Howells struct file *file,
92288ace2fSDavid Howells loff_t start,
93288ace2fSDavid Howells enum netfs_io_origin origin)
94288ace2fSDavid Howells {
95288ace2fSDavid Howells struct netfs_io_request *wreq;
96288ace2fSDavid Howells struct netfs_inode *ictx;
97288ace2fSDavid Howells
98288ace2fSDavid Howells wreq = netfs_alloc_request(mapping, file, start, 0, origin);
99288ace2fSDavid Howells if (IS_ERR(wreq))
100288ace2fSDavid Howells return wreq;
101288ace2fSDavid Howells
102288ace2fSDavid Howells _enter("R=%x", wreq->debug_id);
103288ace2fSDavid Howells
104288ace2fSDavid Howells ictx = netfs_inode(wreq->inode);
105288ace2fSDavid Howells if (test_bit(NETFS_RREQ_WRITE_TO_CACHE, &wreq->flags))
106288ace2fSDavid Howells fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx));
107288ace2fSDavid Howells
108288ace2fSDavid Howells wreq->contiguity = wreq->start;
109288ace2fSDavid Howells wreq->cleaned_to = wreq->start;
110288ace2fSDavid Howells INIT_WORK(&wreq->work, netfs_write_collection_worker);
111288ace2fSDavid Howells
112288ace2fSDavid Howells wreq->io_streams[0].stream_nr = 0;
113288ace2fSDavid Howells wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER;
114288ace2fSDavid Howells wreq->io_streams[0].prepare_write = ictx->ops->prepare_write;
115288ace2fSDavid Howells wreq->io_streams[0].issue_write = ictx->ops->issue_write;
116288ace2fSDavid Howells wreq->io_streams[0].collected_to = start;
117288ace2fSDavid Howells wreq->io_streams[0].transferred = LONG_MAX;
118288ace2fSDavid Howells
119288ace2fSDavid Howells wreq->io_streams[1].stream_nr = 1;
120288ace2fSDavid Howells wreq->io_streams[1].source = NETFS_WRITE_TO_CACHE;
121288ace2fSDavid Howells wreq->io_streams[1].collected_to = start;
122288ace2fSDavid Howells wreq->io_streams[1].transferred = LONG_MAX;
123288ace2fSDavid Howells if (fscache_resources_valid(&wreq->cache_resources)) {
124288ace2fSDavid Howells wreq->io_streams[1].avail = true;
125288ace2fSDavid Howells wreq->io_streams[1].prepare_write = wreq->cache_resources.ops->prepare_write_subreq;
126288ace2fSDavid Howells wreq->io_streams[1].issue_write = wreq->cache_resources.ops->issue_write;
127288ace2fSDavid Howells }
128288ace2fSDavid Howells
129288ace2fSDavid Howells return wreq;
130288ace2fSDavid Howells }
131288ace2fSDavid Howells
132288ace2fSDavid Howells /**
133288ace2fSDavid Howells * netfs_prepare_write_failed - Note write preparation failed
134288ace2fSDavid Howells * @subreq: The subrequest to mark
135288ace2fSDavid Howells *
136288ace2fSDavid Howells * Mark a subrequest to note that preparation for write failed.
137288ace2fSDavid Howells */
netfs_prepare_write_failed(struct netfs_io_subrequest * subreq)138288ace2fSDavid Howells void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq)
139288ace2fSDavid Howells {
140288ace2fSDavid Howells __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
141288ace2fSDavid Howells trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed);
142288ace2fSDavid Howells }
143288ace2fSDavid Howells EXPORT_SYMBOL(netfs_prepare_write_failed);
144288ace2fSDavid Howells
145288ace2fSDavid Howells /*
146288ace2fSDavid Howells * Prepare a write subrequest. We need to allocate a new subrequest
147288ace2fSDavid Howells * if we don't have one.
148288ace2fSDavid Howells */
netfs_prepare_write(struct netfs_io_request * wreq,struct netfs_io_stream * stream,loff_t start)149288ace2fSDavid Howells static void netfs_prepare_write(struct netfs_io_request *wreq,
150288ace2fSDavid Howells struct netfs_io_stream *stream,
151288ace2fSDavid Howells loff_t start)
152288ace2fSDavid Howells {
153288ace2fSDavid Howells struct netfs_io_subrequest *subreq;
154288ace2fSDavid Howells
155288ace2fSDavid Howells subreq = netfs_alloc_subrequest(wreq);
156288ace2fSDavid Howells subreq->source = stream->source;
157288ace2fSDavid Howells subreq->start = start;
158288ace2fSDavid Howells subreq->max_len = ULONG_MAX;
159288ace2fSDavid Howells subreq->max_nr_segs = INT_MAX;
160288ace2fSDavid Howells subreq->stream_nr = stream->stream_nr;
161288ace2fSDavid Howells
162288ace2fSDavid Howells _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
163288ace2fSDavid Howells
164288ace2fSDavid Howells trace_netfs_sreq_ref(wreq->debug_id, subreq->debug_index,
165288ace2fSDavid Howells refcount_read(&subreq->ref),
166288ace2fSDavid Howells netfs_sreq_trace_new);
167288ace2fSDavid Howells
168288ace2fSDavid Howells trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
169288ace2fSDavid Howells
170288ace2fSDavid Howells switch (stream->source) {
171288ace2fSDavid Howells case NETFS_UPLOAD_TO_SERVER:
172288ace2fSDavid Howells netfs_stat(&netfs_n_wh_upload);
173288ace2fSDavid Howells subreq->max_len = wreq->wsize;
174288ace2fSDavid Howells break;
175288ace2fSDavid Howells case NETFS_WRITE_TO_CACHE:
176288ace2fSDavid Howells netfs_stat(&netfs_n_wh_write);
177288ace2fSDavid Howells break;
178288ace2fSDavid Howells default:
179288ace2fSDavid Howells WARN_ON_ONCE(1);
180288ace2fSDavid Howells break;
181288ace2fSDavid Howells }
182288ace2fSDavid Howells
183288ace2fSDavid Howells if (stream->prepare_write)
184288ace2fSDavid Howells stream->prepare_write(subreq);
185288ace2fSDavid Howells
186288ace2fSDavid Howells __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
187288ace2fSDavid Howells
188288ace2fSDavid Howells /* We add to the end of the list whilst the collector may be walking
189288ace2fSDavid Howells * the list. The collector only goes nextwards and uses the lock to
190288ace2fSDavid Howells * remove entries off of the front.
191288ace2fSDavid Howells */
192288ace2fSDavid Howells spin_lock(&wreq->lock);
193288ace2fSDavid Howells list_add_tail(&subreq->rreq_link, &stream->subrequests);
194288ace2fSDavid Howells if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
195288ace2fSDavid Howells stream->front = subreq;
196288ace2fSDavid Howells if (!stream->active) {
197288ace2fSDavid Howells stream->collected_to = stream->front->start;
198288ace2fSDavid Howells /* Write list pointers before active flag */
199288ace2fSDavid Howells smp_store_release(&stream->active, true);
200288ace2fSDavid Howells }
201288ace2fSDavid Howells }
202288ace2fSDavid Howells
203288ace2fSDavid Howells spin_unlock(&wreq->lock);
204288ace2fSDavid Howells
205288ace2fSDavid Howells stream->construct = subreq;
206288ace2fSDavid Howells }
207288ace2fSDavid Howells
208288ace2fSDavid Howells /*
209288ace2fSDavid Howells * Set the I/O iterator for the filesystem/cache to use and dispatch the I/O
210288ace2fSDavid Howells * operation. The operation may be asynchronous and should call
211288ace2fSDavid Howells * netfs_write_subrequest_terminated() when complete.
212288ace2fSDavid Howells */
netfs_do_issue_write(struct netfs_io_stream * stream,struct netfs_io_subrequest * subreq)213288ace2fSDavid Howells static void netfs_do_issue_write(struct netfs_io_stream *stream,
214288ace2fSDavid Howells struct netfs_io_subrequest *subreq)
215288ace2fSDavid Howells {
216288ace2fSDavid Howells struct netfs_io_request *wreq = subreq->rreq;
217288ace2fSDavid Howells
218288ace2fSDavid Howells _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len);
219288ace2fSDavid Howells
220288ace2fSDavid Howells if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
221288ace2fSDavid Howells return netfs_write_subrequest_terminated(subreq, subreq->error, false);
222288ace2fSDavid Howells
223288ace2fSDavid Howells // TODO: Use encrypted buffer
224288ace2fSDavid Howells if (test_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags)) {
225288ace2fSDavid Howells subreq->io_iter = wreq->io_iter;
226288ace2fSDavid Howells iov_iter_advance(&subreq->io_iter,
227288ace2fSDavid Howells subreq->start + subreq->transferred - wreq->start);
228288ace2fSDavid Howells iov_iter_truncate(&subreq->io_iter,
229288ace2fSDavid Howells subreq->len - subreq->transferred);
230288ace2fSDavid Howells } else {
231288ace2fSDavid Howells iov_iter_xarray(&subreq->io_iter, ITER_SOURCE, &wreq->mapping->i_pages,
232288ace2fSDavid Howells subreq->start + subreq->transferred,
233288ace2fSDavid Howells subreq->len - subreq->transferred);
234288ace2fSDavid Howells }
235288ace2fSDavid Howells
236288ace2fSDavid Howells trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
237288ace2fSDavid Howells stream->issue_write(subreq);
238288ace2fSDavid Howells }
239288ace2fSDavid Howells
netfs_reissue_write(struct netfs_io_stream * stream,struct netfs_io_subrequest * subreq)240288ace2fSDavid Howells void netfs_reissue_write(struct netfs_io_stream *stream,
241288ace2fSDavid Howells struct netfs_io_subrequest *subreq)
242288ace2fSDavid Howells {
243288ace2fSDavid Howells __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
244288ace2fSDavid Howells netfs_do_issue_write(stream, subreq);
245288ace2fSDavid Howells }
246288ace2fSDavid Howells
netfs_issue_write(struct netfs_io_request * wreq,struct netfs_io_stream * stream)247288ace2fSDavid Howells static void netfs_issue_write(struct netfs_io_request *wreq,
248288ace2fSDavid Howells struct netfs_io_stream *stream)
249288ace2fSDavid Howells {
250288ace2fSDavid Howells struct netfs_io_subrequest *subreq = stream->construct;
251288ace2fSDavid Howells
252288ace2fSDavid Howells if (!subreq)
253288ace2fSDavid Howells return;
254288ace2fSDavid Howells stream->construct = NULL;
255288ace2fSDavid Howells
256288ace2fSDavid Howells if (subreq->start + subreq->len > wreq->start + wreq->submitted)
2579b038d00SDavid Howells WRITE_ONCE(wreq->submitted, subreq->start + subreq->len - wreq->start);
258288ace2fSDavid Howells netfs_do_issue_write(stream, subreq);
259288ace2fSDavid Howells }
260288ace2fSDavid Howells
261288ace2fSDavid Howells /*
262288ace2fSDavid Howells * Add data to the write subrequest, dispatching each as we fill it up or if it
263288ace2fSDavid Howells * is discontiguous with the previous. We only fill one part at a time so that
264288ace2fSDavid Howells * we can avoid overrunning the credits obtained (cifs) and try to parallelise
265288ace2fSDavid Howells * content-crypto preparation with network writes.
266288ace2fSDavid Howells */
netfs_advance_write(struct netfs_io_request * wreq,struct netfs_io_stream * stream,loff_t start,size_t len,bool to_eof)267288ace2fSDavid Howells int netfs_advance_write(struct netfs_io_request *wreq,
268288ace2fSDavid Howells struct netfs_io_stream *stream,
269288ace2fSDavid Howells loff_t start, size_t len, bool to_eof)
270288ace2fSDavid Howells {
271288ace2fSDavid Howells struct netfs_io_subrequest *subreq = stream->construct;
272288ace2fSDavid Howells size_t part;
273288ace2fSDavid Howells
274288ace2fSDavid Howells if (!stream->avail) {
275288ace2fSDavid Howells _leave("no write");
276288ace2fSDavid Howells return len;
277288ace2fSDavid Howells }
278288ace2fSDavid Howells
279288ace2fSDavid Howells _enter("R=%x[%x]", wreq->debug_id, subreq ? subreq->debug_index : 0);
280288ace2fSDavid Howells
281288ace2fSDavid Howells if (subreq && start != subreq->start + subreq->len) {
282288ace2fSDavid Howells netfs_issue_write(wreq, stream);
283288ace2fSDavid Howells subreq = NULL;
284288ace2fSDavid Howells }
285288ace2fSDavid Howells
286288ace2fSDavid Howells if (!stream->construct)
287288ace2fSDavid Howells netfs_prepare_write(wreq, stream, start);
288288ace2fSDavid Howells subreq = stream->construct;
289288ace2fSDavid Howells
290288ace2fSDavid Howells part = min(subreq->max_len - subreq->len, len);
291288ace2fSDavid Howells _debug("part %zx/%zx %zx/%zx", subreq->len, subreq->max_len, part, len);
292288ace2fSDavid Howells subreq->len += part;
293288ace2fSDavid Howells subreq->nr_segs++;
294288ace2fSDavid Howells
295288ace2fSDavid Howells if (subreq->len >= subreq->max_len ||
296288ace2fSDavid Howells subreq->nr_segs >= subreq->max_nr_segs ||
297288ace2fSDavid Howells to_eof) {
298288ace2fSDavid Howells netfs_issue_write(wreq, stream);
299288ace2fSDavid Howells subreq = NULL;
300288ace2fSDavid Howells }
301288ace2fSDavid Howells
302288ace2fSDavid Howells return part;
303288ace2fSDavid Howells }
304288ace2fSDavid Howells
305288ace2fSDavid Howells /*
306288ace2fSDavid Howells * Write some of a pending folio data back to the server.
307288ace2fSDavid Howells */
netfs_write_folio(struct netfs_io_request * wreq,struct writeback_control * wbc,struct folio * folio)308288ace2fSDavid Howells static int netfs_write_folio(struct netfs_io_request *wreq,
309288ace2fSDavid Howells struct writeback_control *wbc,
310288ace2fSDavid Howells struct folio *folio)
311288ace2fSDavid Howells {
312288ace2fSDavid Howells struct netfs_io_stream *upload = &wreq->io_streams[0];
313288ace2fSDavid Howells struct netfs_io_stream *cache = &wreq->io_streams[1];
314288ace2fSDavid Howells struct netfs_io_stream *stream;
315288ace2fSDavid Howells struct netfs_group *fgroup; /* TODO: Use this with ceph */
316288ace2fSDavid Howells struct netfs_folio *finfo;
317288ace2fSDavid Howells size_t fsize = folio_size(folio), flen = fsize, foff = 0;
318288ace2fSDavid Howells loff_t fpos = folio_pos(folio), i_size;
319288ace2fSDavid Howells bool to_eof = false, streamw = false;
320288ace2fSDavid Howells bool debug = false;
321288ace2fSDavid Howells
322288ace2fSDavid Howells _enter("");
323288ace2fSDavid Howells
324288ace2fSDavid Howells /* netfs_perform_write() may shift i_size around the page or from out
325288ace2fSDavid Howells * of the page to beyond it, but cannot move i_size into or through the
326288ace2fSDavid Howells * page since we have it locked.
327288ace2fSDavid Howells */
328288ace2fSDavid Howells i_size = i_size_read(wreq->inode);
329288ace2fSDavid Howells
330288ace2fSDavid Howells if (fpos >= i_size) {
331288ace2fSDavid Howells /* mmap beyond eof. */
332288ace2fSDavid Howells _debug("beyond eof");
333288ace2fSDavid Howells folio_start_writeback(folio);
334288ace2fSDavid Howells folio_unlock(folio);
335288ace2fSDavid Howells wreq->nr_group_rel += netfs_folio_written_back(folio);
336288ace2fSDavid Howells netfs_put_group_many(wreq->group, wreq->nr_group_rel);
337288ace2fSDavid Howells wreq->nr_group_rel = 0;
338288ace2fSDavid Howells return 0;
339288ace2fSDavid Howells }
340288ace2fSDavid Howells
341288ace2fSDavid Howells if (fpos + fsize > wreq->i_size)
342288ace2fSDavid Howells wreq->i_size = i_size;
343288ace2fSDavid Howells
344288ace2fSDavid Howells fgroup = netfs_folio_group(folio);
345288ace2fSDavid Howells finfo = netfs_folio_info(folio);
346288ace2fSDavid Howells if (finfo) {
347288ace2fSDavid Howells foff = finfo->dirty_offset;
348288ace2fSDavid Howells flen = foff + finfo->dirty_len;
349288ace2fSDavid Howells streamw = true;
350288ace2fSDavid Howells }
351288ace2fSDavid Howells
352288ace2fSDavid Howells if (wreq->origin == NETFS_WRITETHROUGH) {
353288ace2fSDavid Howells to_eof = false;
354288ace2fSDavid Howells if (flen > i_size - fpos)
355288ace2fSDavid Howells flen = i_size - fpos;
356288ace2fSDavid Howells } else if (flen > i_size - fpos) {
357288ace2fSDavid Howells flen = i_size - fpos;
358288ace2fSDavid Howells if (!streamw)
359288ace2fSDavid Howells folio_zero_segment(folio, flen, fsize);
360288ace2fSDavid Howells to_eof = true;
361288ace2fSDavid Howells } else if (flen == i_size - fpos) {
362288ace2fSDavid Howells to_eof = true;
363288ace2fSDavid Howells }
364288ace2fSDavid Howells flen -= foff;
365288ace2fSDavid Howells
366288ace2fSDavid Howells _debug("folio %zx %zx %zx", foff, flen, fsize);
367288ace2fSDavid Howells
368288ace2fSDavid Howells /* Deal with discontinuities in the stream of dirty pages. These can
369288ace2fSDavid Howells * arise from a number of sources:
370288ace2fSDavid Howells *
371288ace2fSDavid Howells * (1) Intervening non-dirty pages from random-access writes, multiple
372288ace2fSDavid Howells * flushers writing back different parts simultaneously and manual
373288ace2fSDavid Howells * syncing.
374288ace2fSDavid Howells *
375288ace2fSDavid Howells * (2) Partially-written pages from write-streaming.
376288ace2fSDavid Howells *
377288ace2fSDavid Howells * (3) Pages that belong to a different write-back group (eg. Ceph
378288ace2fSDavid Howells * snapshots).
379288ace2fSDavid Howells *
380288ace2fSDavid Howells * (4) Actually-clean pages that were marked for write to the cache
381288ace2fSDavid Howells * when they were read. Note that these appear as a special
382288ace2fSDavid Howells * write-back group.
383288ace2fSDavid Howells */
384288ace2fSDavid Howells if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
385288ace2fSDavid Howells netfs_issue_write(wreq, upload);
386288ace2fSDavid Howells } else if (fgroup != wreq->group) {
387288ace2fSDavid Howells /* We can't write this page to the server yet. */
388288ace2fSDavid Howells kdebug("wrong group");
389288ace2fSDavid Howells folio_redirty_for_writepage(wbc, folio);
390288ace2fSDavid Howells folio_unlock(folio);
391288ace2fSDavid Howells netfs_issue_write(wreq, upload);
392288ace2fSDavid Howells netfs_issue_write(wreq, cache);
393288ace2fSDavid Howells return 0;
394288ace2fSDavid Howells }
395288ace2fSDavid Howells
396288ace2fSDavid Howells if (foff > 0)
397288ace2fSDavid Howells netfs_issue_write(wreq, upload);
398288ace2fSDavid Howells if (streamw)
399288ace2fSDavid Howells netfs_issue_write(wreq, cache);
400288ace2fSDavid Howells
401288ace2fSDavid Howells /* Flip the page to the writeback state and unlock. If we're called
402288ace2fSDavid Howells * from write-through, then the page has already been put into the wb
403288ace2fSDavid Howells * state.
404288ace2fSDavid Howells */
405288ace2fSDavid Howells if (wreq->origin == NETFS_WRITEBACK)
406288ace2fSDavid Howells folio_start_writeback(folio);
407288ace2fSDavid Howells folio_unlock(folio);
408288ace2fSDavid Howells
409288ace2fSDavid Howells if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
410288ace2fSDavid Howells if (!fscache_resources_valid(&wreq->cache_resources)) {
411288ace2fSDavid Howells trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
412288ace2fSDavid Howells netfs_issue_write(wreq, upload);
413288ace2fSDavid Howells netfs_folio_written_back(folio);
414288ace2fSDavid Howells return 0;
415288ace2fSDavid Howells }
416288ace2fSDavid Howells trace_netfs_folio(folio, netfs_folio_trace_store_copy);
417288ace2fSDavid Howells } else if (!upload->construct) {
418288ace2fSDavid Howells trace_netfs_folio(folio, netfs_folio_trace_store);
419288ace2fSDavid Howells } else {
420288ace2fSDavid Howells trace_netfs_folio(folio, netfs_folio_trace_store_plus);
421288ace2fSDavid Howells }
422288ace2fSDavid Howells
423288ace2fSDavid Howells /* Move the submission point forward to allow for write-streaming data
424288ace2fSDavid Howells * not starting at the front of the page. We don't do write-streaming
425288ace2fSDavid Howells * with the cache as the cache requires DIO alignment.
426288ace2fSDavid Howells *
427288ace2fSDavid Howells * Also skip uploading for data that's been read and just needs copying
428288ace2fSDavid Howells * to the cache.
429288ace2fSDavid Howells */
430288ace2fSDavid Howells for (int s = 0; s < NR_IO_STREAMS; s++) {
431288ace2fSDavid Howells stream = &wreq->io_streams[s];
432288ace2fSDavid Howells stream->submit_max_len = fsize;
433288ace2fSDavid Howells stream->submit_off = foff;
434288ace2fSDavid Howells stream->submit_len = flen;
435288ace2fSDavid Howells if ((stream->source == NETFS_WRITE_TO_CACHE && streamw) ||
436288ace2fSDavid Howells (stream->source == NETFS_UPLOAD_TO_SERVER &&
437288ace2fSDavid Howells fgroup == NETFS_FOLIO_COPY_TO_CACHE)) {
438288ace2fSDavid Howells stream->submit_off = UINT_MAX;
439288ace2fSDavid Howells stream->submit_len = 0;
440288ace2fSDavid Howells stream->submit_max_len = 0;
441288ace2fSDavid Howells }
442288ace2fSDavid Howells }
443288ace2fSDavid Howells
444288ace2fSDavid Howells /* Attach the folio to one or more subrequests. For a big folio, we
445288ace2fSDavid Howells * could end up with thousands of subrequests if the wsize is small -
446288ace2fSDavid Howells * but we might need to wait during the creation of subrequests for
447288ace2fSDavid Howells * network resources (eg. SMB credits).
448288ace2fSDavid Howells */
449288ace2fSDavid Howells for (;;) {
450288ace2fSDavid Howells ssize_t part;
451288ace2fSDavid Howells size_t lowest_off = ULONG_MAX;
452288ace2fSDavid Howells int choose_s = -1;
453288ace2fSDavid Howells
454288ace2fSDavid Howells /* Always add to the lowest-submitted stream first. */
455288ace2fSDavid Howells for (int s = 0; s < NR_IO_STREAMS; s++) {
456288ace2fSDavid Howells stream = &wreq->io_streams[s];
457288ace2fSDavid Howells if (stream->submit_len > 0 &&
458288ace2fSDavid Howells stream->submit_off < lowest_off) {
459288ace2fSDavid Howells lowest_off = stream->submit_off;
460288ace2fSDavid Howells choose_s = s;
461288ace2fSDavid Howells }
462288ace2fSDavid Howells }
463288ace2fSDavid Howells
464288ace2fSDavid Howells if (choose_s < 0)
465288ace2fSDavid Howells break;
466288ace2fSDavid Howells stream = &wreq->io_streams[choose_s];
467288ace2fSDavid Howells
468288ace2fSDavid Howells part = netfs_advance_write(wreq, stream, fpos + stream->submit_off,
469288ace2fSDavid Howells stream->submit_len, to_eof);
470288ace2fSDavid Howells atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
471288ace2fSDavid Howells stream->submit_off += part;
472288ace2fSDavid Howells stream->submit_max_len -= part;
473288ace2fSDavid Howells if (part > stream->submit_len)
474288ace2fSDavid Howells stream->submit_len = 0;
475288ace2fSDavid Howells else
476288ace2fSDavid Howells stream->submit_len -= part;
477288ace2fSDavid Howells if (part > 0)
478288ace2fSDavid Howells debug = true;
479288ace2fSDavid Howells }
480288ace2fSDavid Howells
481288ace2fSDavid Howells atomic64_set(&wreq->issued_to, fpos + fsize);
482288ace2fSDavid Howells
483288ace2fSDavid Howells if (!debug)
484288ace2fSDavid Howells kdebug("R=%x: No submit", wreq->debug_id);
485288ace2fSDavid Howells
486288ace2fSDavid Howells if (flen < fsize)
487288ace2fSDavid Howells for (int s = 0; s < NR_IO_STREAMS; s++)
488288ace2fSDavid Howells netfs_issue_write(wreq, &wreq->io_streams[s]);
489288ace2fSDavid Howells
490288ace2fSDavid Howells _leave(" = 0");
491288ace2fSDavid Howells return 0;
492288ace2fSDavid Howells }
493288ace2fSDavid Howells
494288ace2fSDavid Howells /*
495288ace2fSDavid Howells * Write some of the pending data back to the server
496288ace2fSDavid Howells */
netfs_writepages(struct address_space * mapping,struct writeback_control * wbc)4972df86547SDavid Howells int netfs_writepages(struct address_space *mapping,
498288ace2fSDavid Howells struct writeback_control *wbc)
499288ace2fSDavid Howells {
500288ace2fSDavid Howells struct netfs_inode *ictx = netfs_inode(mapping->host);
501288ace2fSDavid Howells struct netfs_io_request *wreq = NULL;
502288ace2fSDavid Howells struct folio *folio;
503288ace2fSDavid Howells int error = 0;
504288ace2fSDavid Howells
505288ace2fSDavid Howells if (wbc->sync_mode == WB_SYNC_ALL)
506288ace2fSDavid Howells mutex_lock(&ictx->wb_lock);
507288ace2fSDavid Howells else if (!mutex_trylock(&ictx->wb_lock))
508288ace2fSDavid Howells return 0;
509288ace2fSDavid Howells
510288ace2fSDavid Howells /* Need the first folio to be able to set up the op. */
511288ace2fSDavid Howells folio = writeback_iter(mapping, wbc, NULL, &error);
512288ace2fSDavid Howells if (!folio)
513288ace2fSDavid Howells goto out;
514288ace2fSDavid Howells
515288ace2fSDavid Howells wreq = netfs_create_write_req(mapping, NULL, folio_pos(folio), NETFS_WRITEBACK);
516288ace2fSDavid Howells if (IS_ERR(wreq)) {
517288ace2fSDavid Howells error = PTR_ERR(wreq);
518288ace2fSDavid Howells goto couldnt_start;
519288ace2fSDavid Howells }
520288ace2fSDavid Howells
521288ace2fSDavid Howells trace_netfs_write(wreq, netfs_write_trace_writeback);
5224824e591SDavid Howells netfs_stat(&netfs_n_wh_writepages);
523288ace2fSDavid Howells
524288ace2fSDavid Howells do {
525288ace2fSDavid Howells _debug("wbiter %lx %llx", folio->index, wreq->start + wreq->submitted);
526288ace2fSDavid Howells
527288ace2fSDavid Howells /* It appears we don't have to handle cyclic writeback wrapping. */
528288ace2fSDavid Howells WARN_ON_ONCE(wreq && folio_pos(folio) < wreq->start + wreq->submitted);
529288ace2fSDavid Howells
530288ace2fSDavid Howells if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE &&
531288ace2fSDavid Howells unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) {
532288ace2fSDavid Howells set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags);
533288ace2fSDavid Howells wreq->netfs_ops->begin_writeback(wreq);
534288ace2fSDavid Howells }
535288ace2fSDavid Howells
536288ace2fSDavid Howells error = netfs_write_folio(wreq, wbc, folio);
537288ace2fSDavid Howells if (error < 0)
538288ace2fSDavid Howells break;
539288ace2fSDavid Howells } while ((folio = writeback_iter(mapping, wbc, folio, &error)));
540288ace2fSDavid Howells
541288ace2fSDavid Howells for (int s = 0; s < NR_IO_STREAMS; s++)
542288ace2fSDavid Howells netfs_issue_write(wreq, &wreq->io_streams[s]);
543288ace2fSDavid Howells smp_wmb(); /* Write lists before ALL_QUEUED. */
544288ace2fSDavid Howells set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
545288ace2fSDavid Howells
546288ace2fSDavid Howells mutex_unlock(&ictx->wb_lock);
547288ace2fSDavid Howells
548288ace2fSDavid Howells netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
549288ace2fSDavid Howells _leave(" = %d", error);
550288ace2fSDavid Howells return error;
551288ace2fSDavid Howells
552288ace2fSDavid Howells couldnt_start:
553288ace2fSDavid Howells netfs_kill_dirty_pages(mapping, wbc, folio);
554288ace2fSDavid Howells out:
555288ace2fSDavid Howells mutex_unlock(&ictx->wb_lock);
556288ace2fSDavid Howells _leave(" = %d", error);
557288ace2fSDavid Howells return error;
558288ace2fSDavid Howells }
5592df86547SDavid Howells EXPORT_SYMBOL(netfs_writepages);
560288ace2fSDavid Howells
561288ace2fSDavid Howells /*
562288ace2fSDavid Howells * Begin a write operation for writing through the pagecache.
563288ace2fSDavid Howells */
netfs_begin_writethrough(struct kiocb * iocb,size_t len)5642df86547SDavid Howells struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len)
565288ace2fSDavid Howells {
566288ace2fSDavid Howells struct netfs_io_request *wreq = NULL;
567288ace2fSDavid Howells struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp));
568288ace2fSDavid Howells
569288ace2fSDavid Howells mutex_lock(&ictx->wb_lock);
570288ace2fSDavid Howells
571288ace2fSDavid Howells wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp,
572288ace2fSDavid Howells iocb->ki_pos, NETFS_WRITETHROUGH);
573288ace2fSDavid Howells if (IS_ERR(wreq)) {
574288ace2fSDavid Howells mutex_unlock(&ictx->wb_lock);
575288ace2fSDavid Howells return wreq;
576288ace2fSDavid Howells }
577288ace2fSDavid Howells
578288ace2fSDavid Howells wreq->io_streams[0].avail = true;
579288ace2fSDavid Howells trace_netfs_write(wreq, netfs_write_trace_writethrough);
580288ace2fSDavid Howells return wreq;
581288ace2fSDavid Howells }
582288ace2fSDavid Howells
583288ace2fSDavid Howells /*
584288ace2fSDavid Howells * Advance the state of the write operation used when writing through the
585288ace2fSDavid Howells * pagecache. Data has been copied into the pagecache that we need to append
586288ace2fSDavid Howells * to the request. If we've added more than wsize then we need to create a new
587288ace2fSDavid Howells * subrequest.
588288ace2fSDavid Howells */
netfs_advance_writethrough(struct netfs_io_request * wreq,struct writeback_control * wbc,struct folio * folio,size_t copied,bool to_page_end,struct folio ** writethrough_cache)5892df86547SDavid Howells int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
590288ace2fSDavid Howells struct folio *folio, size_t copied, bool to_page_end,
591288ace2fSDavid Howells struct folio **writethrough_cache)
592288ace2fSDavid Howells {
593288ace2fSDavid Howells _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u",
594288ace2fSDavid Howells wreq->debug_id, wreq->iter.count, wreq->wsize, copied, to_page_end);
595288ace2fSDavid Howells
596288ace2fSDavid Howells if (!*writethrough_cache) {
597288ace2fSDavid Howells if (folio_test_dirty(folio))
598288ace2fSDavid Howells /* Sigh. mmap. */
599288ace2fSDavid Howells folio_clear_dirty_for_io(folio);
600288ace2fSDavid Howells
601288ace2fSDavid Howells /* We can make multiple writes to the folio... */
602288ace2fSDavid Howells folio_start_writeback(folio);
603288ace2fSDavid Howells if (wreq->len == 0)
604288ace2fSDavid Howells trace_netfs_folio(folio, netfs_folio_trace_wthru);
605288ace2fSDavid Howells else
606288ace2fSDavid Howells trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
607288ace2fSDavid Howells *writethrough_cache = folio;
608288ace2fSDavid Howells }
609288ace2fSDavid Howells
610288ace2fSDavid Howells wreq->len += copied;
611288ace2fSDavid Howells if (!to_page_end)
612288ace2fSDavid Howells return 0;
613288ace2fSDavid Howells
614288ace2fSDavid Howells *writethrough_cache = NULL;
615288ace2fSDavid Howells return netfs_write_folio(wreq, wbc, folio);
616288ace2fSDavid Howells }
617288ace2fSDavid Howells
618288ace2fSDavid Howells /*
619288ace2fSDavid Howells * End a write operation used when writing through the pagecache.
620288ace2fSDavid Howells */
netfs_end_writethrough(struct netfs_io_request * wreq,struct writeback_control * wbc,struct folio * writethrough_cache)6212df86547SDavid Howells int netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
622288ace2fSDavid Howells struct folio *writethrough_cache)
623288ace2fSDavid Howells {
624288ace2fSDavid Howells struct netfs_inode *ictx = netfs_inode(wreq->inode);
625288ace2fSDavid Howells int ret;
626288ace2fSDavid Howells
627288ace2fSDavid Howells _enter("R=%x", wreq->debug_id);
628288ace2fSDavid Howells
629288ace2fSDavid Howells if (writethrough_cache)
630288ace2fSDavid Howells netfs_write_folio(wreq, wbc, writethrough_cache);
631288ace2fSDavid Howells
632288ace2fSDavid Howells netfs_issue_write(wreq, &wreq->io_streams[0]);
633288ace2fSDavid Howells netfs_issue_write(wreq, &wreq->io_streams[1]);
634288ace2fSDavid Howells smp_wmb(); /* Write lists before ALL_QUEUED. */
635288ace2fSDavid Howells set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
636288ace2fSDavid Howells
637288ace2fSDavid Howells mutex_unlock(&ictx->wb_lock);
638288ace2fSDavid Howells
639*2c6b5310SDavid Howells if (wreq->iocb) {
640*2c6b5310SDavid Howells ret = -EIOCBQUEUED;
641*2c6b5310SDavid Howells } else {
642*2c6b5310SDavid Howells wait_on_bit(&wreq->flags, NETFS_RREQ_IN_PROGRESS, TASK_UNINTERRUPTIBLE);
643288ace2fSDavid Howells ret = wreq->error;
644*2c6b5310SDavid Howells }
645288ace2fSDavid Howells netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
646288ace2fSDavid Howells return ret;
647288ace2fSDavid Howells }
648288ace2fSDavid Howells
649288ace2fSDavid Howells /*
650288ace2fSDavid Howells * Write data to the server without going through the pagecache and without
651288ace2fSDavid Howells * writing it to the local cache.
652288ace2fSDavid Howells */
netfs_unbuffered_write(struct netfs_io_request * wreq,bool may_wait,size_t len)653288ace2fSDavid Howells int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len)
654288ace2fSDavid Howells {
655288ace2fSDavid Howells struct netfs_io_stream *upload = &wreq->io_streams[0];
656288ace2fSDavid Howells ssize_t part;
657288ace2fSDavid Howells loff_t start = wreq->start;
658288ace2fSDavid Howells int error = 0;
659288ace2fSDavid Howells
660288ace2fSDavid Howells _enter("%zx", len);
661288ace2fSDavid Howells
662288ace2fSDavid Howells if (wreq->origin == NETFS_DIO_WRITE)
663288ace2fSDavid Howells inode_dio_begin(wreq->inode);
664288ace2fSDavid Howells
665288ace2fSDavid Howells while (len) {
666288ace2fSDavid Howells // TODO: Prepare content encryption
667288ace2fSDavid Howells
668288ace2fSDavid Howells _debug("unbuffered %zx", len);
669288ace2fSDavid Howells part = netfs_advance_write(wreq, upload, start, len, false);
670288ace2fSDavid Howells start += part;
671288ace2fSDavid Howells len -= part;
672288ace2fSDavid Howells if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) {
673288ace2fSDavid Howells trace_netfs_rreq(wreq, netfs_rreq_trace_wait_pause);
674288ace2fSDavid Howells wait_on_bit(&wreq->flags, NETFS_RREQ_PAUSE, TASK_UNINTERRUPTIBLE);
675288ace2fSDavid Howells }
676288ace2fSDavid Howells if (test_bit(NETFS_RREQ_FAILED, &wreq->flags))
677288ace2fSDavid Howells break;
678288ace2fSDavid Howells }
679288ace2fSDavid Howells
680288ace2fSDavid Howells netfs_issue_write(wreq, upload);
681288ace2fSDavid Howells
682288ace2fSDavid Howells smp_wmb(); /* Write lists before ALL_QUEUED. */
683288ace2fSDavid Howells set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
684288ace2fSDavid Howells if (list_empty(&upload->subrequests))
685288ace2fSDavid Howells netfs_wake_write_collector(wreq, false);
686288ace2fSDavid Howells
687288ace2fSDavid Howells _leave(" = %d", error);
688288ace2fSDavid Howells return error;
689288ace2fSDavid Howells }
690