1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem read subrequest retrying.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#include <linux/fs.h>
9#include <linux/slab.h>
10#include "internal.h"
11
12static void netfs_reissue_read(struct netfs_io_request *rreq,
13 struct netfs_io_subrequest *subreq)
14{
15 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
16 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
17 netfs_stat(&netfs_n_rh_retry_read_subreq);
18 subreq->rreq->netfs_ops->issue_read(subreq);
19}
20
21/*
22 * Go through the list of failed/short reads, retrying all retryable ones. We
23 * need to switch failed cache reads to network downloads.
24 */
25static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
26{
27 struct netfs_io_subrequest *subreq;
28 struct netfs_io_stream *stream = &rreq->io_streams[0];
29 struct list_head *next;
30
31 _enter("R=%x", rreq->debug_id);
32
33 if (list_empty(head: &stream->subrequests))
34 return;
35
36 if (rreq->netfs_ops->retry_request)
37 rreq->netfs_ops->retry_request(rreq, NULL);
38
39 /* If there's no renegotiation to do, just resend each retryable subreq
40 * up to the first permanently failed one.
41 */
42 if (!rreq->netfs_ops->prepare_read &&
43 !rreq->cache_resources.ops) {
44 list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
45 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
46 break;
47 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
48 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
49 subreq->retry_count++;
50 netfs_reset_iter(subreq);
51 netfs_get_subrequest(subreq, what: netfs_sreq_trace_get_resubmit);
52 netfs_reissue_read(rreq, subreq);
53 }
54 }
55 return;
56 }
57
58 /* Okay, we need to renegotiate all the download requests and flip any
59 * failed cache reads over to being download requests and negotiate
60 * those also. All fully successful subreqs have been removed from the
61 * list and any spare data from those has been donated.
62 *
63 * What we do is decant the list and rebuild it one subreq at a time so
64 * that we don't end up with donations jumping over a gap we're busy
65 * populating with smaller subrequests. In the event that the subreq
66 * we just launched finishes before we insert the next subreq, it'll
67 * fill in rreq->prev_donated instead.
68 *
69 * Note: Alternatively, we could split the tail subrequest right before
70 * we reissue it and fix up the donations under lock.
71 */
72 next = stream->subrequests.next;
73
74 do {
75 struct netfs_io_subrequest *from, *to, *tmp;
76 struct iov_iter source;
77 unsigned long long start, len;
78 size_t part;
79 bool boundary = false, subreq_superfluous = false;
80
81 /* Go through the subreqs and find the next span of contiguous
82 * buffer that we then rejig (cifs, for example, needs the
83 * rsize renegotiating) and reissue.
84 */
85 from = list_entry(next, struct netfs_io_subrequest, rreq_link);
86 to = from;
87 start = from->start + from->transferred;
88 len = from->len - from->transferred;
89
90 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx",
91 rreq->debug_id, from->debug_index,
92 from->start, from->transferred, from->len);
93
94 if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
95 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
96 goto abandon;
97
98 list_for_each_continue(next, &stream->subrequests) {
99 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
100 if (subreq->start + subreq->transferred != start + len ||
101 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
102 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
103 break;
104 to = subreq;
105 len += to->len;
106 }
107
108 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
109
110 /* Determine the set of buffers we're going to use. Each
111 * subreq gets a subset of a single overall contiguous buffer.
112 */
113 netfs_reset_iter(subreq: from);
114 source = from->io_iter;
115 source.count = len;
116
117 /* Work through the sublist. */
118 subreq = from;
119 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
120 if (!len) {
121 subreq_superfluous = true;
122 break;
123 }
124 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
125 subreq->start = start - subreq->transferred;
126 subreq->len = len + subreq->transferred;
127 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
128 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
129 subreq->retry_count++;
130
131 trace_netfs_sreq(sreq: subreq, what: netfs_sreq_trace_retry);
132
133 /* Renegotiate max_len (rsize) */
134 stream->sreq_max_len = subreq->len;
135 if (rreq->netfs_ops->prepare_read &&
136 rreq->netfs_ops->prepare_read(subreq) < 0) {
137 trace_netfs_sreq(sreq: subreq, what: netfs_sreq_trace_reprep_failed);
138 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
139 goto abandon;
140 }
141
142 part = umin(len, stream->sreq_max_len);
143 if (unlikely(stream->sreq_max_segs))
144 part = netfs_limit_iter(iter: &source, start_offset: 0, max_size: part, max_segs: stream->sreq_max_segs);
145 subreq->len = subreq->transferred + part;
146 subreq->io_iter = source;
147 iov_iter_truncate(i: &subreq->io_iter, count: part);
148 iov_iter_advance(i: &source, bytes: part);
149 len -= part;
150 start += part;
151 if (!len) {
152 if (boundary)
153 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
154 } else {
155 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
156 }
157
158 netfs_get_subrequest(subreq, what: netfs_sreq_trace_get_resubmit);
159 netfs_reissue_read(rreq, subreq);
160 if (subreq == to) {
161 subreq_superfluous = false;
162 break;
163 }
164 }
165
166 /* If we managed to use fewer subreqs, we can discard the
167 * excess; if we used the same number, then we're done.
168 */
169 if (!len) {
170 if (!subreq_superfluous)
171 continue;
172 list_for_each_entry_safe_from(subreq, tmp,
173 &stream->subrequests, rreq_link) {
174 trace_netfs_sreq(sreq: subreq, what: netfs_sreq_trace_superfluous);
175 list_del(entry: &subreq->rreq_link);
176 netfs_put_subrequest(subreq, what: netfs_sreq_trace_put_done);
177 if (subreq == to)
178 break;
179 }
180 continue;
181 }
182
183 /* We ran out of subrequests, so we need to allocate some more
184 * and insert them after.
185 */
186 do {
187 subreq = netfs_alloc_subrequest(rreq);
188 if (!subreq) {
189 subreq = to;
190 goto abandon_after;
191 }
192 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
193 subreq->start = start;
194 subreq->len = len;
195 subreq->stream_nr = stream->stream_nr;
196 subreq->retry_count = 1;
197
198 trace_netfs_sreq_ref(rreq_debug_id: rreq->debug_id, subreq_debug_index: subreq->debug_index,
199 ref: refcount_read(r: &subreq->ref),
200 what: netfs_sreq_trace_new);
201
202 list_add(new: &subreq->rreq_link, head: &to->rreq_link);
203 to = list_next_entry(to, rreq_link);
204 trace_netfs_sreq(sreq: subreq, what: netfs_sreq_trace_retry);
205
206 stream->sreq_max_len = umin(len, rreq->rsize);
207 stream->sreq_max_segs = 0;
208 if (unlikely(stream->sreq_max_segs))
209 part = netfs_limit_iter(iter: &source, start_offset: 0, max_size: part, max_segs: stream->sreq_max_segs);
210
211 netfs_stat(&netfs_n_rh_download);
212 if (rreq->netfs_ops->prepare_read(subreq) < 0) {
213 trace_netfs_sreq(sreq: subreq, what: netfs_sreq_trace_reprep_failed);
214 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
215 goto abandon;
216 }
217
218 part = umin(len, stream->sreq_max_len);
219 subreq->len = subreq->transferred + part;
220 subreq->io_iter = source;
221 iov_iter_truncate(i: &subreq->io_iter, count: part);
222 iov_iter_advance(i: &source, bytes: part);
223
224 len -= part;
225 start += part;
226 if (!len && boundary) {
227 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags);
228 boundary = false;
229 }
230
231 netfs_reissue_read(rreq, subreq);
232 } while (len);
233
234 } while (!list_is_head(list: next, head: &stream->subrequests));
235
236 return;
237
238 /* If we hit an error, fail all remaining incomplete subrequests */
239abandon_after:
240 if (list_is_last(list: &subreq->rreq_link, head: &stream->subrequests))
241 return;
242 subreq = list_next_entry(subreq, rreq_link);
243abandon:
244 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
245 if (!subreq->error &&
246 !test_bit(NETFS_SREQ_FAILED, &subreq->flags) &&
247 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
248 continue;
249 subreq->error = -ENOMEM;
250 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
251 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
252 }
253}
254
255/*
256 * Retry reads.
257 */
258void netfs_retry_reads(struct netfs_io_request *rreq)
259{
260 struct netfs_io_stream *stream = &rreq->io_streams[0];
261
262 netfs_stat(&netfs_n_rh_retry_read_req);
263
264 /* Wait for all outstanding I/O to quiesce before performing retries as
265 * we may need to renegotiate the I/O sizes.
266 */
267 set_bit(NETFS_RREQ_RETRYING, addr: &rreq->flags);
268 netfs_wait_for_in_progress_stream(rreq, stream);
269 clear_bit(NETFS_RREQ_RETRYING, addr: &rreq->flags);
270
271 trace_netfs_rreq(rreq, what: netfs_rreq_trace_resubmit);
272 netfs_retry_read_subrequests(rreq);
273}
274
275/*
276 * Unlock any the pages that haven't been unlocked yet due to abandoned
277 * subrequests.
278 */
279void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
280{
281 struct folio_queue *p;
282
283 for (p = rreq->buffer.tail; p; p = p->next) {
284 for (int slot = 0; slot < folioq_count(folioq: p); slot++) {
285 struct folio *folio = folioq_folio(folioq: p, slot);
286
287 if (folio && !folioq_is_marked2(folioq: p, slot)) {
288 trace_netfs_folio(folio, why: netfs_folio_trace_abandon);
289 folio_unlock(folio);
290 }
291 }
292 }
293}
294