1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2010 Red Hat, Inc.
4 * Copyright (c) 2016-2025 Christoph Hellwig.
5 */
6#include <linux/fscrypt.h>
7#include <linux/pagemap.h>
8#include <linux/iomap.h>
9#include <linux/task_io_accounting_ops.h>
10#include "internal.h"
11#include "trace.h"
12
13#include "../internal.h"
14
15/*
16 * Private flags for iomap_dio, must not overlap with the public ones in
17 * iomap.h:
18 */
19#define IOMAP_DIO_NO_INVALIDATE (1U << 25)
20#define IOMAP_DIO_CALLER_COMP (1U << 26)
21#define IOMAP_DIO_INLINE_COMP (1U << 27)
22#define IOMAP_DIO_WRITE_THROUGH (1U << 28)
23#define IOMAP_DIO_NEED_SYNC (1U << 29)
24#define IOMAP_DIO_WRITE (1U << 30)
25#define IOMAP_DIO_DIRTY (1U << 31)
26
27/*
28 * Used for sub block zeroing in iomap_dio_zero()
29 */
30#define IOMAP_ZERO_PAGE_SIZE (SZ_64K)
31#define IOMAP_ZERO_PAGE_ORDER (get_order(IOMAP_ZERO_PAGE_SIZE))
32static struct page *zero_page;
33
34struct iomap_dio {
35 struct kiocb *iocb;
36 const struct iomap_dio_ops *dops;
37 loff_t i_size;
38 loff_t size;
39 atomic_t ref;
40 unsigned flags;
41 int error;
42 size_t done_before;
43 bool wait_for_completion;
44
45 union {
46 /* used during submission and for synchronous completion: */
47 struct {
48 struct iov_iter *iter;
49 struct task_struct *waiter;
50 } submit;
51
52 /* used for aio completion: */
53 struct {
54 struct work_struct work;
55 } aio;
56 };
57};
58
59static struct bio *iomap_dio_alloc_bio(const struct iomap_iter *iter,
60 struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)
61{
62 if (dio->dops && dio->dops->bio_set)
63 return bio_alloc_bioset(bdev: iter->iomap.bdev, nr_vecs, opf,
64 GFP_KERNEL, bs: dio->dops->bio_set);
65 return bio_alloc(bdev: iter->iomap.bdev, nr_vecs, opf, GFP_KERNEL);
66}
67
68static void iomap_dio_submit_bio(const struct iomap_iter *iter,
69 struct iomap_dio *dio, struct bio *bio, loff_t pos)
70{
71 struct kiocb *iocb = dio->iocb;
72
73 atomic_inc(v: &dio->ref);
74
75 /* Sync dio can't be polled reliably */
76 if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(kiocb: iocb)) {
77 bio_set_polled(bio, kiocb: iocb);
78 WRITE_ONCE(iocb->private, bio);
79 }
80
81 if (dio->dops && dio->dops->submit_io) {
82 dio->dops->submit_io(iter, bio, pos);
83 } else {
84 WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_ANON_WRITE);
85 submit_bio(bio);
86 }
87}
88
89ssize_t iomap_dio_complete(struct iomap_dio *dio)
90{
91 const struct iomap_dio_ops *dops = dio->dops;
92 struct kiocb *iocb = dio->iocb;
93 loff_t offset = iocb->ki_pos;
94 ssize_t ret = dio->error;
95
96 if (dops && dops->end_io)
97 ret = dops->end_io(iocb, dio->size, ret, dio->flags);
98
99 if (likely(!ret)) {
100 ret = dio->size;
101 /* check for short read */
102 if (offset + ret > dio->i_size &&
103 !(dio->flags & IOMAP_DIO_WRITE))
104 ret = dio->i_size - offset;
105 }
106
107 /*
108 * Try again to invalidate clean pages which might have been cached by
109 * non-direct readahead, or faulted in by get_user_pages() if the source
110 * of the write was an mmap'ed region of the file we're writing. Either
111 * one is a pretty crazy thing to do, so we don't support it 100%. If
112 * this invalidation fails, tough, the write still worked...
113 *
114 * And this page cache invalidation has to be after ->end_io(), as some
115 * filesystems convert unwritten extents to real allocations in
116 * ->end_io() when necessary, otherwise a racing buffer read would cache
117 * zeros from unwritten extents.
118 */
119 if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE) &&
120 !(dio->flags & IOMAP_DIO_NO_INVALIDATE))
121 kiocb_invalidate_post_direct_write(iocb, count: dio->size);
122
123 inode_dio_end(inode: file_inode(f: iocb->ki_filp));
124
125 if (ret > 0) {
126 iocb->ki_pos += ret;
127
128 /*
129 * If this is a DSYNC write, make sure we push it to stable
130 * storage now that we've written data.
131 */
132 if (dio->flags & IOMAP_DIO_NEED_SYNC)
133 ret = generic_write_sync(iocb, count: ret);
134 if (ret > 0)
135 ret += dio->done_before;
136 }
137 trace_iomap_dio_complete(iocb, error: dio->error, ret);
138 kfree(objp: dio);
139 return ret;
140}
141EXPORT_SYMBOL_GPL(iomap_dio_complete);
142
143static ssize_t iomap_dio_deferred_complete(void *data)
144{
145 return iomap_dio_complete(data);
146}
147
148static void iomap_dio_complete_work(struct work_struct *work)
149{
150 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
151 struct kiocb *iocb = dio->iocb;
152
153 iocb->ki_complete(iocb, iomap_dio_complete(dio));
154}
155
156/*
157 * Set an error in the dio if none is set yet. We have to use cmpxchg
158 * as the submission context and the completion context(s) can race to
159 * update the error.
160 */
161static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
162{
163 cmpxchg(&dio->error, 0, ret);
164}
165
166/*
167 * Called when dio->ref reaches zero from an I/O completion.
168 */
169static void iomap_dio_done(struct iomap_dio *dio)
170{
171 struct kiocb *iocb = dio->iocb;
172
173 if (dio->wait_for_completion) {
174 /*
175 * Synchronous I/O, task itself will handle any completion work
176 * that needs after IO. All we need to do is wake the task.
177 */
178 struct task_struct *waiter = dio->submit.waiter;
179
180 WRITE_ONCE(dio->submit.waiter, NULL);
181 blk_wake_io_task(waiter);
182 } else if (dio->flags & IOMAP_DIO_INLINE_COMP) {
183 WRITE_ONCE(iocb->private, NULL);
184 iomap_dio_complete_work(work: &dio->aio.work);
185 } else if (dio->flags & IOMAP_DIO_CALLER_COMP) {
186 /*
187 * If this dio is flagged with IOMAP_DIO_CALLER_COMP, then
188 * schedule our completion that way to avoid an async punt to a
189 * workqueue.
190 */
191 /* only polled IO cares about private cleared */
192 iocb->private = dio;
193 iocb->dio_complete = iomap_dio_deferred_complete;
194
195 /*
196 * Invoke ->ki_complete() directly. We've assigned our
197 * dio_complete callback handler, and since the issuer set
198 * IOCB_DIO_CALLER_COMP, we know their ki_complete handler will
199 * notice ->dio_complete being set and will defer calling that
200 * handler until it can be done from a safe task context.
201 *
202 * Note that the 'res' being passed in here is not important
203 * for this case. The actual completion value of the request
204 * will be gotten from dio_complete when that is run by the
205 * issuer.
206 */
207 iocb->ki_complete(iocb, 0);
208 } else {
209 struct inode *inode = file_inode(f: iocb->ki_filp);
210
211 /*
212 * Async DIO completion that requires filesystem level
213 * completion work gets punted to a work queue to complete as
214 * the operation may require more IO to be issued to finalise
215 * filesystem metadata changes or guarantee data integrity.
216 */
217 INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
218 queue_work(wq: inode->i_sb->s_dio_done_wq, work: &dio->aio.work);
219 }
220}
221
222void iomap_dio_bio_end_io(struct bio *bio)
223{
224 struct iomap_dio *dio = bio->bi_private;
225 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
226
227 if (bio->bi_status)
228 iomap_dio_set_error(dio, ret: blk_status_to_errno(status: bio->bi_status));
229
230 if (atomic_dec_and_test(v: &dio->ref))
231 iomap_dio_done(dio);
232
233 if (should_dirty) {
234 bio_check_pages_dirty(bio);
235 } else {
236 bio_release_pages(bio, mark_dirty: false);
237 bio_put(bio);
238 }
239}
240EXPORT_SYMBOL_GPL(iomap_dio_bio_end_io);
241
242u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend)
243{
244 struct iomap_dio *dio = ioend->io_bio.bi_private;
245 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
246 u32 vec_count = ioend->io_bio.bi_vcnt;
247
248 if (ioend->io_error)
249 iomap_dio_set_error(dio, ret: ioend->io_error);
250
251 if (atomic_dec_and_test(v: &dio->ref)) {
252 /*
253 * Try to avoid another context switch for the completion given
254 * that we are already called from the ioend completion
255 * workqueue, but never invalidate pages from this thread to
256 * avoid deadlocks with buffered I/O completions. Tough luck if
257 * you hit the tiny race with someone dirtying the range now
258 * between this check and the actual completion.
259 */
260 if (!dio->iocb->ki_filp->f_mapping->nrpages) {
261 dio->flags |= IOMAP_DIO_INLINE_COMP;
262 dio->flags |= IOMAP_DIO_NO_INVALIDATE;
263 }
264 dio->flags &= ~IOMAP_DIO_CALLER_COMP;
265 iomap_dio_done(dio);
266 }
267
268 if (should_dirty) {
269 bio_check_pages_dirty(bio: &ioend->io_bio);
270 } else {
271 bio_release_pages(bio: &ioend->io_bio, mark_dirty: false);
272 bio_put(&ioend->io_bio);
273 }
274
275 /*
276 * Return the number of bvecs completed as even direct I/O completions
277 * do significant per-folio work and we'll still want to give up the
278 * CPU after a lot of completions.
279 */
280 return vec_count;
281}
282
283static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
284 loff_t pos, unsigned len)
285{
286 struct inode *inode = file_inode(f: dio->iocb->ki_filp);
287 struct bio *bio;
288
289 if (!len)
290 return 0;
291 /*
292 * Max block size supported is 64k
293 */
294 if (WARN_ON_ONCE(len > IOMAP_ZERO_PAGE_SIZE))
295 return -EINVAL;
296
297 bio = iomap_dio_alloc_bio(iter, dio, nr_vecs: 1, opf: REQ_OP_WRITE | REQ_SYNC | REQ_IDLE);
298 fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk: pos >> inode->i_blkbits,
299 GFP_KERNEL);
300 bio->bi_iter.bi_sector = iomap_sector(iomap: &iter->iomap, pos);
301 bio->bi_private = dio;
302 bio->bi_end_io = iomap_dio_bio_end_io;
303
304 __bio_add_page(bio, page: zero_page, len, off: 0);
305 iomap_dio_submit_bio(iter, dio, bio, pos);
306 return 0;
307}
308
309/*
310 * Use a FUA write if we need datasync semantics and this is a pure data I/O
311 * that doesn't require any metadata updates (including after I/O completion
312 * such as unwritten extent conversion) and the underlying device either
313 * doesn't have a volatile write cache or supports FUA.
314 * This allows us to avoid cache flushes on I/O completion.
315 */
316static inline bool iomap_dio_can_use_fua(const struct iomap *iomap,
317 struct iomap_dio *dio)
318{
319 if (iomap->flags & (IOMAP_F_SHARED | IOMAP_F_DIRTY))
320 return false;
321 if (!(dio->flags & IOMAP_DIO_WRITE_THROUGH))
322 return false;
323 return !bdev_write_cache(bdev: iomap->bdev) || bdev_fua(bdev: iomap->bdev);
324}
325
326static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
327{
328 const struct iomap *iomap = &iter->iomap;
329 struct inode *inode = iter->inode;
330 unsigned int fs_block_size = i_blocksize(node: inode), pad;
331 const loff_t length = iomap_length(iter);
332 loff_t pos = iter->pos;
333 blk_opf_t bio_opf = REQ_SYNC | REQ_IDLE;
334 struct bio *bio;
335 bool need_zeroout = false;
336 int nr_pages, ret = 0;
337 u64 copied = 0;
338 size_t orig_count;
339
340 if ((pos | length) & (bdev_logical_block_size(bdev: iomap->bdev) - 1))
341 return -EINVAL;
342
343 if (dio->flags & IOMAP_DIO_WRITE) {
344 bio_opf |= REQ_OP_WRITE;
345
346 if (iomap->flags & IOMAP_F_ATOMIC_BIO) {
347 /*
348 * Ensure that the mapping covers the full write
349 * length, otherwise it won't be submitted as a single
350 * bio, which is required to use hardware atomics.
351 */
352 if (length != iter->len)
353 return -EINVAL;
354 bio_opf |= REQ_ATOMIC;
355 }
356
357 if (iomap->type == IOMAP_UNWRITTEN) {
358 dio->flags |= IOMAP_DIO_UNWRITTEN;
359 need_zeroout = true;
360 }
361
362 if (iomap->flags & IOMAP_F_SHARED)
363 dio->flags |= IOMAP_DIO_COW;
364
365 if (iomap->flags & IOMAP_F_NEW)
366 need_zeroout = true;
367 else if (iomap->type == IOMAP_MAPPED &&
368 iomap_dio_can_use_fua(iomap, dio))
369 bio_opf |= REQ_FUA;
370
371 if (!(bio_opf & REQ_FUA))
372 dio->flags &= ~IOMAP_DIO_WRITE_THROUGH;
373
374 /*
375 * We can only do deferred completion for pure overwrites that
376 * don't require additional I/O at completion time.
377 *
378 * This rules out writes that need zeroing or extent conversion,
379 * extend the file size, or issue metadata I/O or cache flushes
380 * during completion processing.
381 */
382 if (need_zeroout || (pos >= i_size_read(inode)) ||
383 ((dio->flags & IOMAP_DIO_NEED_SYNC) &&
384 !(bio_opf & REQ_FUA)))
385 dio->flags &= ~IOMAP_DIO_CALLER_COMP;
386 } else {
387 bio_opf |= REQ_OP_READ;
388 }
389
390 /*
391 * Save the original count and trim the iter to just the extent we
392 * are operating on right now. The iter will be re-expanded once
393 * we are done.
394 */
395 orig_count = iov_iter_count(i: dio->submit.iter);
396 iov_iter_truncate(i: dio->submit.iter, count: length);
397
398 if (!iov_iter_count(i: dio->submit.iter))
399 goto out;
400
401 /*
402 * The rules for polled IO completions follow the guidelines as the
403 * ones we set for inline and deferred completions. If none of those
404 * are available for this IO, clear the polled flag.
405 */
406 if (!(dio->flags & (IOMAP_DIO_INLINE_COMP|IOMAP_DIO_CALLER_COMP)))
407 dio->iocb->ki_flags &= ~IOCB_HIPRI;
408
409 if (need_zeroout) {
410 /* zero out from the start of the block to the write offset */
411 pad = pos & (fs_block_size - 1);
412
413 ret = iomap_dio_zero(iter, dio, pos: pos - pad, len: pad);
414 if (ret)
415 goto out;
416 }
417
418 nr_pages = bio_iov_vecs_to_alloc(iter: dio->submit.iter, BIO_MAX_VECS);
419 do {
420 size_t n;
421 if (dio->error) {
422 iov_iter_revert(i: dio->submit.iter, bytes: copied);
423 copied = ret = 0;
424 goto out;
425 }
426
427 bio = iomap_dio_alloc_bio(iter, dio, nr_vecs: nr_pages, opf: bio_opf);
428 fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk: pos >> inode->i_blkbits,
429 GFP_KERNEL);
430 bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
431 bio->bi_write_hint = inode->i_write_hint;
432 bio->bi_ioprio = dio->iocb->ki_ioprio;
433 bio->bi_private = dio;
434 bio->bi_end_io = iomap_dio_bio_end_io;
435
436 ret = bio_iov_iter_get_pages(bio, iter: dio->submit.iter,
437 len_align_mask: bdev_logical_block_size(bdev: iomap->bdev) - 1);
438 if (unlikely(ret)) {
439 /*
440 * We have to stop part way through an IO. We must fall
441 * through to the sub-block tail zeroing here, otherwise
442 * this short IO may expose stale data in the tail of
443 * the block we haven't written data to.
444 */
445 bio_put(bio);
446 goto zero_tail;
447 }
448
449 n = bio->bi_iter.bi_size;
450 if (WARN_ON_ONCE((bio_opf & REQ_ATOMIC) && n != length)) {
451 /*
452 * An atomic write bio must cover the complete length,
453 * which it doesn't, so error. We may need to zero out
454 * the tail (complete FS block), similar to when
455 * bio_iov_iter_get_pages() returns an error, above.
456 */
457 ret = -EINVAL;
458 bio_put(bio);
459 goto zero_tail;
460 }
461 if (dio->flags & IOMAP_DIO_WRITE)
462 task_io_account_write(bytes: n);
463 else if (dio->flags & IOMAP_DIO_DIRTY)
464 bio_set_pages_dirty(bio);
465
466 dio->size += n;
467 copied += n;
468
469 nr_pages = bio_iov_vecs_to_alloc(iter: dio->submit.iter,
470 BIO_MAX_VECS);
471 /*
472 * We can only poll for single bio I/Os.
473 */
474 if (nr_pages)
475 dio->iocb->ki_flags &= ~IOCB_HIPRI;
476 iomap_dio_submit_bio(iter, dio, bio, pos);
477 pos += n;
478 } while (nr_pages);
479
480 /*
481 * We need to zeroout the tail of a sub-block write if the extent type
482 * requires zeroing or the write extends beyond EOF. If we don't zero
483 * the block tail in the latter case, we can expose stale data via mmap
484 * reads of the EOF block.
485 */
486zero_tail:
487 if (need_zeroout ||
488 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
489 /* zero out from the end of the write to the end of the block */
490 pad = pos & (fs_block_size - 1);
491 if (pad)
492 ret = iomap_dio_zero(iter, dio, pos,
493 len: fs_block_size - pad);
494 }
495out:
496 /* Undo iter limitation to current extent */
497 iov_iter_reexpand(i: dio->submit.iter, count: orig_count - copied);
498 if (copied)
499 return iomap_iter_advance(iter, count: &copied);
500 return ret;
501}
502
503static int iomap_dio_hole_iter(struct iomap_iter *iter, struct iomap_dio *dio)
504{
505 loff_t length = iov_iter_zero(bytes: iomap_length(iter), dio->submit.iter);
506
507 dio->size += length;
508 if (!length)
509 return -EFAULT;
510 return iomap_iter_advance(iter, count: &length);
511}
512
513static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio)
514{
515 const struct iomap *iomap = &iomi->iomap;
516 struct iov_iter *iter = dio->submit.iter;
517 void *inline_data = iomap_inline_data(iomap, pos: iomi->pos);
518 loff_t length = iomap_length(iter: iomi);
519 loff_t pos = iomi->pos;
520 u64 copied;
521
522 if (WARN_ON_ONCE(!inline_data))
523 return -EIO;
524
525 if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap)))
526 return -EIO;
527
528 if (dio->flags & IOMAP_DIO_WRITE) {
529 loff_t size = iomi->inode->i_size;
530
531 if (pos > size)
532 memset(s: iomap_inline_data(iomap, pos: size), c: 0, n: pos - size);
533 copied = copy_from_iter(addr: inline_data, bytes: length, i: iter);
534 if (copied) {
535 if (pos + copied > size)
536 i_size_write(inode: iomi->inode, i_size: pos + copied);
537 mark_inode_dirty(inode: iomi->inode);
538 }
539 } else {
540 copied = copy_to_iter(addr: inline_data, bytes: length, i: iter);
541 }
542 dio->size += copied;
543 if (!copied)
544 return -EFAULT;
545 return iomap_iter_advance(iter: iomi, count: &copied);
546}
547
548static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
549{
550 switch (iter->iomap.type) {
551 case IOMAP_HOLE:
552 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
553 return -EIO;
554 return iomap_dio_hole_iter(iter, dio);
555 case IOMAP_UNWRITTEN:
556 if (!(dio->flags & IOMAP_DIO_WRITE))
557 return iomap_dio_hole_iter(iter, dio);
558 return iomap_dio_bio_iter(iter, dio);
559 case IOMAP_MAPPED:
560 return iomap_dio_bio_iter(iter, dio);
561 case IOMAP_INLINE:
562 return iomap_dio_inline_iter(iomi: iter, dio);
563 case IOMAP_DELALLOC:
564 /*
565 * DIO is not serialised against mmap() access at all, and so
566 * if the page_mkwrite occurs between the writeback and the
567 * iomap_iter() call in the DIO path, then it will see the
568 * DELALLOC block that the page-mkwrite allocated.
569 */
570 pr_warn_ratelimited("Direct I/O collision with buffered writes! File: %pD4 Comm: %.20s\n",
571 dio->iocb->ki_filp, current->comm);
572 return -EIO;
573 default:
574 WARN_ON_ONCE(1);
575 return -EIO;
576 }
577}
578
579/*
580 * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
581 * is being issued as AIO or not. This allows us to optimise pure data writes
582 * to use REQ_FUA rather than requiring generic_write_sync() to issue a
583 * REQ_FLUSH post write. This is slightly tricky because a single request here
584 * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
585 * may be pure data writes. In that case, we still need to do a full data sync
586 * completion.
587 *
588 * When page faults are disabled and @dio_flags includes IOMAP_DIO_PARTIAL,
589 * __iomap_dio_rw can return a partial result if it encounters a non-resident
590 * page in @iter after preparing a transfer. In that case, the non-resident
591 * pages can be faulted in and the request resumed with @done_before set to the
592 * number of bytes previously transferred. The request will then complete with
593 * the correct total number of bytes transferred; this is essential for
594 * completing partial requests asynchronously.
595 *
596 * Returns -ENOTBLK In case of a page invalidation invalidation failure for
597 * writes. The callers needs to fall back to buffered I/O in this case.
598 */
599struct iomap_dio *
600__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
601 const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
602 unsigned int dio_flags, void *private, size_t done_before)
603{
604 struct inode *inode = file_inode(f: iocb->ki_filp);
605 struct iomap_iter iomi = {
606 .inode = inode,
607 .pos = iocb->ki_pos,
608 .len = iov_iter_count(i: iter),
609 .flags = IOMAP_DIRECT,
610 .private = private,
611 };
612 bool wait_for_completion =
613 is_sync_kiocb(kiocb: iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT);
614 struct blk_plug plug;
615 struct iomap_dio *dio;
616 loff_t ret = 0;
617
618 trace_iomap_dio_rw_begin(iocb, iter, dio_flags, done_before);
619
620 if (!iomi.len)
621 return NULL;
622
623 dio = kmalloc(sizeof(*dio), GFP_KERNEL);
624 if (!dio)
625 return ERR_PTR(error: -ENOMEM);
626
627 dio->iocb = iocb;
628 atomic_set(v: &dio->ref, i: 1);
629 dio->size = 0;
630 dio->i_size = i_size_read(inode);
631 dio->dops = dops;
632 dio->error = 0;
633 dio->flags = 0;
634 dio->done_before = done_before;
635
636 dio->submit.iter = iter;
637 dio->submit.waiter = current;
638
639 if (iocb->ki_flags & IOCB_NOWAIT)
640 iomi.flags |= IOMAP_NOWAIT;
641
642 if (iov_iter_rw(i: iter) == READ) {
643 /* reads can always complete inline */
644 dio->flags |= IOMAP_DIO_INLINE_COMP;
645
646 if (iomi.pos >= dio->i_size)
647 goto out_free_dio;
648
649 if (user_backed_iter(i: iter))
650 dio->flags |= IOMAP_DIO_DIRTY;
651
652 ret = kiocb_write_and_wait(iocb, count: iomi.len);
653 if (ret)
654 goto out_free_dio;
655 } else {
656 iomi.flags |= IOMAP_WRITE;
657 dio->flags |= IOMAP_DIO_WRITE;
658
659 /*
660 * Flag as supporting deferred completions, if the issuer
661 * groks it. This can avoid a workqueue punt for writes.
662 * We may later clear this flag if we need to do other IO
663 * as part of this IO completion.
664 */
665 if (iocb->ki_flags & IOCB_DIO_CALLER_COMP)
666 dio->flags |= IOMAP_DIO_CALLER_COMP;
667
668 if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) {
669 ret = -EAGAIN;
670 if (iomi.pos >= dio->i_size ||
671 iomi.pos + iomi.len > dio->i_size)
672 goto out_free_dio;
673 iomi.flags |= IOMAP_OVERWRITE_ONLY;
674 }
675
676 if (iocb->ki_flags & IOCB_ATOMIC)
677 iomi.flags |= IOMAP_ATOMIC;
678
679 /* for data sync or sync, we need sync completion processing */
680 if (iocb_is_dsync(iocb)) {
681 dio->flags |= IOMAP_DIO_NEED_SYNC;
682
683 /*
684 * For datasync only writes, we optimistically try using
685 * WRITE_THROUGH for this IO. This flag requires either
686 * FUA writes through the device's write cache, or a
687 * normal write to a device without a volatile write
688 * cache. For the former, Any non-FUA write that occurs
689 * will clear this flag, hence we know before completion
690 * whether a cache flush is necessary.
691 */
692 if (!(iocb->ki_flags & IOCB_SYNC))
693 dio->flags |= IOMAP_DIO_WRITE_THROUGH;
694 }
695
696 /*
697 * Try to invalidate cache pages for the range we are writing.
698 * If this invalidation fails, let the caller fall back to
699 * buffered I/O.
700 */
701 ret = kiocb_invalidate_pages(iocb, count: iomi.len);
702 if (ret) {
703 if (ret != -EAGAIN) {
704 trace_iomap_dio_invalidate_fail(inode, off: iomi.pos,
705 len: iomi.len);
706 if (iocb->ki_flags & IOCB_ATOMIC) {
707 /*
708 * folio invalidation failed, maybe
709 * this is transient, unlock and see if
710 * the caller tries again.
711 */
712 ret = -EAGAIN;
713 } else {
714 /* fall back to buffered write */
715 ret = -ENOTBLK;
716 }
717 }
718 goto out_free_dio;
719 }
720
721 if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) {
722 ret = sb_init_dio_done_wq(sb: inode->i_sb);
723 if (ret < 0)
724 goto out_free_dio;
725 }
726 }
727
728 inode_dio_begin(inode);
729
730 blk_start_plug(&plug);
731 while ((ret = iomap_iter(iter: &iomi, ops)) > 0) {
732 iomi.status = iomap_dio_iter(iter: &iomi, dio);
733
734 /*
735 * We can only poll for single bio I/Os.
736 */
737 iocb->ki_flags &= ~IOCB_HIPRI;
738 }
739
740 blk_finish_plug(&plug);
741
742 /*
743 * We only report that we've read data up to i_size.
744 * Revert iter to a state corresponding to that as some callers (such
745 * as the splice code) rely on it.
746 */
747 if (iov_iter_rw(i: iter) == READ && iomi.pos >= dio->i_size)
748 iov_iter_revert(i: iter, bytes: iomi.pos - dio->i_size);
749
750 if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {
751 if (!(iocb->ki_flags & IOCB_NOWAIT))
752 wait_for_completion = true;
753 ret = 0;
754 }
755
756 /* magic error code to fall back to buffered I/O */
757 if (ret == -ENOTBLK) {
758 wait_for_completion = true;
759 ret = 0;
760 }
761 if (ret < 0)
762 iomap_dio_set_error(dio, ret);
763
764 /*
765 * If all the writes we issued were already written through to the
766 * media, we don't need to flush the cache on IO completion. Clear the
767 * sync flag for this case.
768 */
769 if (dio->flags & IOMAP_DIO_WRITE_THROUGH)
770 dio->flags &= ~IOMAP_DIO_NEED_SYNC;
771
772 /*
773 * We are about to drop our additional submission reference, which
774 * might be the last reference to the dio. There are three different
775 * ways we can progress here:
776 *
777 * (a) If this is the last reference we will always complete and free
778 * the dio ourselves.
779 * (b) If this is not the last reference, and we serve an asynchronous
780 * iocb, we must never touch the dio after the decrement, the
781 * I/O completion handler will complete and free it.
782 * (c) If this is not the last reference, but we serve a synchronous
783 * iocb, the I/O completion handler will wake us up on the drop
784 * of the final reference, and we will complete and free it here
785 * after we got woken by the I/O completion handler.
786 */
787 dio->wait_for_completion = wait_for_completion;
788 if (!atomic_dec_and_test(v: &dio->ref)) {
789 if (!wait_for_completion) {
790 trace_iomap_dio_rw_queued(inode, off: iomi.pos, len: iomi.len);
791 return ERR_PTR(error: -EIOCBQUEUED);
792 }
793
794 for (;;) {
795 set_current_state(TASK_UNINTERRUPTIBLE);
796 if (!READ_ONCE(dio->submit.waiter))
797 break;
798
799 blk_io_schedule();
800 }
801 __set_current_state(TASK_RUNNING);
802 }
803
804 return dio;
805
806out_free_dio:
807 kfree(objp: dio);
808 if (ret)
809 return ERR_PTR(error: ret);
810 return NULL;
811}
812EXPORT_SYMBOL_GPL(__iomap_dio_rw);
813
814ssize_t
815iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
816 const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
817 unsigned int dio_flags, void *private, size_t done_before)
818{
819 struct iomap_dio *dio;
820
821 dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,
822 done_before);
823 if (IS_ERR_OR_NULL(ptr: dio))
824 return PTR_ERR_OR_ZERO(ptr: dio);
825 return iomap_dio_complete(dio);
826}
827EXPORT_SYMBOL_GPL(iomap_dio_rw);
828
829static int __init iomap_dio_init(void)
830{
831 zero_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
832 IOMAP_ZERO_PAGE_ORDER);
833
834 if (!zero_page)
835 return -ENOMEM;
836
837 return 0;
838}
839fs_initcall(iomap_dio_init);
840