1// SPDX-License-Identifier: GPL-2.0
2/*
3 * I/O and data path helper functionality.
4 *
5 * Borrowed from NFS Copyright (c) 2016 Trond Myklebust
6 */
7
8#include <linux/kernel.h>
9#include <linux/netfs.h>
10#include "internal.h"
11
12/*
13 * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish
14 * @inode: inode to wait for
15 *
16 * Waits for all pending direct I/O requests to finish so that we can
17 * proceed with a truncate or equivalent operation.
18 *
19 * Must be called under a lock that serializes taking new references
20 * to i_dio_count, usually by inode->i_mutex.
21 */
22static int netfs_inode_dio_wait_interruptible(struct inode *inode)
23{
24 if (inode_dio_finished(inode))
25 return 0;
26
27 inode_dio_wait_interruptible(inode);
28 return !inode_dio_finished(inode) ? -ERESTARTSYS : 0;
29}
30
31/* Call with exclusively locked inode->i_rwsem */
32static int netfs_block_o_direct(struct netfs_inode *ictx)
33{
34 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags))
35 return 0;
36 clear_bit(NETFS_ICTX_ODIRECT, addr: &ictx->flags);
37 return netfs_inode_dio_wait_interruptible(inode: &ictx->inode);
38}
39
40/**
41 * netfs_start_io_read - declare the file is being used for buffered reads
42 * @inode: file inode
43 *
44 * Declare that a buffered read operation is about to start, and ensure
45 * that we block all direct I/O.
46 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset,
47 * and holds a shared lock on inode->i_rwsem to ensure that the flag
48 * cannot be changed.
49 * In practice, this means that buffered read operations are allowed to
50 * execute in parallel, thanks to the shared lock, whereas direct I/O
51 * operations need to wait to grab an exclusive lock in order to set
52 * NETFS_ICTX_ODIRECT.
53 * Note that buffered writes and truncates both take a write lock on
54 * inode->i_rwsem, meaning that those are serialised w.r.t. the reads.
55 */
56int netfs_start_io_read(struct inode *inode)
57 __acquires(inode->i_rwsem)
58{
59 struct netfs_inode *ictx = netfs_inode(inode);
60
61 /* Be an optimist! */
62 if (down_read_interruptible(sem: &inode->i_rwsem) < 0)
63 return -ERESTARTSYS;
64 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0)
65 return 0;
66 up_read(sem: &inode->i_rwsem);
67
68 /* Slow path.... */
69 if (down_write_killable(sem: &inode->i_rwsem) < 0)
70 return -ERESTARTSYS;
71 if (netfs_block_o_direct(ictx) < 0) {
72 up_write(sem: &inode->i_rwsem);
73 return -ERESTARTSYS;
74 }
75 downgrade_write(sem: &inode->i_rwsem);
76 return 0;
77}
78EXPORT_SYMBOL(netfs_start_io_read);
79
80/**
81 * netfs_end_io_read - declare that the buffered read operation is done
82 * @inode: file inode
83 *
84 * Declare that a buffered read operation is done, and release the shared
85 * lock on inode->i_rwsem.
86 */
87void netfs_end_io_read(struct inode *inode)
88 __releases(inode->i_rwsem)
89{
90 up_read(sem: &inode->i_rwsem);
91}
92EXPORT_SYMBOL(netfs_end_io_read);
93
94/**
95 * netfs_start_io_write - declare the file is being used for buffered writes
96 * @inode: file inode
97 *
98 * Declare that a buffered read operation is about to start, and ensure
99 * that we block all direct I/O.
100 */
101int netfs_start_io_write(struct inode *inode)
102 __acquires(inode->i_rwsem)
103{
104 struct netfs_inode *ictx = netfs_inode(inode);
105
106 if (down_write_killable(sem: &inode->i_rwsem) < 0)
107 return -ERESTARTSYS;
108 if (netfs_block_o_direct(ictx) < 0) {
109 up_write(sem: &inode->i_rwsem);
110 return -ERESTARTSYS;
111 }
112 downgrade_write(sem: &inode->i_rwsem);
113 return 0;
114}
115EXPORT_SYMBOL(netfs_start_io_write);
116
117/**
118 * netfs_end_io_write - declare that the buffered write operation is done
119 * @inode: file inode
120 *
121 * Declare that a buffered write operation is done, and release the
122 * lock on inode->i_rwsem.
123 */
124void netfs_end_io_write(struct inode *inode)
125 __releases(inode->i_rwsem)
126{
127 up_read(sem: &inode->i_rwsem);
128}
129EXPORT_SYMBOL(netfs_end_io_write);
130
131/* Call with exclusively locked inode->i_rwsem */
132static int netfs_block_buffered(struct inode *inode)
133{
134 struct netfs_inode *ictx = netfs_inode(inode);
135 int ret;
136
137 if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) {
138 set_bit(NETFS_ICTX_ODIRECT, addr: &ictx->flags);
139 if (inode->i_mapping->nrpages != 0) {
140 unmap_mapping_range(mapping: inode->i_mapping, holebegin: 0, holelen: 0, even_cows: 0);
141 ret = filemap_fdatawait(mapping: inode->i_mapping);
142 if (ret < 0) {
143 clear_bit(NETFS_ICTX_ODIRECT, addr: &ictx->flags);
144 return ret;
145 }
146 }
147 }
148 return 0;
149}
150
151/**
152 * netfs_start_io_direct - declare the file is being used for direct i/o
153 * @inode: file inode
154 *
155 * Declare that a direct I/O operation is about to start, and ensure
156 * that we block all buffered I/O.
157 * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set,
158 * and holds a shared lock on inode->i_rwsem to ensure that the flag
159 * cannot be changed.
160 * In practice, this means that direct I/O operations are allowed to
161 * execute in parallel, thanks to the shared lock, whereas buffered I/O
162 * operations need to wait to grab an exclusive lock in order to clear
163 * NETFS_ICTX_ODIRECT.
164 * Note that buffered writes and truncates both take a write lock on
165 * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT.
166 */
167int netfs_start_io_direct(struct inode *inode)
168 __acquires(inode->i_rwsem)
169{
170 struct netfs_inode *ictx = netfs_inode(inode);
171 int ret;
172
173 /* Be an optimist! */
174 if (down_read_interruptible(sem: &inode->i_rwsem) < 0)
175 return -ERESTARTSYS;
176 if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0)
177 return 0;
178 up_read(sem: &inode->i_rwsem);
179
180 /* Slow path.... */
181 if (down_write_killable(sem: &inode->i_rwsem) < 0)
182 return -ERESTARTSYS;
183 ret = netfs_block_buffered(inode);
184 if (ret < 0) {
185 up_write(sem: &inode->i_rwsem);
186 return ret;
187 }
188 downgrade_write(sem: &inode->i_rwsem);
189 return 0;
190}
191EXPORT_SYMBOL(netfs_start_io_direct);
192
193/**
194 * netfs_end_io_direct - declare that the direct i/o operation is done
195 * @inode: file inode
196 *
197 * Declare that a direct I/O operation is done, and release the shared
198 * lock on inode->i_rwsem.
199 */
200void netfs_end_io_direct(struct inode *inode)
201 __releases(inode->i_rwsem)
202{
203 up_read(sem: &inode->i_rwsem);
204}
205EXPORT_SYMBOL(netfs_end_io_direct);
206