1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/file.h>
5#include <linux/io_uring/cmd.h>
6#include <linux/security.h>
7#include <linux/nospec.h>
8
9#include <uapi/linux/io_uring.h>
10
11#include "io_uring.h"
12#include "alloc_cache.h"
13#include "rsrc.h"
14#include "kbuf.h"
15#include "uring_cmd.h"
16#include "poll.h"
17
18void io_cmd_cache_free(const void *entry)
19{
20 struct io_async_cmd *ac = (struct io_async_cmd *)entry;
21
22 io_vec_free(iv: &ac->vec);
23 kfree(objp: ac);
24}
25
26static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
27{
28 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
29 struct io_async_cmd *ac = req->async_data;
30
31 if (issue_flags & IO_URING_F_UNLOCKED)
32 return;
33
34 io_alloc_cache_vec_kasan(iv: &ac->vec);
35 if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
36 io_vec_free(iv: &ac->vec);
37
38 if (io_alloc_cache_put(cache: &req->ctx->cmd_cache, entry: ac)) {
39 ioucmd->sqe = NULL;
40 io_req_async_data_clear(req, extra_flags: REQ_F_NEED_CLEANUP);
41 }
42}
43
44void io_uring_cmd_cleanup(struct io_kiocb *req)
45{
46 io_req_uring_cleanup(req, issue_flags: 0);
47}
48
49bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
50 struct io_uring_task *tctx, bool cancel_all)
51{
52 struct hlist_node *tmp;
53 struct io_kiocb *req;
54 bool ret = false;
55
56 lockdep_assert_held(&ctx->uring_lock);
57
58 hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
59 hash_node) {
60 struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
61 struct io_uring_cmd);
62 struct file *file = req->file;
63
64 if (!cancel_all && req->tctx != tctx)
65 continue;
66
67 if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
68 file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL |
69 IO_URING_F_COMPLETE_DEFER);
70 ret = true;
71 }
72 }
73 io_submit_flush_completions(ctx);
74 return ret;
75}
76
77static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
78 unsigned int issue_flags)
79{
80 struct io_kiocb *req = cmd_to_io_kiocb(ptr: cmd);
81 struct io_ring_ctx *ctx = req->ctx;
82
83 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE))
84 return;
85
86 cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
87 io_ring_submit_lock(ctx, issue_flags);
88 hlist_del(n: &req->hash_node);
89 io_ring_submit_unlock(ctx, issue_flags);
90}
91
92/*
93 * Mark this command as concelable, then io_uring_try_cancel_uring_cmd()
94 * will try to cancel this issued command by sending ->uring_cmd() with
95 * issue_flags of IO_URING_F_CANCEL.
96 *
97 * The command is guaranteed to not be done when calling ->uring_cmd()
98 * with IO_URING_F_CANCEL, but it is driver's responsibility to deal
99 * with race between io_uring canceling and normal completion.
100 */
101void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
102 unsigned int issue_flags)
103{
104 struct io_kiocb *req = cmd_to_io_kiocb(ptr: cmd);
105 struct io_ring_ctx *ctx = req->ctx;
106
107 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
108 cmd->flags |= IORING_URING_CMD_CANCELABLE;
109 io_ring_submit_lock(ctx, issue_flags);
110 hlist_add_head(n: &req->hash_node, h: &ctx->cancelable_uring_cmd);
111 io_ring_submit_unlock(ctx, issue_flags);
112 }
113}
114EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
115
116static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw)
117{
118 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
119 unsigned int flags = IO_URING_F_COMPLETE_DEFER;
120
121 if (io_should_terminate_tw(ctx: req->ctx))
122 flags |= IO_URING_F_TASK_DEAD;
123
124 /* task_work executor checks the deffered list completion */
125 ioucmd->task_work_cb(ioucmd, flags);
126}
127
128void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
129 io_uring_cmd_tw_t task_work_cb,
130 unsigned flags)
131{
132 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
133
134 if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
135 return;
136
137 ioucmd->task_work_cb = task_work_cb;
138 req->io_task_work.func = io_uring_cmd_work;
139 __io_req_task_work_add(req, flags);
140}
141EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
142
143static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
144 u64 extra1, u64 extra2)
145{
146 req->big_cqe.extra1 = extra1;
147 req->big_cqe.extra2 = extra2;
148}
149
150/*
151 * Called by consumers of io_uring_cmd, if they originally returned
152 * -EIOCBQUEUED upon receiving the command.
153 */
154void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
155 unsigned issue_flags, bool is_cqe32)
156{
157 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
158
159 if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
160 return;
161
162 io_uring_cmd_del_cancelable(cmd: ioucmd, issue_flags);
163
164 if (ret < 0)
165 req_set_fail(req);
166
167 io_req_set_res(req, res: ret, cflags: 0);
168 if (is_cqe32) {
169 if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
170 req->cqe.flags |= IORING_CQE_F_32;
171 io_req_set_cqe32_extra(req, extra1: res2, extra2: 0);
172 }
173 io_req_uring_cleanup(req, issue_flags);
174 if (req->ctx->flags & IORING_SETUP_IOPOLL) {
175 /* order with io_iopoll_req_issued() checking ->iopoll_complete */
176 smp_store_release(&req->iopoll_completed, 1);
177 } else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
178 if (WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED))
179 return;
180 io_req_complete_defer(req);
181 } else {
182 req->io_task_work.func = io_req_task_complete;
183 io_req_task_work_add(req);
184 }
185}
186EXPORT_SYMBOL_GPL(__io_uring_cmd_done);
187
188int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
189{
190 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
191 struct io_async_cmd *ac;
192
193 if (sqe->__pad1)
194 return -EINVAL;
195
196 ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
197 if (ioucmd->flags & ~IORING_URING_CMD_MASK)
198 return -EINVAL;
199
200 if (ioucmd->flags & IORING_URING_CMD_FIXED) {
201 if (ioucmd->flags & IORING_URING_CMD_MULTISHOT)
202 return -EINVAL;
203 req->buf_index = READ_ONCE(sqe->buf_index);
204 }
205
206 if (!!(ioucmd->flags & IORING_URING_CMD_MULTISHOT) !=
207 !!(req->flags & REQ_F_BUFFER_SELECT))
208 return -EINVAL;
209
210 ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
211
212 ac = io_uring_alloc_async_data(cache: &req->ctx->cmd_cache, req);
213 if (!ac)
214 return -ENOMEM;
215 ioucmd->sqe = sqe;
216 return 0;
217}
218
219void io_uring_cmd_sqe_copy(struct io_kiocb *req)
220{
221 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
222 struct io_async_cmd *ac = req->async_data;
223
224 /* Should not happen, as REQ_F_SQE_COPIED covers this */
225 if (WARN_ON_ONCE(ioucmd->sqe == ac->sqes))
226 return;
227 memcpy(to: ac->sqes, from: ioucmd->sqe, len: uring_sqe_size(ctx: req->ctx));
228 ioucmd->sqe = ac->sqes;
229}
230
231int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
232{
233 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
234 struct io_ring_ctx *ctx = req->ctx;
235 struct file *file = req->file;
236 int ret;
237
238 if (!file->f_op->uring_cmd)
239 return -EOPNOTSUPP;
240
241 ret = security_uring_cmd(ioucmd);
242 if (ret)
243 return ret;
244
245 if (ctx->flags & IORING_SETUP_SQE128)
246 issue_flags |= IO_URING_F_SQE128;
247 if (ctx->flags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED))
248 issue_flags |= IO_URING_F_CQE32;
249 if (io_is_compat(ctx))
250 issue_flags |= IO_URING_F_COMPAT;
251 if (ctx->flags & IORING_SETUP_IOPOLL) {
252 if (!file->f_op->uring_cmd_iopoll)
253 return -EOPNOTSUPP;
254 issue_flags |= IO_URING_F_IOPOLL;
255 req->iopoll_completed = 0;
256 if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) {
257 /* make sure every req only blocks once */
258 req->flags &= ~REQ_F_IOPOLL_STATE;
259 req->iopoll_start = ktime_get_ns();
260 }
261 }
262
263 ret = file->f_op->uring_cmd(ioucmd, issue_flags);
264 if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) {
265 if (ret >= 0)
266 return IOU_ISSUE_SKIP_COMPLETE;
267 }
268 if (ret == -EAGAIN) {
269 ioucmd->flags |= IORING_URING_CMD_REISSUE;
270 return ret;
271 }
272 if (ret == -EIOCBQUEUED)
273 return ret;
274 if (ret < 0)
275 req_set_fail(req);
276 io_req_uring_cleanup(req, issue_flags);
277 io_req_set_res(req, res: ret, cflags: 0);
278 return IOU_COMPLETE;
279}
280
281int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
282 struct iov_iter *iter,
283 struct io_uring_cmd *ioucmd,
284 unsigned int issue_flags)
285{
286 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
287
288 if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
289 return -EINVAL;
290
291 return io_import_reg_buf(req, iter, buf_addr: ubuf, len, ddir: rw, issue_flags);
292}
293EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);
294
295int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
296 const struct iovec __user *uvec,
297 size_t uvec_segs,
298 int ddir, struct iov_iter *iter,
299 unsigned issue_flags)
300{
301 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
302 struct io_async_cmd *ac = req->async_data;
303 int ret;
304
305 if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
306 return -EINVAL;
307
308 ret = io_prep_reg_iovec(req, iv: &ac->vec, uvec, uvec_segs);
309 if (ret)
310 return ret;
311
312 return io_import_reg_vec(ddir, iter, req, vec: &ac->vec, nr_iovs: uvec_segs,
313 issue_flags);
314}
315EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);
316
317void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
318{
319 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
320
321 io_req_queue_iowq(req);
322}
323
324int io_cmd_poll_multishot(struct io_uring_cmd *cmd,
325 unsigned int issue_flags, __poll_t mask)
326{
327 struct io_kiocb *req = cmd_to_io_kiocb(ptr: cmd);
328 int ret;
329
330 if (likely(req->flags & REQ_F_APOLL_MULTISHOT))
331 return 0;
332
333 req->flags |= REQ_F_APOLL_MULTISHOT;
334 mask &= ~EPOLLONESHOT;
335
336 ret = io_arm_apoll(req, issue_flags, mask);
337 return ret == IO_APOLL_OK ? -EIOCBQUEUED : -ECANCELED;
338}
339
340bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd,
341 unsigned int issue_flags,
342 struct io_uring_cqe cqe[2])
343{
344 struct io_kiocb *req = cmd_to_io_kiocb(ptr: cmd);
345
346 if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_MULTISHOT)))
347 return false;
348 return io_req_post_cqe32(req, src_cqe: cqe);
349}
350
351/*
352 * Work with io_uring_mshot_cmd_post_cqe() together for committing the
353 * provided buffer upfront
354 */
355struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,
356 unsigned buf_group, size_t *len,
357 unsigned int issue_flags)
358{
359 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
360
361 if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
362 return (struct io_br_sel) { .val = -EINVAL };
363
364 if (WARN_ON_ONCE(!io_do_buffer_select(req)))
365 return (struct io_br_sel) { .val = -EINVAL };
366
367 return io_buffer_select(req, len, buf_group, issue_flags);
368}
369EXPORT_SYMBOL_GPL(io_uring_cmd_buffer_select);
370
371/*
372 * Return true if this multishot uring_cmd needs to be completed, otherwise
373 * the event CQE is posted successfully.
374 *
375 * This function must use `struct io_br_sel` returned from
376 * io_uring_cmd_buffer_select() for committing the buffer in the same
377 * uring_cmd submission context.
378 */
379bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
380 struct io_br_sel *sel, unsigned int issue_flags)
381{
382 struct io_kiocb *req = cmd_to_io_kiocb(ptr: ioucmd);
383 unsigned int cflags = 0;
384
385 if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
386 return true;
387
388 if (sel->val > 0) {
389 cflags = io_put_kbuf(req, len: sel->val, bl: sel->buf_list);
390 if (io_req_post_cqe(req, res: sel->val, cflags: cflags | IORING_CQE_F_MORE))
391 return false;
392 }
393
394 io_kbuf_recycle(req, bl: sel->buf_list, issue_flags);
395 if (sel->val < 0)
396 req_set_fail(req);
397 io_req_set_res(req, res: sel->val, cflags);
398 return true;
399}
400EXPORT_SYMBOL_GPL(io_uring_mshot_cmd_post_cqe);
401