1// SPDX-License-Identifier: GPL-2.0
2#ifndef IOU_ZC_RX_H
3#define IOU_ZC_RX_H
4
5#include <linux/io_uring_types.h>
6#include <linux/dma-buf.h>
7#include <linux/socket.h>
8#include <net/page_pool/types.h>
9#include <net/net_trackers.h>
10
11struct io_zcrx_mem {
12 unsigned long size;
13 bool is_dmabuf;
14
15 struct page **pages;
16 unsigned long nr_folios;
17 struct sg_table page_sg_table;
18 unsigned long account_pages;
19 struct sg_table *sgt;
20
21 struct dma_buf_attachment *attach;
22 struct dma_buf *dmabuf;
23};
24
25struct io_zcrx_area {
26 struct net_iov_area nia;
27 struct io_zcrx_ifq *ifq;
28 atomic_t *user_refs;
29
30 bool is_mapped;
31 u16 area_id;
32
33 /* freelist */
34 spinlock_t freelist_lock ____cacheline_aligned_in_smp;
35 u32 free_count;
36 u32 *freelist;
37
38 struct io_zcrx_mem mem;
39};
40
41struct io_zcrx_ifq {
42 struct io_ring_ctx *ctx;
43 struct io_zcrx_area *area;
44 unsigned niov_shift;
45
46 spinlock_t rq_lock ____cacheline_aligned_in_smp;
47 struct io_uring *rq_ring;
48 struct io_uring_zcrx_rqe *rqes;
49 u32 cached_rq_head;
50 u32 rq_entries;
51
52 u32 if_rxq;
53 struct device *dev;
54 struct net_device *netdev;
55 netdevice_tracker netdev_tracker;
56
57 /*
58 * Page pool and net configuration lock, can be taken deeper in the
59 * net stack.
60 */
61 struct mutex pp_lock;
62 struct io_mapped_region region;
63};
64
65#if defined(CONFIG_IO_URING_ZCRX)
66int io_zcrx_return_bufs(struct io_ring_ctx *ctx,
67 void __user *arg, unsigned nr_arg);
68int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
69 struct io_uring_zcrx_ifq_reg __user *arg);
70void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
71void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);
72int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
73 struct socket *sock, unsigned int flags,
74 unsigned issue_flags, unsigned int *len);
75struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
76 unsigned int id);
77#else
78static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
79 struct io_uring_zcrx_ifq_reg __user *arg)
80{
81 return -EOPNOTSUPP;
82}
83static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
84{
85}
86static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)
87{
88}
89static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
90 struct socket *sock, unsigned int flags,
91 unsigned issue_flags, unsigned int *len)
92{
93 return -EOPNOTSUPP;
94}
95static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
96 unsigned int id)
97{
98 return NULL;
99}
100static inline int io_zcrx_return_bufs(struct io_ring_ctx *ctx,
101 void __user *arg, unsigned nr_arg)
102{
103 return -EOPNOTSUPP;
104}
105#endif
106
107int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
108int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
109
110#endif
111