1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * Network memory
4 *
5 * Author: Mina Almasry <almasrymina@google.com>
6 */
7
8#ifndef _NET_NETMEM_H
9#define _NET_NETMEM_H
10
11#include <linux/dma-mapping.h>
12#include <linux/mm.h>
13#include <net/net_debug.h>
14
15/* These fields in struct page are used by the page_pool and net stack:
16 *
17 * struct {
18 * unsigned long pp_magic;
19 * struct page_pool *pp;
20 * unsigned long _pp_mapping_pad;
21 * unsigned long dma_addr;
22 * atomic_long_t pp_ref_count;
23 * };
24 *
25 * We mirror the page_pool fields here so the page_pool can access these
26 * fields without worrying whether the underlying fields belong to a
27 * page or netmem_desc.
28 *
29 * CAUTION: Do not update the fields in netmem_desc without also
30 * updating the anonymous aliasing union in struct net_iov.
31 */
32struct netmem_desc {
33 unsigned long _flags;
34 unsigned long pp_magic;
35 struct page_pool *pp;
36 unsigned long _pp_mapping_pad;
37 unsigned long dma_addr;
38 atomic_long_t pp_ref_count;
39};
40
41#define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \
42 static_assert(offsetof(struct page, pg) == \
43 offsetof(struct netmem_desc, desc))
44NETMEM_DESC_ASSERT_OFFSET(flags, _flags);
45NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic);
46NETMEM_DESC_ASSERT_OFFSET(pp, pp);
47NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
48NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr);
49NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
50#undef NETMEM_DESC_ASSERT_OFFSET
51
52/*
53 * Since struct netmem_desc uses the space in struct page, the size
54 * should be checked, until struct netmem_desc has its own instance from
55 * slab, to avoid conflicting with other members within struct page.
56 */
57static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount));
58
59/* net_iov */
60
61DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers);
62
63/* We overload the LSB of the struct page pointer to indicate whether it's
64 * a page or net_iov.
65 */
66#define NET_IOV 0x01UL
67
68enum net_iov_type {
69 NET_IOV_DMABUF,
70 NET_IOV_IOURING,
71
72 /* Force size to unsigned long to make the NET_IOV_ASSERTS below pass.
73 */
74 NET_IOV_MAX = ULONG_MAX
75};
76
77/* A memory descriptor representing abstract networking I/O vectors,
78 * generally for non-pages memory that doesn't have its corresponding
79 * struct page and needs to be explicitly allocated through slab.
80 *
81 * net_iovs are allocated and used by networking code, and the size of
82 * the chunk is PAGE_SIZE.
83 *
84 * This memory can be any form of non-struct paged memory. Examples
85 * include imported dmabuf memory and imported io_uring memory. See
86 * net_iov_type for all the supported types.
87 *
88 * @pp_magic: pp field, similar to the one in struct page/struct
89 * netmem_desc.
90 * @pp: the pp this net_iov belongs to, if any.
91 * @dma_addr: the dma addrs of the net_iov. Needed for the network
92 * card to send/receive this net_iov.
93 * @pp_ref_count: the pp ref count of this net_iov, exactly the same
94 * usage as struct page/struct netmem_desc.
95 * @owner: the net_iov_area this net_iov belongs to, if any.
96 * @type: the type of the memory. Different types of net_iovs are
97 * supported.
98 */
99struct net_iov {
100 union {
101 struct netmem_desc desc;
102
103 /* XXX: The following part should be removed once all
104 * the references to them are converted so as to be
105 * accessed via netmem_desc e.g. niov->desc.pp instead
106 * of niov->pp.
107 */
108 struct {
109 unsigned long _flags;
110 unsigned long pp_magic;
111 struct page_pool *pp;
112 unsigned long _pp_mapping_pad;
113 unsigned long dma_addr;
114 atomic_long_t pp_ref_count;
115 };
116 };
117 struct net_iov_area *owner;
118 enum net_iov_type type;
119};
120
121struct net_iov_area {
122 /* Array of net_iovs for this area. */
123 struct net_iov *niovs;
124 size_t num_niovs;
125
126 /* Offset into the dma-buf where this chunk starts. */
127 unsigned long base_virtual;
128};
129
130/* net_iov is union'ed with struct netmem_desc mirroring struct page, so
131 * the page_pool can access these fields without worrying whether the
132 * underlying fields are accessed via netmem_desc or directly via
133 * net_iov, until all the references to them are converted so as to be
134 * accessed via netmem_desc e.g. niov->desc.pp instead of niov->pp.
135 *
136 * The non-net stack fields of struct page are private to the mm stack
137 * and must never be mirrored to net_iov.
138 */
139#define NET_IOV_ASSERT_OFFSET(desc, iov) \
140 static_assert(offsetof(struct netmem_desc, desc) == \
141 offsetof(struct net_iov, iov))
142NET_IOV_ASSERT_OFFSET(_flags, _flags);
143NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic);
144NET_IOV_ASSERT_OFFSET(pp, pp);
145NET_IOV_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
146NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr);
147NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
148#undef NET_IOV_ASSERT_OFFSET
149
150static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov)
151{
152 return niov->owner;
153}
154
155static inline unsigned int net_iov_idx(const struct net_iov *niov)
156{
157 return niov - net_iov_owner(niov)->niovs;
158}
159
160/* netmem */
161
162/**
163 * typedef netmem_ref - a nonexistent type marking a reference to generic
164 * network memory.
165 *
166 * A netmem_ref can be a struct page* or a struct net_iov* underneath.
167 *
168 * Use the supplied helpers to obtain the underlying memory pointer and fields.
169 */
170typedef unsigned long __bitwise netmem_ref;
171
172static inline bool netmem_is_net_iov(const netmem_ref netmem)
173{
174 return (__force unsigned long)netmem & NET_IOV;
175}
176
177/**
178 * __netmem_to_page - unsafely get pointer to the &page backing @netmem
179 * @netmem: netmem reference to convert
180 *
181 * Unsafe version of netmem_to_page(). When @netmem is always page-backed,
182 * e.g. when it's a header buffer, performs faster and generates smaller
183 * object code (no check for the LSB, no WARN). When @netmem points to IOV,
184 * provokes undefined behaviour.
185 *
186 * Return: pointer to the &page (garbage if @netmem is not page-backed).
187 */
188static inline struct page *__netmem_to_page(netmem_ref netmem)
189{
190 return (__force struct page *)netmem;
191}
192
193static inline struct page *netmem_to_page(netmem_ref netmem)
194{
195 if (WARN_ON_ONCE(netmem_is_net_iov(netmem)))
196 return NULL;
197
198 return __netmem_to_page(netmem);
199}
200
201static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem)
202{
203 if (netmem_is_net_iov(netmem))
204 return (struct net_iov *)((__force unsigned long)netmem &
205 ~NET_IOV);
206
207 DEBUG_NET_WARN_ON_ONCE(true);
208 return NULL;
209}
210
211static inline netmem_ref net_iov_to_netmem(struct net_iov *niov)
212{
213 return (__force netmem_ref)((unsigned long)niov | NET_IOV);
214}
215
216#define page_to_netmem(p) (_Generic((p), \
217 const struct page * : (__force const netmem_ref)(p), \
218 struct page * : (__force netmem_ref)(p)))
219
220/**
221 * virt_to_netmem - convert virtual memory pointer to a netmem reference
222 * @data: host memory pointer to convert
223 *
224 * Return: netmem reference to the &page backing this virtual address.
225 */
226static inline netmem_ref virt_to_netmem(const void *data)
227{
228 return page_to_netmem(virt_to_page(data));
229}
230
231static inline int netmem_ref_count(netmem_ref netmem)
232{
233 /* The non-pp refcount of net_iov is always 1. On net_iov, we only
234 * support pp refcounting which uses the pp_ref_count field.
235 */
236 if (netmem_is_net_iov(netmem))
237 return 1;
238
239 return page_ref_count(page: netmem_to_page(netmem));
240}
241
242static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
243{
244 if (netmem_is_net_iov(netmem))
245 return 0;
246
247 return page_to_pfn(netmem_to_page(netmem));
248}
249
250/**
251 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing
252 * @netmem
253 * @netmem: netmem reference to convert
254 *
255 * Unsafe version that can be used only when @netmem is always backed by
256 * system memory, performs faster and generates smaller object code (no
257 * check for the LSB, no WARN). When @netmem points to IOV, provokes
258 * undefined behaviour.
259 *
260 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed
261 * by system memory).
262 */
263static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem)
264{
265 return (__force struct netmem_desc *)netmem;
266}
267
268/* __netmem_clear_lsb - convert netmem_ref to struct net_iov * for access to
269 * common fields.
270 * @netmem: netmem reference to extract as net_iov.
271 *
272 * All the sub types of netmem_ref (page, net_iov) have the same pp, pp_magic,
273 * dma_addr, and pp_ref_count fields at the same offsets. Thus, we can access
274 * these fields without a type check to make sure that the underlying mem is
275 * net_iov or page.
276 *
277 * The resulting value of this function can only be used to access the fields
278 * that are NET_IOV_ASSERT_OFFSET'd. Accessing any other fields will result in
279 * undefined behavior.
280 *
281 * Return: the netmem_ref cast to net_iov* regardless of its underlying type.
282 */
283static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem)
284{
285 return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV);
286}
287
288/* XXX: How to extract netmem_desc from page must be changed, once
289 * netmem_desc no longer overlays on page and will be allocated through
290 * slab.
291 */
292#define __pp_page_to_nmdesc(p) (_Generic((p), \
293 const struct page * : (const struct netmem_desc *)(p), \
294 struct page * : (struct netmem_desc *)(p)))
295
296/* CAUTION: Check if the page is a pp page before calling this helper or
297 * know it's a pp page.
298 */
299#define pp_page_to_nmdesc(p) \
300({ \
301 DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p)); \
302 __pp_page_to_nmdesc(p); \
303})
304
305/**
306 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem
307 * @netmem: netmem reference to get the pointer from
308 *
309 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed,
310 * e.g. when it's a header buffer, performs faster and generates smaller
311 * object code (avoids clearing the LSB). When @netmem points to IOV,
312 * provokes invalid memory access.
313 *
314 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed).
315 */
316static inline struct page_pool *__netmem_get_pp(netmem_ref netmem)
317{
318 return __netmem_to_nmdesc(netmem)->pp;
319}
320
321static inline struct page_pool *netmem_get_pp(netmem_ref netmem)
322{
323 return __netmem_clear_lsb(netmem)->pp;
324}
325
326static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem)
327{
328 return &__netmem_clear_lsb(netmem)->pp_ref_count;
329}
330
331static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid)
332{
333 /* NUMA node preference only makes sense if we're allocating
334 * system memory. Memory providers (which give us net_iovs)
335 * choose for us.
336 */
337 if (netmem_is_net_iov(netmem))
338 return true;
339
340 return page_to_nid(page: netmem_to_page(netmem)) == pref_nid;
341}
342
343static inline netmem_ref netmem_compound_head(netmem_ref netmem)
344{
345 /* niov are never compounded */
346 if (netmem_is_net_iov(netmem))
347 return netmem;
348
349 return page_to_netmem(compound_head(netmem_to_page(netmem)));
350}
351
352/**
353 * __netmem_address - unsafely get pointer to the memory backing @netmem
354 * @netmem: netmem reference to get the pointer for
355 *
356 * Unsafe version of netmem_address(). When @netmem is always page-backed,
357 * e.g. when it's a header buffer, performs faster and generates smaller
358 * object code (no check for the LSB). When @netmem points to IOV, provokes
359 * undefined behaviour.
360 *
361 * Return: pointer to the memory (garbage if @netmem is not page-backed).
362 */
363static inline void *__netmem_address(netmem_ref netmem)
364{
365 return page_address(__netmem_to_page(netmem));
366}
367
368static inline void *netmem_address(netmem_ref netmem)
369{
370 if (netmem_is_net_iov(netmem))
371 return NULL;
372
373 return __netmem_address(netmem);
374}
375
376/**
377 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure
378 * @netmem: netmem reference to check
379 *
380 * Return: true if @netmem is page-backed and the page was allocated under
381 * memory pressure, false otherwise.
382 */
383static inline bool netmem_is_pfmemalloc(netmem_ref netmem)
384{
385 if (netmem_is_net_iov(netmem))
386 return false;
387
388 return page_is_pfmemalloc(page: netmem_to_page(netmem));
389}
390
391static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
392{
393 return __netmem_clear_lsb(netmem)->dma_addr;
394}
395
396void get_netmem(netmem_ref netmem);
397void put_netmem(netmem_ref netmem);
398
399#define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \
400 do { \
401 if (!netmem_is_net_iov(NETMEM)) \
402 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \
403 else \
404 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \
405 } while (0)
406
407static inline void netmem_dma_unmap_page_attrs(struct device *dev,
408 dma_addr_t addr, size_t size,
409 enum dma_data_direction dir,
410 unsigned long attrs)
411{
412 if (!addr)
413 return;
414
415 dma_unmap_page_attrs(dev, addr, size, dir, attrs);
416}
417
418#endif /* _NET_NETMEM_H */
419