1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Linux Socket Filter - Kernel level socket filtering
4 *
5 * Based on the design of the Berkeley Packet Filter. The new
6 * internal format has been designed by PLUMgrid:
7 *
8 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
9 *
10 * Authors:
11 *
12 * Jay Schulist <jschlst@samba.org>
13 * Alexei Starovoitov <ast@plumgrid.com>
14 * Daniel Borkmann <dborkman@redhat.com>
15 *
16 * Andi Kleen - Fix a few bad bugs and races.
17 * Kris Katterjohn - Added many additional checks in bpf_check_classic()
18 */
19
20#include <linux/atomic.h>
21#include <linux/bpf_verifier.h>
22#include <linux/module.h>
23#include <linux/types.h>
24#include <linux/mm.h>
25#include <linux/fcntl.h>
26#include <linux/socket.h>
27#include <linux/sock_diag.h>
28#include <linux/in.h>
29#include <linux/inet.h>
30#include <linux/netdevice.h>
31#include <linux/if_packet.h>
32#include <linux/if_arp.h>
33#include <linux/gfp.h>
34#include <net/inet_common.h>
35#include <net/ip.h>
36#include <net/protocol.h>
37#include <net/netlink.h>
38#include <linux/skbuff.h>
39#include <linux/skmsg.h>
40#include <net/sock.h>
41#include <net/flow_dissector.h>
42#include <linux/errno.h>
43#include <linux/timer.h>
44#include <linux/uaccess.h>
45#include <linux/unaligned.h>
46#include <linux/filter.h>
47#include <linux/ratelimit.h>
48#include <linux/seccomp.h>
49#include <linux/if_vlan.h>
50#include <linux/bpf.h>
51#include <linux/btf.h>
52#include <net/sch_generic.h>
53#include <net/cls_cgroup.h>
54#include <net/dst_metadata.h>
55#include <net/dst.h>
56#include <net/sock_reuseport.h>
57#include <net/busy_poll.h>
58#include <net/tcp.h>
59#include <net/xfrm.h>
60#include <net/udp.h>
61#include <linux/bpf_trace.h>
62#include <net/xdp_sock.h>
63#include <linux/inetdevice.h>
64#include <net/inet_hashtables.h>
65#include <net/inet6_hashtables.h>
66#include <net/ip_fib.h>
67#include <net/nexthop.h>
68#include <net/flow.h>
69#include <net/arp.h>
70#include <net/ipv6.h>
71#include <net/net_namespace.h>
72#include <linux/seg6_local.h>
73#include <net/seg6.h>
74#include <net/seg6_local.h>
75#include <net/lwtunnel.h>
76#include <net/ipv6_stubs.h>
77#include <net/bpf_sk_storage.h>
78#include <net/transp_v6.h>
79#include <linux/btf_ids.h>
80#include <net/tls.h>
81#include <net/xdp.h>
82#include <net/mptcp.h>
83#include <net/netfilter/nf_conntrack_bpf.h>
84#include <net/netkit.h>
85#include <linux/un.h>
86#include <net/xdp_sock_drv.h>
87#include <net/inet_dscp.h>
88
89#include "dev.h"
90
91/* Keep the struct bpf_fib_lookup small so that it fits into a cacheline */
92static_assert(sizeof(struct bpf_fib_lookup) == 64, "struct bpf_fib_lookup size check");
93
94static const struct bpf_func_proto *
95bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
96
97int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len)
98{
99 if (in_compat_syscall()) {
100 struct compat_sock_fprog f32;
101
102 if (len != sizeof(f32))
103 return -EINVAL;
104 if (copy_from_sockptr(dst: &f32, src, size: sizeof(f32)))
105 return -EFAULT;
106 memset(s: dst, c: 0, n: sizeof(*dst));
107 dst->len = f32.len;
108 dst->filter = compat_ptr(uptr: f32.filter);
109 } else {
110 if (len != sizeof(*dst))
111 return -EINVAL;
112 if (copy_from_sockptr(dst, src, size: sizeof(*dst)))
113 return -EFAULT;
114 }
115
116 return 0;
117}
118EXPORT_SYMBOL_GPL(copy_bpf_fprog_from_user);
119
120/**
121 * sk_filter_trim_cap - run a packet through a socket filter
122 * @sk: sock associated with &sk_buff
123 * @skb: buffer to filter
124 * @cap: limit on how short the eBPF program may trim the packet
125 * @reason: record drop reason on errors (negative return value)
126 *
127 * Run the eBPF program and then cut skb->data to correct size returned by
128 * the program. If pkt_len is 0 we toss packet. If skb->len is smaller
129 * than pkt_len we keep whole skb->data. This is the socket level
130 * wrapper to bpf_prog_run. It returns 0 if the packet should
131 * be accepted or -EPERM if the packet should be tossed.
132 *
133 */
134int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb,
135 unsigned int cap, enum skb_drop_reason *reason)
136{
137 int err;
138 struct sk_filter *filter;
139
140 /*
141 * If the skb was allocated from pfmemalloc reserves, only
142 * allow SOCK_MEMALLOC sockets to use it as this socket is
143 * helping free memory
144 */
145 if (skb_pfmemalloc(skb) && !sock_flag(sk, flag: SOCK_MEMALLOC)) {
146 NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
147 *reason = SKB_DROP_REASON_PFMEMALLOC;
148 return -ENOMEM;
149 }
150 err = BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb);
151 if (err) {
152 *reason = SKB_DROP_REASON_SOCKET_FILTER;
153 return err;
154 }
155
156 err = security_sock_rcv_skb(sk, skb);
157 if (err) {
158 *reason = SKB_DROP_REASON_SECURITY_HOOK;
159 return err;
160 }
161
162 rcu_read_lock();
163 filter = rcu_dereference(sk->sk_filter);
164 if (filter) {
165 struct sock *save_sk = skb->sk;
166 unsigned int pkt_len;
167
168 skb->sk = sk;
169 pkt_len = bpf_prog_run_save_cb(prog: filter->prog, skb);
170 skb->sk = save_sk;
171 err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
172 if (err)
173 *reason = SKB_DROP_REASON_SOCKET_FILTER;
174 }
175 rcu_read_unlock();
176
177 return err;
178}
179EXPORT_SYMBOL(sk_filter_trim_cap);
180
181BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb)
182{
183 return skb_get_poff(skb);
184}
185
186BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
187{
188 struct nlattr *nla;
189
190 if (skb_is_nonlinear(skb))
191 return 0;
192
193 if (skb->len < sizeof(struct nlattr))
194 return 0;
195
196 if (a > skb->len - sizeof(struct nlattr))
197 return 0;
198
199 nla = nla_find(head: (struct nlattr *) &skb->data[a], len: skb->len - a, attrtype: x);
200 if (nla)
201 return (void *) nla - (void *) skb->data;
202
203 return 0;
204}
205
206BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
207{
208 struct nlattr *nla;
209
210 if (skb_is_nonlinear(skb))
211 return 0;
212
213 if (skb->len < sizeof(struct nlattr))
214 return 0;
215
216 if (a > skb->len - sizeof(struct nlattr))
217 return 0;
218
219 nla = (struct nlattr *) &skb->data[a];
220 if (!nla_ok(nla, remaining: skb->len - a))
221 return 0;
222
223 nla = nla_find_nested(nla, attrtype: x);
224 if (nla)
225 return (void *) nla - (void *) skb->data;
226
227 return 0;
228}
229
230static int bpf_skb_load_helper_convert_offset(const struct sk_buff *skb, int offset)
231{
232 if (likely(offset >= 0))
233 return offset;
234
235 if (offset >= SKF_NET_OFF)
236 return offset - SKF_NET_OFF + skb_network_offset(skb);
237
238 if (offset >= SKF_LL_OFF && skb_mac_header_was_set(skb))
239 return offset - SKF_LL_OFF + skb_mac_offset(skb);
240
241 return INT_MIN;
242}
243
244BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *,
245 data, int, headlen, int, offset)
246{
247 u8 tmp;
248 const int len = sizeof(tmp);
249
250 offset = bpf_skb_load_helper_convert_offset(skb, offset);
251 if (offset == INT_MIN)
252 return -EFAULT;
253
254 if (headlen - offset >= len)
255 return *(u8 *)(data + offset);
256 if (!skb_copy_bits(skb, offset, to: &tmp, len: sizeof(tmp)))
257 return tmp;
258 else
259 return -EFAULT;
260}
261
262BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb,
263 int, offset)
264{
265 return ____bpf_skb_load_helper_8(skb, data: skb->data, headlen: skb->len - skb->data_len,
266 offset);
267}
268
269BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *,
270 data, int, headlen, int, offset)
271{
272 __be16 tmp;
273 const int len = sizeof(tmp);
274
275 offset = bpf_skb_load_helper_convert_offset(skb, offset);
276 if (offset == INT_MIN)
277 return -EFAULT;
278
279 if (headlen - offset >= len)
280 return get_unaligned_be16(p: data + offset);
281 if (!skb_copy_bits(skb, offset, to: &tmp, len: sizeof(tmp)))
282 return be16_to_cpu(tmp);
283 else
284 return -EFAULT;
285}
286
287BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb,
288 int, offset)
289{
290 return ____bpf_skb_load_helper_16(skb, data: skb->data, headlen: skb->len - skb->data_len,
291 offset);
292}
293
294BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *,
295 data, int, headlen, int, offset)
296{
297 __be32 tmp;
298 const int len = sizeof(tmp);
299
300 offset = bpf_skb_load_helper_convert_offset(skb, offset);
301 if (offset == INT_MIN)
302 return -EFAULT;
303
304 if (headlen - offset >= len)
305 return get_unaligned_be32(p: data + offset);
306 if (!skb_copy_bits(skb, offset, to: &tmp, len: sizeof(tmp)))
307 return be32_to_cpu(tmp);
308 else
309 return -EFAULT;
310}
311
312BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb,
313 int, offset)
314{
315 return ____bpf_skb_load_helper_32(skb, data: skb->data, headlen: skb->len - skb->data_len,
316 offset);
317}
318
319static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
320 struct bpf_insn *insn_buf)
321{
322 struct bpf_insn *insn = insn_buf;
323
324 switch (skb_field) {
325 case SKF_AD_MARK:
326 BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);
327
328 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
329 offsetof(struct sk_buff, mark));
330 break;
331
332 case SKF_AD_PKTTYPE:
333 *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET);
334 *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
335#ifdef __BIG_ENDIAN_BITFIELD
336 *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
337#endif
338 break;
339
340 case SKF_AD_QUEUE:
341 BUILD_BUG_ON(sizeof_field(struct sk_buff, queue_mapping) != 2);
342
343 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
344 offsetof(struct sk_buff, queue_mapping));
345 break;
346
347 case SKF_AD_VLAN_TAG:
348 BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);
349
350 /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */
351 *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
352 offsetof(struct sk_buff, vlan_tci));
353 break;
354 case SKF_AD_VLAN_TAG_PRESENT:
355 BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_all) != 4);
356 *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
357 offsetof(struct sk_buff, vlan_all));
358 *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1);
359 *insn++ = BPF_ALU32_IMM(BPF_MOV, dst_reg, 1);
360 break;
361 }
362
363 return insn - insn_buf;
364}
365
366static bool convert_bpf_extensions(struct sock_filter *fp,
367 struct bpf_insn **insnp)
368{
369 struct bpf_insn *insn = *insnp;
370 u32 cnt;
371
372 switch (fp->k) {
373 case SKF_AD_OFF + SKF_AD_PROTOCOL:
374 BUILD_BUG_ON(sizeof_field(struct sk_buff, protocol) != 2);
375
376 /* A = *(u16 *) (CTX + offsetof(protocol)) */
377 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
378 offsetof(struct sk_buff, protocol));
379 /* A = ntohs(A) [emitting a nop or swap16] */
380 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
381 break;
382
383 case SKF_AD_OFF + SKF_AD_PKTTYPE:
384 cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn_buf: insn);
385 insn += cnt - 1;
386 break;
387
388 case SKF_AD_OFF + SKF_AD_IFINDEX:
389 case SKF_AD_OFF + SKF_AD_HATYPE:
390 BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4);
391 BUILD_BUG_ON(sizeof_field(struct net_device, type) != 2);
392
393 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
394 BPF_REG_TMP, BPF_REG_CTX,
395 offsetof(struct sk_buff, dev));
396 /* if (tmp != 0) goto pc + 1 */
397 *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1);
398 *insn++ = BPF_EXIT_INSN();
399 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX)
400 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP,
401 offsetof(struct net_device, ifindex));
402 else
403 *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP,
404 offsetof(struct net_device, type));
405 break;
406
407 case SKF_AD_OFF + SKF_AD_MARK:
408 cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn_buf: insn);
409 insn += cnt - 1;
410 break;
411
412 case SKF_AD_OFF + SKF_AD_RXHASH:
413 BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);
414
415 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
416 offsetof(struct sk_buff, hash));
417 break;
418
419 case SKF_AD_OFF + SKF_AD_QUEUE:
420 cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn_buf: insn);
421 insn += cnt - 1;
422 break;
423
424 case SKF_AD_OFF + SKF_AD_VLAN_TAG:
425 cnt = convert_skb_access(SKF_AD_VLAN_TAG,
426 BPF_REG_A, BPF_REG_CTX, insn_buf: insn);
427 insn += cnt - 1;
428 break;
429
430 case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
431 cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT,
432 BPF_REG_A, BPF_REG_CTX, insn_buf: insn);
433 insn += cnt - 1;
434 break;
435
436 case SKF_AD_OFF + SKF_AD_VLAN_TPID:
437 BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_proto) != 2);
438
439 /* A = *(u16 *) (CTX + offsetof(vlan_proto)) */
440 *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
441 offsetof(struct sk_buff, vlan_proto));
442 /* A = ntohs(A) [emitting a nop or swap16] */
443 *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
444 break;
445
446 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
447 case SKF_AD_OFF + SKF_AD_NLATTR:
448 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
449 case SKF_AD_OFF + SKF_AD_CPU:
450 case SKF_AD_OFF + SKF_AD_RANDOM:
451 /* arg1 = CTX */
452 *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
453 /* arg2 = A */
454 *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A);
455 /* arg3 = X */
456 *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X);
457 /* Emit call(arg1=CTX, arg2=A, arg3=X) */
458 switch (fp->k) {
459 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
460 *insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset);
461 break;
462 case SKF_AD_OFF + SKF_AD_NLATTR:
463 *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr);
464 break;
465 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
466 *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest);
467 break;
468 case SKF_AD_OFF + SKF_AD_CPU:
469 *insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id);
470 break;
471 case SKF_AD_OFF + SKF_AD_RANDOM:
472 *insn = BPF_EMIT_CALL(bpf_user_rnd_u32);
473 bpf_user_rnd_init_once();
474 break;
475 }
476 break;
477
478 case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
479 /* A ^= X */
480 *insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X);
481 break;
482
483 default:
484 /* This is just a dummy call to avoid letting the compiler
485 * evict __bpf_call_base() as an optimization. Placed here
486 * where no-one bothers.
487 */
488 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
489 return false;
490 }
491
492 *insnp = insn;
493 return true;
494}
495
496static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
497{
498 const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS);
499 int size = bpf_size_to_bytes(BPF_SIZE(fp->code));
500 bool endian = BPF_SIZE(fp->code) == BPF_H ||
501 BPF_SIZE(fp->code) == BPF_W;
502 bool indirect = BPF_MODE(fp->code) == BPF_IND;
503 const int ip_align = NET_IP_ALIGN;
504 struct bpf_insn *insn = *insnp;
505 int offset = fp->k;
506
507 if (!indirect &&
508 ((unaligned_ok && offset >= 0) ||
509 (!unaligned_ok && offset >= 0 &&
510 offset + ip_align >= 0 &&
511 offset + ip_align % size == 0))) {
512 bool ldx_off_ok = offset <= S16_MAX;
513
514 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
515 if (offset)
516 *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
517 *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
518 size, 2 + endian + (!ldx_off_ok * 2));
519 if (ldx_off_ok) {
520 *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
521 BPF_REG_D, offset);
522 } else {
523 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D);
524 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset);
525 *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
526 BPF_REG_TMP, 0);
527 }
528 if (endian)
529 *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8);
530 *insn++ = BPF_JMP_A(8);
531 }
532
533 *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
534 *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D);
535 *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H);
536 if (!indirect) {
537 *insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset);
538 } else {
539 *insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X);
540 if (fp->k)
541 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset);
542 }
543
544 switch (BPF_SIZE(fp->code)) {
545 case BPF_B:
546 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8);
547 break;
548 case BPF_H:
549 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16);
550 break;
551 case BPF_W:
552 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32);
553 break;
554 default:
555 return false;
556 }
557
558 *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2);
559 *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
560 *insn = BPF_EXIT_INSN();
561
562 *insnp = insn;
563 return true;
564}
565
566/**
567 * bpf_convert_filter - convert filter program
568 * @prog: the user passed filter program
569 * @len: the length of the user passed filter program
570 * @new_prog: allocated 'struct bpf_prog' or NULL
571 * @new_len: pointer to store length of converted program
572 * @seen_ld_abs: bool whether we've seen ld_abs/ind
573 *
574 * Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn'
575 * style extended BPF (eBPF).
576 * Conversion workflow:
577 *
578 * 1) First pass for calculating the new program length:
579 * bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs)
580 *
581 * 2) 2nd pass to remap in two passes: 1st pass finds new
582 * jump offsets, 2nd pass remapping:
583 * bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs)
584 */
585static int bpf_convert_filter(struct sock_filter *prog, int len,
586 struct bpf_prog *new_prog, int *new_len,
587 bool *seen_ld_abs)
588{
589 int new_flen = 0, pass = 0, target, i, stack_off;
590 struct bpf_insn *new_insn, *first_insn = NULL;
591 struct sock_filter *fp;
592 int *addrs = NULL;
593 u8 bpf_src;
594
595 BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
596 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
597
598 if (len <= 0 || len > BPF_MAXINSNS)
599 return -EINVAL;
600
601 if (new_prog) {
602 first_insn = new_prog->insnsi;
603 addrs = kcalloc(len, sizeof(*addrs),
604 GFP_KERNEL | __GFP_NOWARN);
605 if (!addrs)
606 return -ENOMEM;
607 }
608
609do_pass:
610 new_insn = first_insn;
611 fp = prog;
612
613 /* Classic BPF related prologue emission. */
614 if (new_prog) {
615 /* Classic BPF expects A and X to be reset first. These need
616 * to be guaranteed to be the first two instructions.
617 */
618 *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
619 *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X);
620
621 /* All programs must keep CTX in callee saved BPF_REG_CTX.
622 * In eBPF case it's done by the compiler, here we need to
623 * do this ourself. Initial CTX is present in BPF_REG_ARG1.
624 */
625 *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1);
626 if (*seen_ld_abs) {
627 /* For packet access in classic BPF, cache skb->data
628 * in callee-saved BPF R8 and skb->len - skb->data_len
629 * (headlen) in BPF R9. Since classic BPF is read-only
630 * on CTX, we only need to cache it once.
631 */
632 *new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
633 BPF_REG_D, BPF_REG_CTX,
634 offsetof(struct sk_buff, data));
635 *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX,
636 offsetof(struct sk_buff, len));
637 *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX,
638 offsetof(struct sk_buff, data_len));
639 *new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP);
640 }
641 } else {
642 new_insn += 3;
643 }
644
645 for (i = 0; i < len; fp++, i++) {
646 struct bpf_insn tmp_insns[32] = { };
647 struct bpf_insn *insn = tmp_insns;
648
649 if (addrs)
650 addrs[i] = new_insn - first_insn;
651
652 switch (fp->code) {
653 /* All arithmetic insns and skb loads map as-is. */
654 case BPF_ALU | BPF_ADD | BPF_X:
655 case BPF_ALU | BPF_ADD | BPF_K:
656 case BPF_ALU | BPF_SUB | BPF_X:
657 case BPF_ALU | BPF_SUB | BPF_K:
658 case BPF_ALU | BPF_AND | BPF_X:
659 case BPF_ALU | BPF_AND | BPF_K:
660 case BPF_ALU | BPF_OR | BPF_X:
661 case BPF_ALU | BPF_OR | BPF_K:
662 case BPF_ALU | BPF_LSH | BPF_X:
663 case BPF_ALU | BPF_LSH | BPF_K:
664 case BPF_ALU | BPF_RSH | BPF_X:
665 case BPF_ALU | BPF_RSH | BPF_K:
666 case BPF_ALU | BPF_XOR | BPF_X:
667 case BPF_ALU | BPF_XOR | BPF_K:
668 case BPF_ALU | BPF_MUL | BPF_X:
669 case BPF_ALU | BPF_MUL | BPF_K:
670 case BPF_ALU | BPF_DIV | BPF_X:
671 case BPF_ALU | BPF_DIV | BPF_K:
672 case BPF_ALU | BPF_MOD | BPF_X:
673 case BPF_ALU | BPF_MOD | BPF_K:
674 case BPF_ALU | BPF_NEG:
675 case BPF_LD | BPF_ABS | BPF_W:
676 case BPF_LD | BPF_ABS | BPF_H:
677 case BPF_LD | BPF_ABS | BPF_B:
678 case BPF_LD | BPF_IND | BPF_W:
679 case BPF_LD | BPF_IND | BPF_H:
680 case BPF_LD | BPF_IND | BPF_B:
681 /* Check for overloaded BPF extension and
682 * directly convert it if found, otherwise
683 * just move on with mapping.
684 */
685 if (BPF_CLASS(fp->code) == BPF_LD &&
686 BPF_MODE(fp->code) == BPF_ABS &&
687 convert_bpf_extensions(fp, insnp: &insn))
688 break;
689 if (BPF_CLASS(fp->code) == BPF_LD &&
690 convert_bpf_ld_abs(fp, insnp: &insn)) {
691 *seen_ld_abs = true;
692 break;
693 }
694
695 if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) ||
696 fp->code == (BPF_ALU | BPF_MOD | BPF_X)) {
697 *insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X);
698 /* Error with exception code on div/mod by 0.
699 * For cBPF programs, this was always return 0.
700 */
701 *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2);
702 *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
703 *insn++ = BPF_EXIT_INSN();
704 }
705
706 *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k);
707 break;
708
709 /* Jump transformation cannot use BPF block macros
710 * everywhere as offset calculation and target updates
711 * require a bit more work than the rest, i.e. jump
712 * opcodes map as-is, but offsets need adjustment.
713 */
714
715#define BPF_EMIT_JMP \
716 do { \
717 const s32 off_min = S16_MIN, off_max = S16_MAX; \
718 s32 off; \
719 \
720 if (target >= len || target < 0) \
721 goto err; \
722 off = addrs ? addrs[target] - addrs[i] - 1 : 0; \
723 /* Adjust pc relative offset for 2nd or 3rd insn. */ \
724 off -= insn - tmp_insns; \
725 /* Reject anything not fitting into insn->off. */ \
726 if (off < off_min || off > off_max) \
727 goto err; \
728 insn->off = off; \
729 } while (0)
730
731 case BPF_JMP | BPF_JA:
732 target = i + fp->k + 1;
733 insn->code = fp->code;
734 BPF_EMIT_JMP;
735 break;
736
737 case BPF_JMP | BPF_JEQ | BPF_K:
738 case BPF_JMP | BPF_JEQ | BPF_X:
739 case BPF_JMP | BPF_JSET | BPF_K:
740 case BPF_JMP | BPF_JSET | BPF_X:
741 case BPF_JMP | BPF_JGT | BPF_K:
742 case BPF_JMP | BPF_JGT | BPF_X:
743 case BPF_JMP | BPF_JGE | BPF_K:
744 case BPF_JMP | BPF_JGE | BPF_X:
745 if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
746 /* BPF immediates are signed, zero extend
747 * immediate into tmp register and use it
748 * in compare insn.
749 */
750 *insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k);
751
752 insn->dst_reg = BPF_REG_A;
753 insn->src_reg = BPF_REG_TMP;
754 bpf_src = BPF_X;
755 } else {
756 insn->dst_reg = BPF_REG_A;
757 insn->imm = fp->k;
758 bpf_src = BPF_SRC(fp->code);
759 insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0;
760 }
761
762 /* Common case where 'jump_false' is next insn. */
763 if (fp->jf == 0) {
764 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
765 target = i + fp->jt + 1;
766 BPF_EMIT_JMP;
767 break;
768 }
769
770 /* Convert some jumps when 'jump_true' is next insn. */
771 if (fp->jt == 0) {
772 switch (BPF_OP(fp->code)) {
773 case BPF_JEQ:
774 insn->code = BPF_JMP | BPF_JNE | bpf_src;
775 break;
776 case BPF_JGT:
777 insn->code = BPF_JMP | BPF_JLE | bpf_src;
778 break;
779 case BPF_JGE:
780 insn->code = BPF_JMP | BPF_JLT | bpf_src;
781 break;
782 default:
783 goto jmp_rest;
784 }
785
786 target = i + fp->jf + 1;
787 BPF_EMIT_JMP;
788 break;
789 }
790jmp_rest:
791 /* Other jumps are mapped into two insns: Jxx and JA. */
792 target = i + fp->jt + 1;
793 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
794 BPF_EMIT_JMP;
795 insn++;
796
797 insn->code = BPF_JMP | BPF_JA;
798 target = i + fp->jf + 1;
799 BPF_EMIT_JMP;
800 break;
801
802 /* ldxb 4 * ([14] & 0xf) is remapped into 6 insns. */
803 case BPF_LDX | BPF_MSH | BPF_B: {
804 struct sock_filter tmp = {
805 .code = BPF_LD | BPF_ABS | BPF_B,
806 .k = fp->k,
807 };
808
809 *seen_ld_abs = true;
810
811 /* X = A */
812 *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
813 /* A = BPF_R0 = *(u8 *) (skb->data + K) */
814 convert_bpf_ld_abs(fp: &tmp, insnp: &insn);
815 insn++;
816 /* A &= 0xf */
817 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf);
818 /* A <<= 2 */
819 *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2);
820 /* tmp = X */
821 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X);
822 /* X = A */
823 *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
824 /* A = tmp */
825 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP);
826 break;
827 }
828 /* RET_K is remapped into 2 insns. RET_A case doesn't need an
829 * extra mov as BPF_REG_0 is already mapped into BPF_REG_A.
830 */
831 case BPF_RET | BPF_A:
832 case BPF_RET | BPF_K:
833 if (BPF_RVAL(fp->code) == BPF_K)
834 *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0,
835 0, fp->k);
836 *insn = BPF_EXIT_INSN();
837 break;
838
839 /* Store to stack. */
840 case BPF_ST:
841 case BPF_STX:
842 stack_off = fp->k * 4 + 4;
843 *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) ==
844 BPF_ST ? BPF_REG_A : BPF_REG_X,
845 -stack_off);
846 /* check_load_and_stores() verifies that classic BPF can
847 * load from stack only after write, so tracking
848 * stack_depth for ST|STX insns is enough
849 */
850 if (new_prog && new_prog->aux->stack_depth < stack_off)
851 new_prog->aux->stack_depth = stack_off;
852 break;
853
854 /* Load from stack. */
855 case BPF_LD | BPF_MEM:
856 case BPF_LDX | BPF_MEM:
857 stack_off = fp->k * 4 + 4;
858 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
859 BPF_REG_A : BPF_REG_X, BPF_REG_FP,
860 -stack_off);
861 break;
862
863 /* A = K or X = K */
864 case BPF_LD | BPF_IMM:
865 case BPF_LDX | BPF_IMM:
866 *insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ?
867 BPF_REG_A : BPF_REG_X, fp->k);
868 break;
869
870 /* X = A */
871 case BPF_MISC | BPF_TAX:
872 *insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
873 break;
874
875 /* A = X */
876 case BPF_MISC | BPF_TXA:
877 *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X);
878 break;
879
880 /* A = skb->len or X = skb->len */
881 case BPF_LD | BPF_W | BPF_LEN:
882 case BPF_LDX | BPF_W | BPF_LEN:
883 *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
884 BPF_REG_A : BPF_REG_X, BPF_REG_CTX,
885 offsetof(struct sk_buff, len));
886 break;
887
888 /* Access seccomp_data fields. */
889 case BPF_LDX | BPF_ABS | BPF_W:
890 /* A = *(u32 *) (ctx + K) */
891 *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k);
892 break;
893
894 /* Unknown instruction. */
895 default:
896 goto err;
897 }
898
899 insn++;
900 if (new_prog)
901 memcpy(to: new_insn, from: tmp_insns,
902 len: sizeof(*insn) * (insn - tmp_insns));
903 new_insn += insn - tmp_insns;
904 }
905
906 if (!new_prog) {
907 /* Only calculating new length. */
908 *new_len = new_insn - first_insn;
909 if (*seen_ld_abs)
910 *new_len += 4; /* Prologue bits. */
911 return 0;
912 }
913
914 pass++;
915 if (new_flen != new_insn - first_insn) {
916 new_flen = new_insn - first_insn;
917 if (pass > 2)
918 goto err;
919 goto do_pass;
920 }
921
922 kfree(objp: addrs);
923 BUG_ON(*new_len != new_flen);
924 return 0;
925err:
926 kfree(objp: addrs);
927 return -EINVAL;
928}
929
930/* Security:
931 *
932 * As we dont want to clear mem[] array for each packet going through
933 * __bpf_prog_run(), we check that filter loaded by user never try to read
934 * a cell if not previously written, and we check all branches to be sure
935 * a malicious user doesn't try to abuse us.
936 */
937static int check_load_and_stores(const struct sock_filter *filter, int flen)
938{
939 u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */
940 int pc, ret = 0;
941
942 BUILD_BUG_ON(BPF_MEMWORDS > 16);
943
944 masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL);
945 if (!masks)
946 return -ENOMEM;
947
948 memset(s: masks, c: 0xff, n: flen * sizeof(*masks));
949
950 for (pc = 0; pc < flen; pc++) {
951 memvalid &= masks[pc];
952
953 switch (filter[pc].code) {
954 case BPF_ST:
955 case BPF_STX:
956 memvalid |= (1 << filter[pc].k);
957 break;
958 case BPF_LD | BPF_MEM:
959 case BPF_LDX | BPF_MEM:
960 if (!(memvalid & (1 << filter[pc].k))) {
961 ret = -EINVAL;
962 goto error;
963 }
964 break;
965 case BPF_JMP | BPF_JA:
966 /* A jump must set masks on target */
967 masks[pc + 1 + filter[pc].k] &= memvalid;
968 memvalid = ~0;
969 break;
970 case BPF_JMP | BPF_JEQ | BPF_K:
971 case BPF_JMP | BPF_JEQ | BPF_X:
972 case BPF_JMP | BPF_JGE | BPF_K:
973 case BPF_JMP | BPF_JGE | BPF_X:
974 case BPF_JMP | BPF_JGT | BPF_K:
975 case BPF_JMP | BPF_JGT | BPF_X:
976 case BPF_JMP | BPF_JSET | BPF_K:
977 case BPF_JMP | BPF_JSET | BPF_X:
978 /* A jump must set masks on targets */
979 masks[pc + 1 + filter[pc].jt] &= memvalid;
980 masks[pc + 1 + filter[pc].jf] &= memvalid;
981 memvalid = ~0;
982 break;
983 }
984 }
985error:
986 kfree(objp: masks);
987 return ret;
988}
989
990static bool chk_code_allowed(u16 code_to_probe)
991{
992 static const bool codes[] = {
993 /* 32 bit ALU operations */
994 [BPF_ALU | BPF_ADD | BPF_K] = true,
995 [BPF_ALU | BPF_ADD | BPF_X] = true,
996 [BPF_ALU | BPF_SUB | BPF_K] = true,
997 [BPF_ALU | BPF_SUB | BPF_X] = true,
998 [BPF_ALU | BPF_MUL | BPF_K] = true,
999 [BPF_ALU | BPF_MUL | BPF_X] = true,
1000 [BPF_ALU | BPF_DIV | BPF_K] = true,
1001 [BPF_ALU | BPF_DIV | BPF_X] = true,
1002 [BPF_ALU | BPF_MOD | BPF_K] = true,
1003 [BPF_ALU | BPF_MOD | BPF_X] = true,
1004 [BPF_ALU | BPF_AND | BPF_K] = true,
1005 [BPF_ALU | BPF_AND | BPF_X] = true,
1006 [BPF_ALU | BPF_OR | BPF_K] = true,
1007 [BPF_ALU | BPF_OR | BPF_X] = true,
1008 [BPF_ALU | BPF_XOR | BPF_K] = true,
1009 [BPF_ALU | BPF_XOR | BPF_X] = true,
1010 [BPF_ALU | BPF_LSH | BPF_K] = true,
1011 [BPF_ALU | BPF_LSH | BPF_X] = true,
1012 [BPF_ALU | BPF_RSH | BPF_K] = true,
1013 [BPF_ALU | BPF_RSH | BPF_X] = true,
1014 [BPF_ALU | BPF_NEG] = true,
1015 /* Load instructions */
1016 [BPF_LD | BPF_W | BPF_ABS] = true,
1017 [BPF_LD | BPF_H | BPF_ABS] = true,
1018 [BPF_LD | BPF_B | BPF_ABS] = true,
1019 [BPF_LD | BPF_W | BPF_LEN] = true,
1020 [BPF_LD | BPF_W | BPF_IND] = true,
1021 [BPF_LD | BPF_H | BPF_IND] = true,
1022 [BPF_LD | BPF_B | BPF_IND] = true,
1023 [BPF_LD | BPF_IMM] = true,
1024 [BPF_LD | BPF_MEM] = true,
1025 [BPF_LDX | BPF_W | BPF_LEN] = true,
1026 [BPF_LDX | BPF_B | BPF_MSH] = true,
1027 [BPF_LDX | BPF_IMM] = true,
1028 [BPF_LDX | BPF_MEM] = true,
1029 /* Store instructions */
1030 [BPF_ST] = true,
1031 [BPF_STX] = true,
1032 /* Misc instructions */
1033 [BPF_MISC | BPF_TAX] = true,
1034 [BPF_MISC | BPF_TXA] = true,
1035 /* Return instructions */
1036 [BPF_RET | BPF_K] = true,
1037 [BPF_RET | BPF_A] = true,
1038 /* Jump instructions */
1039 [BPF_JMP | BPF_JA] = true,
1040 [BPF_JMP | BPF_JEQ | BPF_K] = true,
1041 [BPF_JMP | BPF_JEQ | BPF_X] = true,
1042 [BPF_JMP | BPF_JGE | BPF_K] = true,
1043 [BPF_JMP | BPF_JGE | BPF_X] = true,
1044 [BPF_JMP | BPF_JGT | BPF_K] = true,
1045 [BPF_JMP | BPF_JGT | BPF_X] = true,
1046 [BPF_JMP | BPF_JSET | BPF_K] = true,
1047 [BPF_JMP | BPF_JSET | BPF_X] = true,
1048 };
1049
1050 if (code_to_probe >= ARRAY_SIZE(codes))
1051 return false;
1052
1053 return codes[code_to_probe];
1054}
1055
1056static bool bpf_check_basics_ok(const struct sock_filter *filter,
1057 unsigned int flen)
1058{
1059 if (filter == NULL)
1060 return false;
1061 if (flen == 0 || flen > BPF_MAXINSNS)
1062 return false;
1063
1064 return true;
1065}
1066
1067/**
1068 * bpf_check_classic - verify socket filter code
1069 * @filter: filter to verify
1070 * @flen: length of filter
1071 *
1072 * Check the user's filter code. If we let some ugly
1073 * filter code slip through kaboom! The filter must contain
1074 * no references or jumps that are out of range, no illegal
1075 * instructions, and must end with a RET instruction.
1076 *
1077 * All jumps are forward as they are not signed.
1078 *
1079 * Returns 0 if the rule set is legal or -EINVAL if not.
1080 */
1081static int bpf_check_classic(const struct sock_filter *filter,
1082 unsigned int flen)
1083{
1084 bool anc_found;
1085 int pc;
1086
1087 /* Check the filter code now */
1088 for (pc = 0; pc < flen; pc++) {
1089 const struct sock_filter *ftest = &filter[pc];
1090
1091 /* May we actually operate on this code? */
1092 if (!chk_code_allowed(code_to_probe: ftest->code))
1093 return -EINVAL;
1094
1095 /* Some instructions need special checks */
1096 switch (ftest->code) {
1097 case BPF_ALU | BPF_DIV | BPF_K:
1098 case BPF_ALU | BPF_MOD | BPF_K:
1099 /* Check for division by zero */
1100 if (ftest->k == 0)
1101 return -EINVAL;
1102 break;
1103 case BPF_ALU | BPF_LSH | BPF_K:
1104 case BPF_ALU | BPF_RSH | BPF_K:
1105 if (ftest->k >= 32)
1106 return -EINVAL;
1107 break;
1108 case BPF_LD | BPF_MEM:
1109 case BPF_LDX | BPF_MEM:
1110 case BPF_ST:
1111 case BPF_STX:
1112 /* Check for invalid memory addresses */
1113 if (ftest->k >= BPF_MEMWORDS)
1114 return -EINVAL;
1115 break;
1116 case BPF_JMP | BPF_JA:
1117 /* Note, the large ftest->k might cause loops.
1118 * Compare this with conditional jumps below,
1119 * where offsets are limited. --ANK (981016)
1120 */
1121 if (ftest->k >= (unsigned int)(flen - pc - 1))
1122 return -EINVAL;
1123 break;
1124 case BPF_JMP | BPF_JEQ | BPF_K:
1125 case BPF_JMP | BPF_JEQ | BPF_X:
1126 case BPF_JMP | BPF_JGE | BPF_K:
1127 case BPF_JMP | BPF_JGE | BPF_X:
1128 case BPF_JMP | BPF_JGT | BPF_K:
1129 case BPF_JMP | BPF_JGT | BPF_X:
1130 case BPF_JMP | BPF_JSET | BPF_K:
1131 case BPF_JMP | BPF_JSET | BPF_X:
1132 /* Both conditionals must be safe */
1133 if (pc + ftest->jt + 1 >= flen ||
1134 pc + ftest->jf + 1 >= flen)
1135 return -EINVAL;
1136 break;
1137 case BPF_LD | BPF_W | BPF_ABS:
1138 case BPF_LD | BPF_H | BPF_ABS:
1139 case BPF_LD | BPF_B | BPF_ABS:
1140 anc_found = false;
1141 if (bpf_anc_helper(ftest) & BPF_ANC)
1142 anc_found = true;
1143 /* Ancillary operation unknown or unsupported */
1144 if (anc_found == false && ftest->k >= SKF_AD_OFF)
1145 return -EINVAL;
1146 }
1147 }
1148
1149 /* Last instruction must be a RET code */
1150 switch (filter[flen - 1].code) {
1151 case BPF_RET | BPF_K:
1152 case BPF_RET | BPF_A:
1153 return check_load_and_stores(filter, flen);
1154 }
1155
1156 return -EINVAL;
1157}
1158
1159static int bpf_prog_store_orig_filter(struct bpf_prog *fp,
1160 const struct sock_fprog *fprog)
1161{
1162 unsigned int fsize = bpf_classic_proglen(fprog);
1163 struct sock_fprog_kern *fkprog;
1164
1165 fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
1166 if (!fp->orig_prog)
1167 return -ENOMEM;
1168
1169 fkprog = fp->orig_prog;
1170 fkprog->len = fprog->len;
1171
1172 fkprog->filter = kmemdup(fp->insns, fsize,
1173 GFP_KERNEL | __GFP_NOWARN);
1174 if (!fkprog->filter) {
1175 kfree(objp: fp->orig_prog);
1176 return -ENOMEM;
1177 }
1178
1179 return 0;
1180}
1181
1182static void bpf_release_orig_filter(struct bpf_prog *fp)
1183{
1184 struct sock_fprog_kern *fprog = fp->orig_prog;
1185
1186 if (fprog) {
1187 kfree(objp: fprog->filter);
1188 kfree(objp: fprog);
1189 }
1190}
1191
1192static void __bpf_prog_release(struct bpf_prog *prog)
1193{
1194 if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) {
1195 bpf_prog_put(prog);
1196 } else {
1197 bpf_release_orig_filter(fp: prog);
1198 bpf_prog_free(fp: prog);
1199 }
1200}
1201
1202static void __sk_filter_release(struct sk_filter *fp)
1203{
1204 __bpf_prog_release(prog: fp->prog);
1205 kfree(objp: fp);
1206}
1207
1208/**
1209 * sk_filter_release_rcu - Release a socket filter by rcu_head
1210 * @rcu: rcu_head that contains the sk_filter to free
1211 */
1212static void sk_filter_release_rcu(struct rcu_head *rcu)
1213{
1214 struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
1215
1216 __sk_filter_release(fp);
1217}
1218
1219/**
1220 * sk_filter_release - release a socket filter
1221 * @fp: filter to remove
1222 *
1223 * Remove a filter from a socket and release its resources.
1224 */
1225static void sk_filter_release(struct sk_filter *fp)
1226{
1227 if (refcount_dec_and_test(r: &fp->refcnt))
1228 call_rcu(head: &fp->rcu, func: sk_filter_release_rcu);
1229}
1230
1231void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1232{
1233 u32 filter_size = bpf_prog_size(proglen: fp->prog->len);
1234
1235 atomic_sub(i: filter_size, v: &sk->sk_omem_alloc);
1236 sk_filter_release(fp);
1237}
1238
1239/* try to charge the socket memory if there is space available
1240 * return true on success
1241 */
1242static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1243{
1244 int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
1245 u32 filter_size = bpf_prog_size(proglen: fp->prog->len);
1246
1247 /* same check as in sock_kmalloc() */
1248 if (filter_size <= optmem_max &&
1249 atomic_read(v: &sk->sk_omem_alloc) + filter_size < optmem_max) {
1250 atomic_add(i: filter_size, v: &sk->sk_omem_alloc);
1251 return true;
1252 }
1253 return false;
1254}
1255
1256bool sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1257{
1258 if (!refcount_inc_not_zero(r: &fp->refcnt))
1259 return false;
1260
1261 if (!__sk_filter_charge(sk, fp)) {
1262 sk_filter_release(fp);
1263 return false;
1264 }
1265 return true;
1266}
1267
1268static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
1269{
1270 struct sock_filter *old_prog;
1271 struct bpf_prog *old_fp;
1272 int err, new_len, old_len = fp->len;
1273 bool seen_ld_abs = false;
1274
1275 /* We are free to overwrite insns et al right here as it won't be used at
1276 * this point in time anymore internally after the migration to the eBPF
1277 * instruction representation.
1278 */
1279 BUILD_BUG_ON(sizeof(struct sock_filter) !=
1280 sizeof(struct bpf_insn));
1281
1282 /* Conversion cannot happen on overlapping memory areas,
1283 * so we need to keep the user BPF around until the 2nd
1284 * pass. At this time, the user BPF is stored in fp->insns.
1285 */
1286 old_prog = kmemdup_array(src: fp->insns, count: old_len, element_size: sizeof(struct sock_filter),
1287 GFP_KERNEL | __GFP_NOWARN);
1288 if (!old_prog) {
1289 err = -ENOMEM;
1290 goto out_err;
1291 }
1292
1293 /* 1st pass: calculate the new program length. */
1294 err = bpf_convert_filter(prog: old_prog, len: old_len, NULL, new_len: &new_len,
1295 seen_ld_abs: &seen_ld_abs);
1296 if (err)
1297 goto out_err_free;
1298
1299 /* Expand fp for appending the new filter representation. */
1300 old_fp = fp;
1301 fp = bpf_prog_realloc(fp_old: old_fp, size: bpf_prog_size(proglen: new_len), gfp_extra_flags: 0);
1302 if (!fp) {
1303 /* The old_fp is still around in case we couldn't
1304 * allocate new memory, so uncharge on that one.
1305 */
1306 fp = old_fp;
1307 err = -ENOMEM;
1308 goto out_err_free;
1309 }
1310
1311 fp->len = new_len;
1312
1313 /* 2nd pass: remap sock_filter insns into bpf_insn insns. */
1314 err = bpf_convert_filter(prog: old_prog, len: old_len, new_prog: fp, new_len: &new_len,
1315 seen_ld_abs: &seen_ld_abs);
1316 if (err)
1317 /* 2nd bpf_convert_filter() can fail only if it fails
1318 * to allocate memory, remapping must succeed. Note,
1319 * that at this time old_fp has already been released
1320 * by krealloc().
1321 */
1322 goto out_err_free;
1323
1324 fp = bpf_prog_select_runtime(fp, err: &err);
1325 if (err)
1326 goto out_err_free;
1327
1328 kfree(objp: old_prog);
1329 return fp;
1330
1331out_err_free:
1332 kfree(objp: old_prog);
1333out_err:
1334 __bpf_prog_release(prog: fp);
1335 return ERR_PTR(error: err);
1336}
1337
1338static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp,
1339 bpf_aux_classic_check_t trans)
1340{
1341 int err;
1342
1343 fp->bpf_func = NULL;
1344 fp->jited = 0;
1345
1346 err = bpf_check_classic(filter: fp->insns, flen: fp->len);
1347 if (err) {
1348 __bpf_prog_release(prog: fp);
1349 return ERR_PTR(error: err);
1350 }
1351
1352 /* There might be additional checks and transformations
1353 * needed on classic filters, f.e. in case of seccomp.
1354 */
1355 if (trans) {
1356 err = trans(fp->insns, fp->len);
1357 if (err) {
1358 __bpf_prog_release(prog: fp);
1359 return ERR_PTR(error: err);
1360 }
1361 }
1362
1363 /* Probe if we can JIT compile the filter and if so, do
1364 * the compilation of the filter.
1365 */
1366 bpf_jit_compile(prog: fp);
1367
1368 /* JIT compiler couldn't process this filter, so do the eBPF translation
1369 * for the optimized interpreter.
1370 */
1371 if (!fp->jited)
1372 fp = bpf_migrate_filter(fp);
1373
1374 return fp;
1375}
1376
1377/**
1378 * bpf_prog_create - create an unattached filter
1379 * @pfp: the unattached filter that is created
1380 * @fprog: the filter program
1381 *
1382 * Create a filter independent of any socket. We first run some
1383 * sanity checks on it to make sure it does not explode on us later.
1384 * If an error occurs or there is insufficient memory for the filter
1385 * a negative errno code is returned. On success the return is zero.
1386 */
1387int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
1388{
1389 unsigned int fsize = bpf_classic_proglen(fprog);
1390 struct bpf_prog *fp;
1391
1392 /* Make sure new filter is there and in the right amounts. */
1393 if (!bpf_check_basics_ok(filter: fprog->filter, flen: fprog->len))
1394 return -EINVAL;
1395
1396 fp = bpf_prog_alloc(size: bpf_prog_size(proglen: fprog->len), gfp_extra_flags: 0);
1397 if (!fp)
1398 return -ENOMEM;
1399
1400 memcpy(to: fp->insns, from: fprog->filter, len: fsize);
1401
1402 fp->len = fprog->len;
1403 /* Since unattached filters are not copied back to user
1404 * space through sk_get_filter(), we do not need to hold
1405 * a copy here, and can spare us the work.
1406 */
1407 fp->orig_prog = NULL;
1408
1409 /* bpf_prepare_filter() already takes care of freeing
1410 * memory in case something goes wrong.
1411 */
1412 fp = bpf_prepare_filter(fp, NULL);
1413 if (IS_ERR(ptr: fp))
1414 return PTR_ERR(ptr: fp);
1415
1416 *pfp = fp;
1417 return 0;
1418}
1419EXPORT_SYMBOL_GPL(bpf_prog_create);
1420
1421/**
1422 * bpf_prog_create_from_user - create an unattached filter from user buffer
1423 * @pfp: the unattached filter that is created
1424 * @fprog: the filter program
1425 * @trans: post-classic verifier transformation handler
1426 * @save_orig: save classic BPF program
1427 *
1428 * This function effectively does the same as bpf_prog_create(), only
1429 * that it builds up its insns buffer from user space provided buffer.
1430 * It also allows for passing a bpf_aux_classic_check_t handler.
1431 */
1432int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
1433 bpf_aux_classic_check_t trans, bool save_orig)
1434{
1435 unsigned int fsize = bpf_classic_proglen(fprog);
1436 struct bpf_prog *fp;
1437 int err;
1438
1439 /* Make sure new filter is there and in the right amounts. */
1440 if (!bpf_check_basics_ok(filter: fprog->filter, flen: fprog->len))
1441 return -EINVAL;
1442
1443 fp = bpf_prog_alloc(size: bpf_prog_size(proglen: fprog->len), gfp_extra_flags: 0);
1444 if (!fp)
1445 return -ENOMEM;
1446
1447 if (copy_from_user(to: fp->insns, from: fprog->filter, n: fsize)) {
1448 __bpf_prog_free(fp);
1449 return -EFAULT;
1450 }
1451
1452 fp->len = fprog->len;
1453 fp->orig_prog = NULL;
1454
1455 if (save_orig) {
1456 err = bpf_prog_store_orig_filter(fp, fprog);
1457 if (err) {
1458 __bpf_prog_free(fp);
1459 return -ENOMEM;
1460 }
1461 }
1462
1463 /* bpf_prepare_filter() already takes care of freeing
1464 * memory in case something goes wrong.
1465 */
1466 fp = bpf_prepare_filter(fp, trans);
1467 if (IS_ERR(ptr: fp))
1468 return PTR_ERR(ptr: fp);
1469
1470 *pfp = fp;
1471 return 0;
1472}
1473EXPORT_SYMBOL_GPL(bpf_prog_create_from_user);
1474
1475void bpf_prog_destroy(struct bpf_prog *fp)
1476{
1477 __bpf_prog_release(prog: fp);
1478}
1479EXPORT_SYMBOL_GPL(bpf_prog_destroy);
1480
1481static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk)
1482{
1483 struct sk_filter *fp, *old_fp;
1484
1485 fp = kmalloc(sizeof(*fp), GFP_KERNEL);
1486 if (!fp)
1487 return -ENOMEM;
1488
1489 fp->prog = prog;
1490
1491 if (!__sk_filter_charge(sk, fp)) {
1492 kfree(objp: fp);
1493 return -ENOMEM;
1494 }
1495 refcount_set(r: &fp->refcnt, n: 1);
1496
1497 old_fp = rcu_dereference_protected(sk->sk_filter,
1498 lockdep_sock_is_held(sk));
1499 rcu_assign_pointer(sk->sk_filter, fp);
1500
1501 if (old_fp)
1502 sk_filter_uncharge(sk, fp: old_fp);
1503
1504 return 0;
1505}
1506
1507static
1508struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
1509{
1510 unsigned int fsize = bpf_classic_proglen(fprog);
1511 struct bpf_prog *prog;
1512 int err;
1513
1514 if (sock_flag(sk, flag: SOCK_FILTER_LOCKED))
1515 return ERR_PTR(error: -EPERM);
1516
1517 /* Make sure new filter is there and in the right amounts. */
1518 if (!bpf_check_basics_ok(filter: fprog->filter, flen: fprog->len))
1519 return ERR_PTR(error: -EINVAL);
1520
1521 prog = bpf_prog_alloc(size: bpf_prog_size(proglen: fprog->len), gfp_extra_flags: 0);
1522 if (!prog)
1523 return ERR_PTR(error: -ENOMEM);
1524
1525 if (copy_from_user(to: prog->insns, from: fprog->filter, n: fsize)) {
1526 __bpf_prog_free(fp: prog);
1527 return ERR_PTR(error: -EFAULT);
1528 }
1529
1530 prog->len = fprog->len;
1531
1532 err = bpf_prog_store_orig_filter(fp: prog, fprog);
1533 if (err) {
1534 __bpf_prog_free(fp: prog);
1535 return ERR_PTR(error: -ENOMEM);
1536 }
1537
1538 /* bpf_prepare_filter() already takes care of freeing
1539 * memory in case something goes wrong.
1540 */
1541 return bpf_prepare_filter(fp: prog, NULL);
1542}
1543
1544/**
1545 * sk_attach_filter - attach a socket filter
1546 * @fprog: the filter program
1547 * @sk: the socket to use
1548 *
1549 * Attach the user's filter code. We first run some sanity checks on
1550 * it to make sure it does not explode on us later. If an error
1551 * occurs or there is insufficient memory for the filter a negative
1552 * errno code is returned. On success the return is zero.
1553 */
1554int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1555{
1556 struct bpf_prog *prog = __get_filter(fprog, sk);
1557 int err;
1558
1559 if (IS_ERR(ptr: prog))
1560 return PTR_ERR(ptr: prog);
1561
1562 err = __sk_attach_prog(prog, sk);
1563 if (err < 0) {
1564 __bpf_prog_release(prog);
1565 return err;
1566 }
1567
1568 return 0;
1569}
1570EXPORT_SYMBOL_GPL(sk_attach_filter);
1571
1572int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1573{
1574 struct bpf_prog *prog = __get_filter(fprog, sk);
1575 int err, optmem_max;
1576
1577 if (IS_ERR(ptr: prog))
1578 return PTR_ERR(ptr: prog);
1579
1580 optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
1581 if (bpf_prog_size(proglen: prog->len) > optmem_max)
1582 err = -ENOMEM;
1583 else
1584 err = reuseport_attach_prog(sk, prog);
1585
1586 if (err)
1587 __bpf_prog_release(prog);
1588
1589 return err;
1590}
1591
1592static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk)
1593{
1594 if (sock_flag(sk, flag: SOCK_FILTER_LOCKED))
1595 return ERR_PTR(error: -EPERM);
1596
1597 return bpf_prog_get_type(ufd, type: BPF_PROG_TYPE_SOCKET_FILTER);
1598}
1599
1600int sk_attach_bpf(u32 ufd, struct sock *sk)
1601{
1602 struct bpf_prog *prog = __get_bpf(ufd, sk);
1603 int err;
1604
1605 if (IS_ERR(ptr: prog))
1606 return PTR_ERR(ptr: prog);
1607
1608 err = __sk_attach_prog(prog, sk);
1609 if (err < 0) {
1610 bpf_prog_put(prog);
1611 return err;
1612 }
1613
1614 return 0;
1615}
1616
1617int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
1618{
1619 struct bpf_prog *prog;
1620 int err, optmem_max;
1621
1622 if (sock_flag(sk, flag: SOCK_FILTER_LOCKED))
1623 return -EPERM;
1624
1625 prog = bpf_prog_get_type(ufd, type: BPF_PROG_TYPE_SOCKET_FILTER);
1626 if (PTR_ERR(ptr: prog) == -EINVAL)
1627 prog = bpf_prog_get_type(ufd, type: BPF_PROG_TYPE_SK_REUSEPORT);
1628 if (IS_ERR(ptr: prog))
1629 return PTR_ERR(ptr: prog);
1630
1631 if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) {
1632 /* Like other non BPF_PROG_TYPE_SOCKET_FILTER
1633 * bpf prog (e.g. sockmap). It depends on the
1634 * limitation imposed by bpf_prog_load().
1635 * Hence, sysctl_optmem_max is not checked.
1636 */
1637 if ((sk->sk_type != SOCK_STREAM &&
1638 sk->sk_type != SOCK_DGRAM) ||
1639 (sk->sk_protocol != IPPROTO_UDP &&
1640 sk->sk_protocol != IPPROTO_TCP) ||
1641 (sk->sk_family != AF_INET &&
1642 sk->sk_family != AF_INET6)) {
1643 err = -ENOTSUPP;
1644 goto err_prog_put;
1645 }
1646 } else {
1647 /* BPF_PROG_TYPE_SOCKET_FILTER */
1648 optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
1649 if (bpf_prog_size(proglen: prog->len) > optmem_max) {
1650 err = -ENOMEM;
1651 goto err_prog_put;
1652 }
1653 }
1654
1655 err = reuseport_attach_prog(sk, prog);
1656err_prog_put:
1657 if (err)
1658 bpf_prog_put(prog);
1659
1660 return err;
1661}
1662
1663void sk_reuseport_prog_free(struct bpf_prog *prog)
1664{
1665 if (!prog)
1666 return;
1667
1668 if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
1669 bpf_prog_put(prog);
1670 else
1671 bpf_prog_destroy(prog);
1672}
1673
1674static inline int __bpf_try_make_writable(struct sk_buff *skb,
1675 unsigned int write_len)
1676{
1677#ifdef CONFIG_DEBUG_NET
1678 /* Avoid a splat in pskb_may_pull_reason() */
1679 if (write_len > INT_MAX)
1680 return -EINVAL;
1681#endif
1682 return skb_ensure_writable(skb, write_len);
1683}
1684
1685static inline int bpf_try_make_writable(struct sk_buff *skb,
1686 unsigned int write_len)
1687{
1688 int err = __bpf_try_make_writable(skb, write_len);
1689
1690 bpf_compute_data_pointers(skb);
1691 return err;
1692}
1693
1694static int bpf_try_make_head_writable(struct sk_buff *skb)
1695{
1696 return bpf_try_make_writable(skb, write_len: skb_headlen(skb));
1697}
1698
1699static inline void bpf_push_mac_rcsum(struct sk_buff *skb)
1700{
1701 if (skb_at_tc_ingress(skb))
1702 skb_postpush_rcsum(skb, start: skb_mac_header(skb), len: skb->mac_len);
1703}
1704
1705static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
1706{
1707 if (skb_at_tc_ingress(skb))
1708 skb_postpull_rcsum(skb, start: skb_mac_header(skb), len: skb->mac_len);
1709}
1710
1711BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
1712 const void *, from, u32, len, u64, flags)
1713{
1714 void *ptr;
1715
1716 if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH)))
1717 return -EINVAL;
1718 if (unlikely(offset > INT_MAX))
1719 return -EFAULT;
1720 if (unlikely(bpf_try_make_writable(skb, offset + len)))
1721 return -EFAULT;
1722
1723 ptr = skb->data + offset;
1724 if (flags & BPF_F_RECOMPUTE_CSUM)
1725 __skb_postpull_rcsum(skb, start: ptr, len, off: offset);
1726
1727 memcpy(to: ptr, from, len);
1728
1729 if (flags & BPF_F_RECOMPUTE_CSUM)
1730 __skb_postpush_rcsum(skb, start: ptr, len, off: offset);
1731 if (flags & BPF_F_INVALIDATE_HASH)
1732 skb_clear_hash(skb);
1733
1734 return 0;
1735}
1736
1737static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
1738 .func = bpf_skb_store_bytes,
1739 .gpl_only = false,
1740 .ret_type = RET_INTEGER,
1741 .arg1_type = ARG_PTR_TO_CTX,
1742 .arg2_type = ARG_ANYTHING,
1743 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
1744 .arg4_type = ARG_CONST_SIZE,
1745 .arg5_type = ARG_ANYTHING,
1746};
1747
1748int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
1749 u32 len, u64 flags)
1750{
1751 return ____bpf_skb_store_bytes(skb, offset, from, len, flags);
1752}
1753
1754BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
1755 void *, to, u32, len)
1756{
1757 void *ptr;
1758
1759 if (unlikely(offset > INT_MAX))
1760 goto err_clear;
1761
1762 ptr = skb_header_pointer(skb, offset, len, buffer: to);
1763 if (unlikely(!ptr))
1764 goto err_clear;
1765 if (ptr != to)
1766 memcpy(to, from: ptr, len);
1767
1768 return 0;
1769err_clear:
1770 memset(s: to, c: 0, n: len);
1771 return -EFAULT;
1772}
1773
1774static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
1775 .func = bpf_skb_load_bytes,
1776 .gpl_only = false,
1777 .ret_type = RET_INTEGER,
1778 .arg1_type = ARG_PTR_TO_CTX,
1779 .arg2_type = ARG_ANYTHING,
1780 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
1781 .arg4_type = ARG_CONST_SIZE,
1782};
1783
1784int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len)
1785{
1786 return ____bpf_skb_load_bytes(skb, offset, to, len);
1787}
1788
1789BPF_CALL_4(bpf_flow_dissector_load_bytes,
1790 const struct bpf_flow_dissector *, ctx, u32, offset,
1791 void *, to, u32, len)
1792{
1793 void *ptr;
1794
1795 if (unlikely(offset > 0xffff))
1796 goto err_clear;
1797
1798 if (unlikely(!ctx->skb))
1799 goto err_clear;
1800
1801 ptr = skb_header_pointer(skb: ctx->skb, offset, len, buffer: to);
1802 if (unlikely(!ptr))
1803 goto err_clear;
1804 if (ptr != to)
1805 memcpy(to, from: ptr, len);
1806
1807 return 0;
1808err_clear:
1809 memset(s: to, c: 0, n: len);
1810 return -EFAULT;
1811}
1812
1813static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = {
1814 .func = bpf_flow_dissector_load_bytes,
1815 .gpl_only = false,
1816 .ret_type = RET_INTEGER,
1817 .arg1_type = ARG_PTR_TO_CTX,
1818 .arg2_type = ARG_ANYTHING,
1819 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
1820 .arg4_type = ARG_CONST_SIZE,
1821};
1822
1823BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb,
1824 u32, offset, void *, to, u32, len, u32, start_header)
1825{
1826 u8 *end = skb_tail_pointer(skb);
1827 u8 *start, *ptr;
1828
1829 if (unlikely(offset > 0xffff))
1830 goto err_clear;
1831
1832 switch (start_header) {
1833 case BPF_HDR_START_MAC:
1834 if (unlikely(!skb_mac_header_was_set(skb)))
1835 goto err_clear;
1836 start = skb_mac_header(skb);
1837 break;
1838 case BPF_HDR_START_NET:
1839 start = skb_network_header(skb);
1840 break;
1841 default:
1842 goto err_clear;
1843 }
1844
1845 ptr = start + offset;
1846
1847 if (likely(ptr + len <= end)) {
1848 memcpy(to, from: ptr, len);
1849 return 0;
1850 }
1851
1852err_clear:
1853 memset(s: to, c: 0, n: len);
1854 return -EFAULT;
1855}
1856
1857static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = {
1858 .func = bpf_skb_load_bytes_relative,
1859 .gpl_only = false,
1860 .ret_type = RET_INTEGER,
1861 .arg1_type = ARG_PTR_TO_CTX,
1862 .arg2_type = ARG_ANYTHING,
1863 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
1864 .arg4_type = ARG_CONST_SIZE,
1865 .arg5_type = ARG_ANYTHING,
1866};
1867
1868BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len)
1869{
1870 /* Idea is the following: should the needed direct read/write
1871 * test fail during runtime, we can pull in more data and redo
1872 * again, since implicitly, we invalidate previous checks here.
1873 *
1874 * Or, since we know how much we need to make read/writeable,
1875 * this can be done once at the program beginning for direct
1876 * access case. By this we overcome limitations of only current
1877 * headroom being accessible.
1878 */
1879 return bpf_try_make_writable(skb, write_len: len ? : skb_headlen(skb));
1880}
1881
1882static const struct bpf_func_proto bpf_skb_pull_data_proto = {
1883 .func = bpf_skb_pull_data,
1884 .gpl_only = false,
1885 .ret_type = RET_INTEGER,
1886 .arg1_type = ARG_PTR_TO_CTX,
1887 .arg2_type = ARG_ANYTHING,
1888};
1889
1890BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
1891{
1892 return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
1893}
1894
1895static const struct bpf_func_proto bpf_sk_fullsock_proto = {
1896 .func = bpf_sk_fullsock,
1897 .gpl_only = false,
1898 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
1899 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
1900};
1901
1902static inline int sk_skb_try_make_writable(struct sk_buff *skb,
1903 unsigned int write_len)
1904{
1905 return __bpf_try_make_writable(skb, write_len);
1906}
1907
1908BPF_CALL_2(sk_skb_pull_data, struct sk_buff *, skb, u32, len)
1909{
1910 /* Idea is the following: should the needed direct read/write
1911 * test fail during runtime, we can pull in more data and redo
1912 * again, since implicitly, we invalidate previous checks here.
1913 *
1914 * Or, since we know how much we need to make read/writeable,
1915 * this can be done once at the program beginning for direct
1916 * access case. By this we overcome limitations of only current
1917 * headroom being accessible.
1918 */
1919 return sk_skb_try_make_writable(skb, write_len: len ? : skb_headlen(skb));
1920}
1921
1922static const struct bpf_func_proto sk_skb_pull_data_proto = {
1923 .func = sk_skb_pull_data,
1924 .gpl_only = false,
1925 .ret_type = RET_INTEGER,
1926 .arg1_type = ARG_PTR_TO_CTX,
1927 .arg2_type = ARG_ANYTHING,
1928};
1929
1930BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset,
1931 u64, from, u64, to, u64, flags)
1932{
1933 __sum16 *ptr;
1934
1935 if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK)))
1936 return -EINVAL;
1937 if (unlikely(offset > 0xffff || offset & 1))
1938 return -EFAULT;
1939 if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
1940 return -EFAULT;
1941
1942 ptr = (__sum16 *)(skb->data + offset);
1943 switch (flags & BPF_F_HDR_FIELD_MASK) {
1944 case 0:
1945 if (unlikely(from != 0))
1946 return -EINVAL;
1947
1948 csum_replace_by_diff(sum: ptr, diff: to);
1949 break;
1950 case 2:
1951 csum_replace2(sum: ptr, old: from, new: to);
1952 break;
1953 case 4:
1954 csum_replace4(sum: ptr, from, to);
1955 break;
1956 default:
1957 return -EINVAL;
1958 }
1959
1960 return 0;
1961}
1962
1963static const struct bpf_func_proto bpf_l3_csum_replace_proto = {
1964 .func = bpf_l3_csum_replace,
1965 .gpl_only = false,
1966 .ret_type = RET_INTEGER,
1967 .arg1_type = ARG_PTR_TO_CTX,
1968 .arg2_type = ARG_ANYTHING,
1969 .arg3_type = ARG_ANYTHING,
1970 .arg4_type = ARG_ANYTHING,
1971 .arg5_type = ARG_ANYTHING,
1972};
1973
1974BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset,
1975 u64, from, u64, to, u64, flags)
1976{
1977 bool is_pseudo = flags & BPF_F_PSEUDO_HDR;
1978 bool is_mmzero = flags & BPF_F_MARK_MANGLED_0;
1979 bool do_mforce = flags & BPF_F_MARK_ENFORCE;
1980 bool is_ipv6 = flags & BPF_F_IPV6;
1981 __sum16 *ptr;
1982
1983 if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE |
1984 BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK | BPF_F_IPV6)))
1985 return -EINVAL;
1986 if (unlikely(offset > 0xffff || offset & 1))
1987 return -EFAULT;
1988 if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
1989 return -EFAULT;
1990
1991 ptr = (__sum16 *)(skb->data + offset);
1992 if (is_mmzero && !do_mforce && !*ptr)
1993 return 0;
1994
1995 switch (flags & BPF_F_HDR_FIELD_MASK) {
1996 case 0:
1997 if (unlikely(from != 0))
1998 return -EINVAL;
1999
2000 inet_proto_csum_replace_by_diff(sum: ptr, skb, diff: to, pseudohdr: is_pseudo, ipv6: is_ipv6);
2001 break;
2002 case 2:
2003 inet_proto_csum_replace2(sum: ptr, skb, from, to, pseudohdr: is_pseudo);
2004 break;
2005 case 4:
2006 inet_proto_csum_replace4(sum: ptr, skb, from, to, pseudohdr: is_pseudo);
2007 break;
2008 default:
2009 return -EINVAL;
2010 }
2011
2012 if (is_mmzero && !*ptr)
2013 *ptr = CSUM_MANGLED_0;
2014 return 0;
2015}
2016
2017static const struct bpf_func_proto bpf_l4_csum_replace_proto = {
2018 .func = bpf_l4_csum_replace,
2019 .gpl_only = false,
2020 .ret_type = RET_INTEGER,
2021 .arg1_type = ARG_PTR_TO_CTX,
2022 .arg2_type = ARG_ANYTHING,
2023 .arg3_type = ARG_ANYTHING,
2024 .arg4_type = ARG_ANYTHING,
2025 .arg5_type = ARG_ANYTHING,
2026};
2027
2028BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size,
2029 __be32 *, to, u32, to_size, __wsum, seed)
2030{
2031 /* This is quite flexible, some examples:
2032 *
2033 * from_size == 0, to_size > 0, seed := csum --> pushing data
2034 * from_size > 0, to_size == 0, seed := csum --> pulling data
2035 * from_size > 0, to_size > 0, seed := 0 --> diffing data
2036 *
2037 * Even for diffing, from_size and to_size don't need to be equal.
2038 */
2039
2040 __wsum ret = seed;
2041
2042 if (from_size && to_size)
2043 ret = csum_sub(csum: csum_partial(buff: to, len: to_size, sum: ret),
2044 addend: csum_partial(buff: from, len: from_size, sum: 0));
2045 else if (to_size)
2046 ret = csum_partial(buff: to, len: to_size, sum: ret);
2047
2048 else if (from_size)
2049 ret = ~csum_partial(buff: from, len: from_size, sum: ~ret);
2050
2051 return csum_from32to16(sum: (__force unsigned int)ret);
2052}
2053
2054static const struct bpf_func_proto bpf_csum_diff_proto = {
2055 .func = bpf_csum_diff,
2056 .gpl_only = false,
2057 .pkt_access = true,
2058 .ret_type = RET_INTEGER,
2059 .arg1_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
2060 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
2061 .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
2062 .arg4_type = ARG_CONST_SIZE_OR_ZERO,
2063 .arg5_type = ARG_ANYTHING,
2064};
2065
2066BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum)
2067{
2068 /* The interface is to be used in combination with bpf_csum_diff()
2069 * for direct packet writes. csum rotation for alignment as well
2070 * as emulating csum_sub() can be done from the eBPF program.
2071 */
2072 if (skb->ip_summed == CHECKSUM_COMPLETE)
2073 return (skb->csum = csum_add(csum: skb->csum, addend: csum));
2074
2075 return -ENOTSUPP;
2076}
2077
2078static const struct bpf_func_proto bpf_csum_update_proto = {
2079 .func = bpf_csum_update,
2080 .gpl_only = false,
2081 .ret_type = RET_INTEGER,
2082 .arg1_type = ARG_PTR_TO_CTX,
2083 .arg2_type = ARG_ANYTHING,
2084};
2085
2086BPF_CALL_2(bpf_csum_level, struct sk_buff *, skb, u64, level)
2087{
2088 /* The interface is to be used in combination with bpf_skb_adjust_room()
2089 * for encap/decap of packet headers when BPF_F_ADJ_ROOM_NO_CSUM_RESET
2090 * is passed as flags, for example.
2091 */
2092 switch (level) {
2093 case BPF_CSUM_LEVEL_INC:
2094 __skb_incr_checksum_unnecessary(skb);
2095 break;
2096 case BPF_CSUM_LEVEL_DEC:
2097 __skb_decr_checksum_unnecessary(skb);
2098 break;
2099 case BPF_CSUM_LEVEL_RESET:
2100 __skb_reset_checksum_unnecessary(skb);
2101 break;
2102 case BPF_CSUM_LEVEL_QUERY:
2103 return skb->ip_summed == CHECKSUM_UNNECESSARY ?
2104 skb->csum_level : -EACCES;
2105 default:
2106 return -EINVAL;
2107 }
2108
2109 return 0;
2110}
2111
2112static const struct bpf_func_proto bpf_csum_level_proto = {
2113 .func = bpf_csum_level,
2114 .gpl_only = false,
2115 .ret_type = RET_INTEGER,
2116 .arg1_type = ARG_PTR_TO_CTX,
2117 .arg2_type = ARG_ANYTHING,
2118};
2119
2120static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
2121{
2122 return dev_forward_skb_nomtu(dev, skb);
2123}
2124
2125static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
2126 struct sk_buff *skb)
2127{
2128 int ret = ____dev_forward_skb(dev, skb, check_mtu: false);
2129
2130 if (likely(!ret)) {
2131 skb->dev = dev;
2132 ret = netif_rx(skb);
2133 }
2134
2135 return ret;
2136}
2137
2138static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
2139{
2140 int ret;
2141
2142 if (dev_xmit_recursion()) {
2143 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2144 kfree_skb(skb);
2145 return -ENETDOWN;
2146 }
2147
2148 skb->dev = dev;
2149 skb_set_redirected_noclear(skb, from_ingress: skb_at_tc_ingress(skb));
2150 skb_clear_tstamp(skb);
2151
2152 dev_xmit_recursion_inc();
2153 ret = dev_queue_xmit(skb);
2154 dev_xmit_recursion_dec();
2155
2156 return ret;
2157}
2158
2159static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev,
2160 u32 flags)
2161{
2162 unsigned int mlen = skb_network_offset(skb);
2163
2164 if (unlikely(skb->len <= mlen)) {
2165 kfree_skb(skb);
2166 return -ERANGE;
2167 }
2168
2169 if (mlen) {
2170 __skb_pull(skb, len: mlen);
2171
2172 /* At ingress, the mac header has already been pulled once.
2173 * At egress, skb_pospull_rcsum has to be done in case that
2174 * the skb is originated from ingress (i.e. a forwarded skb)
2175 * to ensure that rcsum starts at net header.
2176 */
2177 if (!skb_at_tc_ingress(skb))
2178 skb_postpull_rcsum(skb, start: skb_mac_header(skb), len: mlen);
2179 }
2180 skb_pop_mac_header(skb);
2181 skb_reset_mac_len(skb);
2182 return flags & BPF_F_INGRESS ?
2183 __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb);
2184}
2185
2186static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev,
2187 u32 flags)
2188{
2189 /* Verify that a link layer header is carried */
2190 if (unlikely(skb->mac_header >= skb->network_header || skb->len == 0)) {
2191 kfree_skb(skb);
2192 return -ERANGE;
2193 }
2194
2195 bpf_push_mac_rcsum(skb);
2196 return flags & BPF_F_INGRESS ?
2197 __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
2198}
2199
2200static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev,
2201 u32 flags)
2202{
2203 if (dev_is_mac_header_xmit(dev))
2204 return __bpf_redirect_common(skb, dev, flags);
2205 else
2206 return __bpf_redirect_no_mac(skb, dev, flags);
2207}
2208
2209#if IS_ENABLED(CONFIG_IPV6)
2210static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
2211 struct net_device *dev, struct bpf_nh_params *nh)
2212{
2213 u32 hh_len = LL_RESERVED_SPACE(dev);
2214 const struct in6_addr *nexthop;
2215 struct dst_entry *dst = NULL;
2216 struct neighbour *neigh;
2217
2218 if (dev_xmit_recursion()) {
2219 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2220 goto out_drop;
2221 }
2222
2223 skb->dev = dev;
2224 skb_clear_tstamp(skb);
2225
2226 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
2227 skb = skb_expand_head(skb, headroom: hh_len);
2228 if (!skb)
2229 return -ENOMEM;
2230 }
2231
2232 rcu_read_lock();
2233 if (!nh) {
2234 dst = skb_dst(skb);
2235 nexthop = rt6_nexthop(dst_rt6_info(dst),
2236 daddr: &ipv6_hdr(skb)->daddr);
2237 } else {
2238 nexthop = &nh->ipv6_nh;
2239 }
2240 neigh = ip_neigh_gw6(dev, addr: nexthop);
2241 if (likely(!IS_ERR(neigh))) {
2242 int ret;
2243
2244 sock_confirm_neigh(skb, n: neigh);
2245 local_bh_disable();
2246 dev_xmit_recursion_inc();
2247 ret = neigh_output(n: neigh, skb, skip_cache: false);
2248 dev_xmit_recursion_dec();
2249 local_bh_enable();
2250 rcu_read_unlock();
2251 return ret;
2252 }
2253 rcu_read_unlock();
2254 if (dst)
2255 IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
2256out_drop:
2257 kfree_skb(skb);
2258 return -ENETDOWN;
2259}
2260
2261static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
2262 struct bpf_nh_params *nh)
2263{
2264 const struct ipv6hdr *ip6h = ipv6_hdr(skb);
2265 struct net *net = dev_net(dev);
2266 int err, ret = NET_XMIT_DROP;
2267
2268 if (!nh) {
2269 struct dst_entry *dst;
2270 struct flowi6 fl6 = {
2271 .flowi6_flags = FLOWI_FLAG_ANYSRC,
2272 .flowi6_mark = skb->mark,
2273 .flowlabel = ip6_flowinfo(hdr: ip6h),
2274 .flowi6_oif = dev->ifindex,
2275 .flowi6_proto = ip6h->nexthdr,
2276 .daddr = ip6h->daddr,
2277 .saddr = ip6h->saddr,
2278 };
2279
2280 dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
2281 if (IS_ERR(ptr: dst))
2282 goto out_drop;
2283
2284 skb_dst_drop(skb);
2285 skb_dst_set(skb, dst);
2286 } else if (nh->nh_family != AF_INET6) {
2287 goto out_drop;
2288 }
2289
2290 err = bpf_out_neigh_v6(net, skb, dev, nh);
2291 if (unlikely(net_xmit_eval(err)))
2292 DEV_STATS_INC(dev, tx_errors);
2293 else
2294 ret = NET_XMIT_SUCCESS;
2295 goto out_xmit;
2296out_drop:
2297 DEV_STATS_INC(dev, tx_errors);
2298 kfree_skb(skb);
2299out_xmit:
2300 return ret;
2301}
2302#else
2303static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
2304 struct bpf_nh_params *nh)
2305{
2306 kfree_skb(skb);
2307 return NET_XMIT_DROP;
2308}
2309#endif /* CONFIG_IPV6 */
2310
2311#if IS_ENABLED(CONFIG_INET)
2312static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
2313 struct net_device *dev, struct bpf_nh_params *nh)
2314{
2315 u32 hh_len = LL_RESERVED_SPACE(dev);
2316 struct neighbour *neigh;
2317 bool is_v6gw = false;
2318
2319 if (dev_xmit_recursion()) {
2320 net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2321 goto out_drop;
2322 }
2323
2324 skb->dev = dev;
2325 skb_clear_tstamp(skb);
2326
2327 if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
2328 skb = skb_expand_head(skb, headroom: hh_len);
2329 if (!skb)
2330 return -ENOMEM;
2331 }
2332
2333 rcu_read_lock();
2334 if (!nh) {
2335 struct rtable *rt = skb_rtable(skb);
2336
2337 neigh = ip_neigh_for_gw(rt, skb, is_v6gw: &is_v6gw);
2338 } else if (nh->nh_family == AF_INET6) {
2339 neigh = ip_neigh_gw6(dev, addr: &nh->ipv6_nh);
2340 is_v6gw = true;
2341 } else if (nh->nh_family == AF_INET) {
2342 neigh = ip_neigh_gw4(dev, daddr: nh->ipv4_nh);
2343 } else {
2344 rcu_read_unlock();
2345 goto out_drop;
2346 }
2347
2348 if (likely(!IS_ERR(neigh))) {
2349 int ret;
2350
2351 sock_confirm_neigh(skb, n: neigh);
2352 local_bh_disable();
2353 dev_xmit_recursion_inc();
2354 ret = neigh_output(n: neigh, skb, skip_cache: is_v6gw);
2355 dev_xmit_recursion_dec();
2356 local_bh_enable();
2357 rcu_read_unlock();
2358 return ret;
2359 }
2360 rcu_read_unlock();
2361out_drop:
2362 kfree_skb(skb);
2363 return -ENETDOWN;
2364}
2365
2366static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
2367 struct bpf_nh_params *nh)
2368{
2369 const struct iphdr *ip4h = ip_hdr(skb);
2370 struct net *net = dev_net(dev);
2371 int err, ret = NET_XMIT_DROP;
2372
2373 if (!nh) {
2374 struct flowi4 fl4 = {
2375 .flowi4_flags = FLOWI_FLAG_ANYSRC,
2376 .flowi4_mark = skb->mark,
2377 .flowi4_dscp = ip4h_dscp(ip4h),
2378 .flowi4_oif = dev->ifindex,
2379 .flowi4_proto = ip4h->protocol,
2380 .daddr = ip4h->daddr,
2381 .saddr = ip4h->saddr,
2382 };
2383 struct rtable *rt;
2384
2385 rt = ip_route_output_flow(net, flp: &fl4, NULL);
2386 if (IS_ERR(ptr: rt))
2387 goto out_drop;
2388 if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
2389 ip_rt_put(rt);
2390 goto out_drop;
2391 }
2392
2393 skb_dst_drop(skb);
2394 skb_dst_set(skb, dst: &rt->dst);
2395 }
2396
2397 err = bpf_out_neigh_v4(net, skb, dev, nh);
2398 if (unlikely(net_xmit_eval(err)))
2399 DEV_STATS_INC(dev, tx_errors);
2400 else
2401 ret = NET_XMIT_SUCCESS;
2402 goto out_xmit;
2403out_drop:
2404 DEV_STATS_INC(dev, tx_errors);
2405 kfree_skb(skb);
2406out_xmit:
2407 return ret;
2408}
2409#else
2410static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
2411 struct bpf_nh_params *nh)
2412{
2413 kfree_skb(skb);
2414 return NET_XMIT_DROP;
2415}
2416#endif /* CONFIG_INET */
2417
2418static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev,
2419 struct bpf_nh_params *nh)
2420{
2421 struct ethhdr *ethh = eth_hdr(skb);
2422
2423 if (unlikely(skb->mac_header >= skb->network_header))
2424 goto out;
2425 bpf_push_mac_rcsum(skb);
2426 if (is_multicast_ether_addr(addr: ethh->h_dest))
2427 goto out;
2428
2429 skb_pull(skb, len: sizeof(*ethh));
2430 skb_unset_mac_header(skb);
2431 skb_reset_network_header(skb);
2432
2433 if (skb->protocol == htons(ETH_P_IP))
2434 return __bpf_redirect_neigh_v4(skb, dev, nh);
2435 else if (skb->protocol == htons(ETH_P_IPV6))
2436 return __bpf_redirect_neigh_v6(skb, dev, nh);
2437out:
2438 kfree_skb(skb);
2439 return -ENOTSUPP;
2440}
2441
2442/* Internal, non-exposed redirect flags. */
2443enum {
2444 BPF_F_NEIGH = (1ULL << 16),
2445 BPF_F_PEER = (1ULL << 17),
2446 BPF_F_NEXTHOP = (1ULL << 18),
2447#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH | BPF_F_PEER | BPF_F_NEXTHOP)
2448};
2449
2450BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
2451{
2452 struct net_device *dev;
2453 struct sk_buff *clone;
2454 int ret;
2455
2456 BUILD_BUG_ON(BPF_F_REDIRECT_INTERNAL & BPF_F_REDIRECT_FLAGS);
2457
2458 if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
2459 return -EINVAL;
2460
2461 dev = dev_get_by_index_rcu(net: dev_net(dev: skb->dev), ifindex);
2462 if (unlikely(!dev))
2463 return -EINVAL;
2464
2465 clone = skb_clone(skb, GFP_ATOMIC);
2466 if (unlikely(!clone))
2467 return -ENOMEM;
2468
2469 /* For direct write, we need to keep the invariant that the skbs
2470 * we're dealing with need to be uncloned. Should uncloning fail
2471 * here, we need to free the just generated clone to unclone once
2472 * again.
2473 */
2474 ret = bpf_try_make_head_writable(skb);
2475 if (unlikely(ret)) {
2476 kfree_skb(skb: clone);
2477 return -ENOMEM;
2478 }
2479
2480 return __bpf_redirect(skb: clone, dev, flags);
2481}
2482
2483static const struct bpf_func_proto bpf_clone_redirect_proto = {
2484 .func = bpf_clone_redirect,
2485 .gpl_only = false,
2486 .ret_type = RET_INTEGER,
2487 .arg1_type = ARG_PTR_TO_CTX,
2488 .arg2_type = ARG_ANYTHING,
2489 .arg3_type = ARG_ANYTHING,
2490};
2491
2492static struct net_device *skb_get_peer_dev(struct net_device *dev)
2493{
2494 const struct net_device_ops *ops = dev->netdev_ops;
2495
2496 if (likely(ops->ndo_get_peer_dev))
2497 return INDIRECT_CALL_1(ops->ndo_get_peer_dev,
2498 netkit_peer_dev, dev);
2499 return NULL;
2500}
2501
2502int skb_do_redirect(struct sk_buff *skb)
2503{
2504 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
2505 struct net *net = dev_net(dev: skb->dev);
2506 struct net_device *dev;
2507 u32 flags = ri->flags;
2508
2509 dev = dev_get_by_index_rcu(net, ifindex: ri->tgt_index);
2510 ri->tgt_index = 0;
2511 ri->flags = 0;
2512 if (unlikely(!dev))
2513 goto out_drop;
2514 if (flags & BPF_F_PEER) {
2515 if (unlikely(!skb_at_tc_ingress(skb)))
2516 goto out_drop;
2517 dev = skb_get_peer_dev(dev);
2518 if (unlikely(!dev ||
2519 !(dev->flags & IFF_UP) ||
2520 net_eq(net, dev_net(dev))))
2521 goto out_drop;
2522 skb->dev = dev;
2523 dev_sw_netstats_rx_add(dev, len: skb->len);
2524 skb_scrub_packet(skb, xnet: false);
2525 return -EAGAIN;
2526 }
2527 return flags & BPF_F_NEIGH ?
2528 __bpf_redirect_neigh(skb, dev, nh: flags & BPF_F_NEXTHOP ?
2529 &ri->nh : NULL) :
2530 __bpf_redirect(skb, dev, flags);
2531out_drop:
2532 kfree_skb(skb);
2533 return -EINVAL;
2534}
2535
2536BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
2537{
2538 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
2539
2540 if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
2541 return TC_ACT_SHOT;
2542
2543 ri->flags = flags;
2544 ri->tgt_index = ifindex;
2545
2546 return TC_ACT_REDIRECT;
2547}
2548
2549static const struct bpf_func_proto bpf_redirect_proto = {
2550 .func = bpf_redirect,
2551 .gpl_only = false,
2552 .ret_type = RET_INTEGER,
2553 .arg1_type = ARG_ANYTHING,
2554 .arg2_type = ARG_ANYTHING,
2555};
2556
2557BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
2558{
2559 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
2560
2561 if (unlikely(flags))
2562 return TC_ACT_SHOT;
2563
2564 ri->flags = BPF_F_PEER;
2565 ri->tgt_index = ifindex;
2566
2567 return TC_ACT_REDIRECT;
2568}
2569
2570static const struct bpf_func_proto bpf_redirect_peer_proto = {
2571 .func = bpf_redirect_peer,
2572 .gpl_only = false,
2573 .ret_type = RET_INTEGER,
2574 .arg1_type = ARG_ANYTHING,
2575 .arg2_type = ARG_ANYTHING,
2576};
2577
2578BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params,
2579 int, plen, u64, flags)
2580{
2581 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
2582
2583 if (unlikely((plen && plen < sizeof(*params)) || flags))
2584 return TC_ACT_SHOT;
2585
2586 ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0);
2587 ri->tgt_index = ifindex;
2588
2589 BUILD_BUG_ON(sizeof(struct bpf_redir_neigh) != sizeof(struct bpf_nh_params));
2590 if (plen)
2591 memcpy(to: &ri->nh, from: params, len: sizeof(ri->nh));
2592
2593 return TC_ACT_REDIRECT;
2594}
2595
2596static const struct bpf_func_proto bpf_redirect_neigh_proto = {
2597 .func = bpf_redirect_neigh,
2598 .gpl_only = false,
2599 .ret_type = RET_INTEGER,
2600 .arg1_type = ARG_ANYTHING,
2601 .arg2_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
2602 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
2603 .arg4_type = ARG_ANYTHING,
2604};
2605
2606BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes)
2607{
2608 msg->apply_bytes = bytes;
2609 return 0;
2610}
2611
2612static const struct bpf_func_proto bpf_msg_apply_bytes_proto = {
2613 .func = bpf_msg_apply_bytes,
2614 .gpl_only = false,
2615 .ret_type = RET_INTEGER,
2616 .arg1_type = ARG_PTR_TO_CTX,
2617 .arg2_type = ARG_ANYTHING,
2618};
2619
2620BPF_CALL_2(bpf_msg_cork_bytes, struct sk_msg *, msg, u32, bytes)
2621{
2622 msg->cork_bytes = bytes;
2623 return 0;
2624}
2625
2626static void sk_msg_reset_curr(struct sk_msg *msg)
2627{
2628 if (!msg->sg.size) {
2629 msg->sg.curr = msg->sg.start;
2630 msg->sg.copybreak = 0;
2631 } else {
2632 u32 i = msg->sg.end;
2633
2634 sk_msg_iter_var_prev(i);
2635 msg->sg.curr = i;
2636 msg->sg.copybreak = msg->sg.data[i].length;
2637 }
2638}
2639
2640static const struct bpf_func_proto bpf_msg_cork_bytes_proto = {
2641 .func = bpf_msg_cork_bytes,
2642 .gpl_only = false,
2643 .ret_type = RET_INTEGER,
2644 .arg1_type = ARG_PTR_TO_CTX,
2645 .arg2_type = ARG_ANYTHING,
2646};
2647
2648BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start,
2649 u32, end, u64, flags)
2650{
2651 u32 len = 0, offset = 0, copy = 0, poffset = 0, bytes = end - start;
2652 u32 first_sge, last_sge, i, shift, bytes_sg_total;
2653 struct scatterlist *sge;
2654 u8 *raw, *to, *from;
2655 struct page *page;
2656
2657 if (unlikely(flags || end <= start))
2658 return -EINVAL;
2659
2660 /* First find the starting scatterlist element */
2661 i = msg->sg.start;
2662 do {
2663 offset += len;
2664 len = sk_msg_elem(msg, which: i)->length;
2665 if (start < offset + len)
2666 break;
2667 sk_msg_iter_var_next(i);
2668 } while (i != msg->sg.end);
2669
2670 if (unlikely(start >= offset + len))
2671 return -EINVAL;
2672
2673 first_sge = i;
2674 /* The start may point into the sg element so we need to also
2675 * account for the headroom.
2676 */
2677 bytes_sg_total = start - offset + bytes;
2678 if (!test_bit(i, msg->sg.copy) && bytes_sg_total <= len)
2679 goto out;
2680
2681 /* At this point we need to linearize multiple scatterlist
2682 * elements or a single shared page. Either way we need to
2683 * copy into a linear buffer exclusively owned by BPF. Then
2684 * place the buffer in the scatterlist and fixup the original
2685 * entries by removing the entries now in the linear buffer
2686 * and shifting the remaining entries. For now we do not try
2687 * to copy partial entries to avoid complexity of running out
2688 * of sg_entry slots. The downside is reading a single byte
2689 * will copy the entire sg entry.
2690 */
2691 do {
2692 copy += sk_msg_elem(msg, which: i)->length;
2693 sk_msg_iter_var_next(i);
2694 if (bytes_sg_total <= copy)
2695 break;
2696 } while (i != msg->sg.end);
2697 last_sge = i;
2698
2699 if (unlikely(bytes_sg_total > copy))
2700 return -EINVAL;
2701
2702 page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
2703 get_order(copy));
2704 if (unlikely(!page))
2705 return -ENOMEM;
2706
2707 raw = page_address(page);
2708 i = first_sge;
2709 do {
2710 sge = sk_msg_elem(msg, which: i);
2711 from = sg_virt(sg: sge);
2712 len = sge->length;
2713 to = raw + poffset;
2714
2715 memcpy(to, from, len);
2716 poffset += len;
2717 sge->length = 0;
2718 put_page(page: sg_page(sg: sge));
2719
2720 sk_msg_iter_var_next(i);
2721 } while (i != last_sge);
2722
2723 sg_set_page(sg: &msg->sg.data[first_sge], page, len: copy, offset: 0);
2724
2725 /* To repair sg ring we need to shift entries. If we only
2726 * had a single entry though we can just replace it and
2727 * be done. Otherwise walk the ring and shift the entries.
2728 */
2729 WARN_ON_ONCE(last_sge == first_sge);
2730 shift = last_sge > first_sge ?
2731 last_sge - first_sge - 1 :
2732 NR_MSG_FRAG_IDS - first_sge + last_sge - 1;
2733 if (!shift)
2734 goto out;
2735
2736 i = first_sge;
2737 sk_msg_iter_var_next(i);
2738 do {
2739 u32 move_from;
2740
2741 if (i + shift >= NR_MSG_FRAG_IDS)
2742 move_from = i + shift - NR_MSG_FRAG_IDS;
2743 else
2744 move_from = i + shift;
2745 if (move_from == msg->sg.end)
2746 break;
2747
2748 msg->sg.data[i] = msg->sg.data[move_from];
2749 msg->sg.data[move_from].length = 0;
2750 msg->sg.data[move_from].page_link = 0;
2751 msg->sg.data[move_from].offset = 0;
2752 sk_msg_iter_var_next(i);
2753 } while (1);
2754
2755 msg->sg.end = msg->sg.end - shift > msg->sg.end ?
2756 msg->sg.end - shift + NR_MSG_FRAG_IDS :
2757 msg->sg.end - shift;
2758out:
2759 sk_msg_reset_curr(msg);
2760 msg->data = sg_virt(sg: &msg->sg.data[first_sge]) + start - offset;
2761 msg->data_end = msg->data + bytes;
2762 return 0;
2763}
2764
2765static const struct bpf_func_proto bpf_msg_pull_data_proto = {
2766 .func = bpf_msg_pull_data,
2767 .gpl_only = false,
2768 .ret_type = RET_INTEGER,
2769 .arg1_type = ARG_PTR_TO_CTX,
2770 .arg2_type = ARG_ANYTHING,
2771 .arg3_type = ARG_ANYTHING,
2772 .arg4_type = ARG_ANYTHING,
2773};
2774
2775BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start,
2776 u32, len, u64, flags)
2777{
2778 struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge;
2779 u32 new, i = 0, l = 0, space, copy = 0, offset = 0;
2780 u8 *raw, *to, *from;
2781 struct page *page;
2782
2783 if (unlikely(flags))
2784 return -EINVAL;
2785
2786 if (unlikely(len == 0))
2787 return 0;
2788
2789 /* First find the starting scatterlist element */
2790 i = msg->sg.start;
2791 do {
2792 offset += l;
2793 l = sk_msg_elem(msg, which: i)->length;
2794
2795 if (start < offset + l)
2796 break;
2797 sk_msg_iter_var_next(i);
2798 } while (i != msg->sg.end);
2799
2800 if (start > offset + l)
2801 return -EINVAL;
2802
2803 space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
2804
2805 /* If no space available will fallback to copy, we need at
2806 * least one scatterlist elem available to push data into
2807 * when start aligns to the beginning of an element or two
2808 * when it falls inside an element. We handle the start equals
2809 * offset case because its the common case for inserting a
2810 * header.
2811 */
2812 if (!space || (space == 1 && start != offset))
2813 copy = msg->sg.data[i].length;
2814
2815 page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
2816 get_order(copy + len));
2817 if (unlikely(!page))
2818 return -ENOMEM;
2819
2820 if (copy) {
2821 int front, back;
2822
2823 raw = page_address(page);
2824
2825 if (i == msg->sg.end)
2826 sk_msg_iter_var_prev(i);
2827 psge = sk_msg_elem(msg, which: i);
2828 front = start - offset;
2829 back = psge->length - front;
2830 from = sg_virt(sg: psge);
2831
2832 if (front)
2833 memcpy(to: raw, from, len: front);
2834
2835 if (back) {
2836 from += front;
2837 to = raw + front + len;
2838
2839 memcpy(to, from, len: back);
2840 }
2841
2842 put_page(page: sg_page(sg: psge));
2843 new = i;
2844 goto place_new;
2845 }
2846
2847 if (start - offset) {
2848 if (i == msg->sg.end)
2849 sk_msg_iter_var_prev(i);
2850 psge = sk_msg_elem(msg, which: i);
2851 rsge = sk_msg_elem_cpy(msg, which: i);
2852
2853 psge->length = start - offset;
2854 rsge.length -= psge->length;
2855 rsge.offset += start;
2856
2857 sk_msg_iter_var_next(i);
2858 sg_unmark_end(sg: psge);
2859 sg_unmark_end(sg: &rsge);
2860 }
2861
2862 /* Slot(s) to place newly allocated data */
2863 sk_msg_iter_next(msg, end);
2864 new = i;
2865 sk_msg_iter_var_next(i);
2866
2867 if (i == msg->sg.end) {
2868 if (!rsge.length)
2869 goto place_new;
2870 sk_msg_iter_next(msg, end);
2871 goto place_new;
2872 }
2873
2874 /* Shift one or two slots as needed */
2875 sge = sk_msg_elem_cpy(msg, which: new);
2876 sg_unmark_end(sg: &sge);
2877
2878 nsge = sk_msg_elem_cpy(msg, which: i);
2879 if (rsge.length) {
2880 sk_msg_iter_var_next(i);
2881 nnsge = sk_msg_elem_cpy(msg, which: i);
2882 sk_msg_iter_next(msg, end);
2883 }
2884
2885 while (i != msg->sg.end) {
2886 msg->sg.data[i] = sge;
2887 sge = nsge;
2888 sk_msg_iter_var_next(i);
2889 if (rsge.length) {
2890 nsge = nnsge;
2891 nnsge = sk_msg_elem_cpy(msg, which: i);
2892 } else {
2893 nsge = sk_msg_elem_cpy(msg, which: i);
2894 }
2895 }
2896
2897place_new:
2898 /* Place newly allocated data buffer */
2899 sk_mem_charge(sk: msg->sk, size: len);
2900 msg->sg.size += len;
2901 __clear_bit(new, msg->sg.copy);
2902 sg_set_page(sg: &msg->sg.data[new], page, len: len + copy, offset: 0);
2903 if (rsge.length) {
2904 get_page(page: sg_page(sg: &rsge));
2905 sk_msg_iter_var_next(new);
2906 msg->sg.data[new] = rsge;
2907 }
2908
2909 sk_msg_reset_curr(msg);
2910 sk_msg_compute_data_pointers(msg);
2911 return 0;
2912}
2913
2914static const struct bpf_func_proto bpf_msg_push_data_proto = {
2915 .func = bpf_msg_push_data,
2916 .gpl_only = false,
2917 .ret_type = RET_INTEGER,
2918 .arg1_type = ARG_PTR_TO_CTX,
2919 .arg2_type = ARG_ANYTHING,
2920 .arg3_type = ARG_ANYTHING,
2921 .arg4_type = ARG_ANYTHING,
2922};
2923
2924static void sk_msg_shift_left(struct sk_msg *msg, int i)
2925{
2926 struct scatterlist *sge = sk_msg_elem(msg, which: i);
2927 int prev;
2928
2929 put_page(page: sg_page(sg: sge));
2930 do {
2931 prev = i;
2932 sk_msg_iter_var_next(i);
2933 msg->sg.data[prev] = msg->sg.data[i];
2934 } while (i != msg->sg.end);
2935
2936 sk_msg_iter_prev(msg, end);
2937}
2938
2939static void sk_msg_shift_right(struct sk_msg *msg, int i)
2940{
2941 struct scatterlist tmp, sge;
2942
2943 sk_msg_iter_next(msg, end);
2944 sge = sk_msg_elem_cpy(msg, which: i);
2945 sk_msg_iter_var_next(i);
2946 tmp = sk_msg_elem_cpy(msg, which: i);
2947
2948 while (i != msg->sg.end) {
2949 msg->sg.data[i] = sge;
2950 sk_msg_iter_var_next(i);
2951 sge = tmp;
2952 tmp = sk_msg_elem_cpy(msg, which: i);
2953 }
2954}
2955
2956BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,
2957 u32, len, u64, flags)
2958{
2959 u32 i = 0, l = 0, space, offset = 0;
2960 u64 last = start + len;
2961 int pop;
2962
2963 if (unlikely(flags))
2964 return -EINVAL;
2965
2966 if (unlikely(len == 0))
2967 return 0;
2968
2969 /* First find the starting scatterlist element */
2970 i = msg->sg.start;
2971 do {
2972 offset += l;
2973 l = sk_msg_elem(msg, which: i)->length;
2974
2975 if (start < offset + l)
2976 break;
2977 sk_msg_iter_var_next(i);
2978 } while (i != msg->sg.end);
2979
2980 /* Bounds checks: start and pop must be inside message */
2981 if (start >= offset + l || last > msg->sg.size)
2982 return -EINVAL;
2983
2984 space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
2985
2986 pop = len;
2987 /* --------------| offset
2988 * -| start |-------- len -------|
2989 *
2990 * |----- a ----|-------- pop -------|----- b ----|
2991 * |______________________________________________| length
2992 *
2993 *
2994 * a: region at front of scatter element to save
2995 * b: region at back of scatter element to save when length > A + pop
2996 * pop: region to pop from element, same as input 'pop' here will be
2997 * decremented below per iteration.
2998 *
2999 * Two top-level cases to handle when start != offset, first B is non
3000 * zero and second B is zero corresponding to when a pop includes more
3001 * than one element.
3002 *
3003 * Then if B is non-zero AND there is no space allocate space and
3004 * compact A, B regions into page. If there is space shift ring to
3005 * the right free'ing the next element in ring to place B, leaving
3006 * A untouched except to reduce length.
3007 */
3008 if (start != offset) {
3009 struct scatterlist *nsge, *sge = sk_msg_elem(msg, which: i);
3010 int a = start - offset;
3011 int b = sge->length - pop - a;
3012
3013 sk_msg_iter_var_next(i);
3014
3015 if (b > 0) {
3016 if (space) {
3017 sge->length = a;
3018 sk_msg_shift_right(msg, i);
3019 nsge = sk_msg_elem(msg, which: i);
3020 get_page(page: sg_page(sg: sge));
3021 sg_set_page(sg: nsge,
3022 page: sg_page(sg: sge),
3023 len: b, offset: sge->offset + pop + a);
3024 } else {
3025 struct page *page, *orig;
3026 u8 *to, *from;
3027
3028 page = alloc_pages(__GFP_NOWARN |
3029 __GFP_COMP | GFP_ATOMIC,
3030 get_order(a + b));
3031 if (unlikely(!page))
3032 return -ENOMEM;
3033
3034 orig = sg_page(sg: sge);
3035 from = sg_virt(sg: sge);
3036 to = page_address(page);
3037 memcpy(to, from, len: a);
3038 memcpy(to: to + a, from: from + a + pop, len: b);
3039 sg_set_page(sg: sge, page, len: a + b, offset: 0);
3040 put_page(page: orig);
3041 }
3042 pop = 0;
3043 } else {
3044 pop -= (sge->length - a);
3045 sge->length = a;
3046 }
3047 }
3048
3049 /* From above the current layout _must_ be as follows,
3050 *
3051 * -| offset
3052 * -| start
3053 *
3054 * |---- pop ---|---------------- b ------------|
3055 * |____________________________________________| length
3056 *
3057 * Offset and start of the current msg elem are equal because in the
3058 * previous case we handled offset != start and either consumed the
3059 * entire element and advanced to the next element OR pop == 0.
3060 *
3061 * Two cases to handle here are first pop is less than the length
3062 * leaving some remainder b above. Simply adjust the element's layout
3063 * in this case. Or pop >= length of the element so that b = 0. In this
3064 * case advance to next element decrementing pop.
3065 */
3066 while (pop) {
3067 struct scatterlist *sge = sk_msg_elem(msg, which: i);
3068
3069 if (pop < sge->length) {
3070 sge->length -= pop;
3071 sge->offset += pop;
3072 pop = 0;
3073 } else {
3074 pop -= sge->length;
3075 sk_msg_shift_left(msg, i);
3076 }
3077 }
3078
3079 sk_mem_uncharge(sk: msg->sk, size: len - pop);
3080 msg->sg.size -= (len - pop);
3081 sk_msg_reset_curr(msg);
3082 sk_msg_compute_data_pointers(msg);
3083 return 0;
3084}
3085
3086static const struct bpf_func_proto bpf_msg_pop_data_proto = {
3087 .func = bpf_msg_pop_data,
3088 .gpl_only = false,
3089 .ret_type = RET_INTEGER,
3090 .arg1_type = ARG_PTR_TO_CTX,
3091 .arg2_type = ARG_ANYTHING,
3092 .arg3_type = ARG_ANYTHING,
3093 .arg4_type = ARG_ANYTHING,
3094};
3095
3096#ifdef CONFIG_CGROUP_NET_CLASSID
3097BPF_CALL_0(bpf_get_cgroup_classid_curr)
3098{
3099 return __task_get_classid(current);
3100}
3101
3102const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
3103 .func = bpf_get_cgroup_classid_curr,
3104 .gpl_only = false,
3105 .ret_type = RET_INTEGER,
3106};
3107
3108BPF_CALL_1(bpf_skb_cgroup_classid, const struct sk_buff *, skb)
3109{
3110 struct sock *sk = skb_to_full_sk(skb);
3111
3112 if (!sk || !sk_fullsock(sk))
3113 return 0;
3114
3115 return sock_cgroup_classid(skcd: &sk->sk_cgrp_data);
3116}
3117
3118static const struct bpf_func_proto bpf_skb_cgroup_classid_proto = {
3119 .func = bpf_skb_cgroup_classid,
3120 .gpl_only = false,
3121 .ret_type = RET_INTEGER,
3122 .arg1_type = ARG_PTR_TO_CTX,
3123};
3124#endif
3125
3126BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
3127{
3128 return task_get_classid(skb);
3129}
3130
3131static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
3132 .func = bpf_get_cgroup_classid,
3133 .gpl_only = false,
3134 .ret_type = RET_INTEGER,
3135 .arg1_type = ARG_PTR_TO_CTX,
3136};
3137
3138BPF_CALL_1(bpf_get_route_realm, const struct sk_buff *, skb)
3139{
3140 return dst_tclassid(skb);
3141}
3142
3143static const struct bpf_func_proto bpf_get_route_realm_proto = {
3144 .func = bpf_get_route_realm,
3145 .gpl_only = false,
3146 .ret_type = RET_INTEGER,
3147 .arg1_type = ARG_PTR_TO_CTX,
3148};
3149
3150BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb)
3151{
3152 /* If skb_clear_hash() was called due to mangling, we can
3153 * trigger SW recalculation here. Later access to hash
3154 * can then use the inline skb->hash via context directly
3155 * instead of calling this helper again.
3156 */
3157 return skb_get_hash(skb);
3158}
3159
3160static const struct bpf_func_proto bpf_get_hash_recalc_proto = {
3161 .func = bpf_get_hash_recalc,
3162 .gpl_only = false,
3163 .ret_type = RET_INTEGER,
3164 .arg1_type = ARG_PTR_TO_CTX,
3165};
3166
3167BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb)
3168{
3169 /* After all direct packet write, this can be used once for
3170 * triggering a lazy recalc on next skb_get_hash() invocation.
3171 */
3172 skb_clear_hash(skb);
3173 return 0;
3174}
3175
3176static const struct bpf_func_proto bpf_set_hash_invalid_proto = {
3177 .func = bpf_set_hash_invalid,
3178 .gpl_only = false,
3179 .ret_type = RET_INTEGER,
3180 .arg1_type = ARG_PTR_TO_CTX,
3181};
3182
3183BPF_CALL_2(bpf_set_hash, struct sk_buff *, skb, u32, hash)
3184{
3185 /* Set user specified hash as L4(+), so that it gets returned
3186 * on skb_get_hash() call unless BPF prog later on triggers a
3187 * skb_clear_hash().
3188 */
3189 __skb_set_sw_hash(skb, hash, is_l4: true);
3190 return 0;
3191}
3192
3193static const struct bpf_func_proto bpf_set_hash_proto = {
3194 .func = bpf_set_hash,
3195 .gpl_only = false,
3196 .ret_type = RET_INTEGER,
3197 .arg1_type = ARG_PTR_TO_CTX,
3198 .arg2_type = ARG_ANYTHING,
3199};
3200
3201BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto,
3202 u16, vlan_tci)
3203{
3204 int ret;
3205
3206 if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
3207 vlan_proto != htons(ETH_P_8021AD)))
3208 vlan_proto = htons(ETH_P_8021Q);
3209
3210 bpf_push_mac_rcsum(skb);
3211 ret = skb_vlan_push(skb, vlan_proto, vlan_tci);
3212 bpf_pull_mac_rcsum(skb);
3213 skb_reset_mac_len(skb);
3214
3215 bpf_compute_data_pointers(skb);
3216 return ret;
3217}
3218
3219static const struct bpf_func_proto bpf_skb_vlan_push_proto = {
3220 .func = bpf_skb_vlan_push,
3221 .gpl_only = false,
3222 .ret_type = RET_INTEGER,
3223 .arg1_type = ARG_PTR_TO_CTX,
3224 .arg2_type = ARG_ANYTHING,
3225 .arg3_type = ARG_ANYTHING,
3226};
3227
3228BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
3229{
3230 int ret;
3231
3232 bpf_push_mac_rcsum(skb);
3233 ret = skb_vlan_pop(skb);
3234 bpf_pull_mac_rcsum(skb);
3235
3236 bpf_compute_data_pointers(skb);
3237 return ret;
3238}
3239
3240static const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
3241 .func = bpf_skb_vlan_pop,
3242 .gpl_only = false,
3243 .ret_type = RET_INTEGER,
3244 .arg1_type = ARG_PTR_TO_CTX,
3245};
3246
3247static void bpf_skb_change_protocol(struct sk_buff *skb, u16 proto)
3248{
3249 skb->protocol = htons(proto);
3250 if (skb_valid_dst(skb))
3251 skb_dst_drop(skb);
3252}
3253
3254static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
3255{
3256 /* Caller already did skb_cow() with len as headroom,
3257 * so no need to do it here.
3258 */
3259 skb_push(skb, len);
3260 memmove(dest: skb->data, src: skb->data + len, count: off);
3261 memset(s: skb->data + off, c: 0, n: len);
3262
3263 /* No skb_postpush_rcsum(skb, skb->data + off, len)
3264 * needed here as it does not change the skb->csum
3265 * result for checksum complete when summing over
3266 * zeroed blocks.
3267 */
3268 return 0;
3269}
3270
3271static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
3272{
3273 void *old_data;
3274
3275 /* skb_ensure_writable() is not needed here, as we're
3276 * already working on an uncloned skb.
3277 */
3278 if (unlikely(!pskb_may_pull(skb, off + len)))
3279 return -ENOMEM;
3280
3281 old_data = skb->data;
3282 __skb_pull(skb, len);
3283 skb_postpull_rcsum(skb, start: old_data + off, len);
3284 memmove(dest: skb->data, src: old_data, count: off);
3285
3286 return 0;
3287}
3288
3289static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
3290{
3291 bool trans_same = skb->transport_header == skb->network_header;
3292 int ret;
3293
3294 /* There's no need for __skb_push()/__skb_pull() pair to
3295 * get to the start of the mac header as we're guaranteed
3296 * to always start from here under eBPF.
3297 */
3298 ret = bpf_skb_generic_push(skb, off, len);
3299 if (likely(!ret)) {
3300 skb->mac_header -= len;
3301 skb->network_header -= len;
3302 if (trans_same)
3303 skb->transport_header = skb->network_header;
3304 }
3305
3306 return ret;
3307}
3308
3309static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
3310{
3311 bool trans_same = skb->transport_header == skb->network_header;
3312 int ret;
3313
3314 /* Same here, __skb_push()/__skb_pull() pair not needed. */
3315 ret = bpf_skb_generic_pop(skb, off, len);
3316 if (likely(!ret)) {
3317 skb->mac_header += len;
3318 skb->network_header += len;
3319 if (trans_same)
3320 skb->transport_header = skb->network_header;
3321 }
3322
3323 return ret;
3324}
3325
3326static int bpf_skb_proto_4_to_6(struct sk_buff *skb)
3327{
3328 const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3329 u32 off = skb_mac_header_len(skb);
3330 int ret;
3331
3332 ret = skb_cow(skb, headroom: len_diff);
3333 if (unlikely(ret < 0))
3334 return ret;
3335
3336 ret = bpf_skb_net_hdr_push(skb, off, len: len_diff);
3337 if (unlikely(ret < 0))
3338 return ret;
3339
3340 if (skb_is_gso(skb)) {
3341 struct skb_shared_info *shinfo = skb_shinfo(skb);
3342
3343 /* SKB_GSO_TCPV4 needs to be changed into SKB_GSO_TCPV6. */
3344 if (shinfo->gso_type & SKB_GSO_TCPV4) {
3345 shinfo->gso_type &= ~SKB_GSO_TCPV4;
3346 shinfo->gso_type |= SKB_GSO_TCPV6;
3347 }
3348 }
3349
3350 bpf_skb_change_protocol(skb, ETH_P_IPV6);
3351 skb_clear_hash(skb);
3352
3353 return 0;
3354}
3355
3356static int bpf_skb_proto_6_to_4(struct sk_buff *skb)
3357{
3358 const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3359 u32 off = skb_mac_header_len(skb);
3360 int ret;
3361
3362 ret = skb_unclone(skb, GFP_ATOMIC);
3363 if (unlikely(ret < 0))
3364 return ret;
3365
3366 ret = bpf_skb_net_hdr_pop(skb, off, len: len_diff);
3367 if (unlikely(ret < 0))
3368 return ret;
3369
3370 if (skb_is_gso(skb)) {
3371 struct skb_shared_info *shinfo = skb_shinfo(skb);
3372
3373 /* SKB_GSO_TCPV6 needs to be changed into SKB_GSO_TCPV4. */
3374 if (shinfo->gso_type & SKB_GSO_TCPV6) {
3375 shinfo->gso_type &= ~SKB_GSO_TCPV6;
3376 shinfo->gso_type |= SKB_GSO_TCPV4;
3377 }
3378 }
3379
3380 bpf_skb_change_protocol(skb, ETH_P_IP);
3381 skb_clear_hash(skb);
3382
3383 return 0;
3384}
3385
3386static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto)
3387{
3388 __be16 from_proto = skb->protocol;
3389
3390 if (from_proto == htons(ETH_P_IP) &&
3391 to_proto == htons(ETH_P_IPV6))
3392 return bpf_skb_proto_4_to_6(skb);
3393
3394 if (from_proto == htons(ETH_P_IPV6) &&
3395 to_proto == htons(ETH_P_IP))
3396 return bpf_skb_proto_6_to_4(skb);
3397
3398 return -ENOTSUPP;
3399}
3400
3401BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto,
3402 u64, flags)
3403{
3404 int ret;
3405
3406 if (unlikely(flags))
3407 return -EINVAL;
3408
3409 /* General idea is that this helper does the basic groundwork
3410 * needed for changing the protocol, and eBPF program fills the
3411 * rest through bpf_skb_store_bytes(), bpf_lX_csum_replace()
3412 * and other helpers, rather than passing a raw buffer here.
3413 *
3414 * The rationale is to keep this minimal and without a need to
3415 * deal with raw packet data. F.e. even if we would pass buffers
3416 * here, the program still needs to call the bpf_lX_csum_replace()
3417 * helpers anyway. Plus, this way we keep also separation of
3418 * concerns, since f.e. bpf_skb_store_bytes() should only take
3419 * care of stores.
3420 *
3421 * Currently, additional options and extension header space are
3422 * not supported, but flags register is reserved so we can adapt
3423 * that. For offloads, we mark packet as dodgy, so that headers
3424 * need to be verified first.
3425 */
3426 ret = bpf_skb_proto_xlat(skb, to_proto: proto);
3427 bpf_compute_data_pointers(skb);
3428 return ret;
3429}
3430
3431static const struct bpf_func_proto bpf_skb_change_proto_proto = {
3432 .func = bpf_skb_change_proto,
3433 .gpl_only = false,
3434 .ret_type = RET_INTEGER,
3435 .arg1_type = ARG_PTR_TO_CTX,
3436 .arg2_type = ARG_ANYTHING,
3437 .arg3_type = ARG_ANYTHING,
3438};
3439
3440BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type)
3441{
3442 /* We only allow a restricted subset to be changed for now. */
3443 if (unlikely(!skb_pkt_type_ok(skb->pkt_type) ||
3444 !skb_pkt_type_ok(pkt_type)))
3445 return -EINVAL;
3446
3447 skb->pkt_type = pkt_type;
3448 return 0;
3449}
3450
3451static const struct bpf_func_proto bpf_skb_change_type_proto = {
3452 .func = bpf_skb_change_type,
3453 .gpl_only = false,
3454 .ret_type = RET_INTEGER,
3455 .arg1_type = ARG_PTR_TO_CTX,
3456 .arg2_type = ARG_ANYTHING,
3457};
3458
3459static u32 bpf_skb_net_base_len(const struct sk_buff *skb)
3460{
3461 switch (skb->protocol) {
3462 case htons(ETH_P_IP):
3463 return sizeof(struct iphdr);
3464 case htons(ETH_P_IPV6):
3465 return sizeof(struct ipv6hdr);
3466 default:
3467 return ~0U;
3468 }
3469}
3470
3471#define BPF_F_ADJ_ROOM_ENCAP_L3_MASK (BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | \
3472 BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3473
3474#define BPF_F_ADJ_ROOM_DECAP_L3_MASK (BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | \
3475 BPF_F_ADJ_ROOM_DECAP_L3_IPV6)
3476
3477#define BPF_F_ADJ_ROOM_MASK (BPF_F_ADJ_ROOM_FIXED_GSO | \
3478 BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \
3479 BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \
3480 BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \
3481 BPF_F_ADJ_ROOM_ENCAP_L2_ETH | \
3482 BPF_F_ADJ_ROOM_ENCAP_L2( \
3483 BPF_ADJ_ROOM_ENCAP_L2_MASK) | \
3484 BPF_F_ADJ_ROOM_DECAP_L3_MASK)
3485
3486static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
3487 u64 flags)
3488{
3489 u8 inner_mac_len = flags >> BPF_ADJ_ROOM_ENCAP_L2_SHIFT;
3490 bool encap = flags & BPF_F_ADJ_ROOM_ENCAP_L3_MASK;
3491 u16 mac_len = 0, inner_net = 0, inner_trans = 0;
3492 unsigned int gso_type = SKB_GSO_DODGY;
3493 int ret;
3494
3495 if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
3496 /* udp gso_size delineates datagrams, only allow if fixed */
3497 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
3498 !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3499 return -ENOTSUPP;
3500 }
3501
3502 ret = skb_cow_head(skb, headroom: len_diff);
3503 if (unlikely(ret < 0))
3504 return ret;
3505
3506 if (encap) {
3507 if (skb->protocol != htons(ETH_P_IP) &&
3508 skb->protocol != htons(ETH_P_IPV6))
3509 return -ENOTSUPP;
3510
3511 if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 &&
3512 flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3513 return -EINVAL;
3514
3515 if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE &&
3516 flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
3517 return -EINVAL;
3518
3519 if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH &&
3520 inner_mac_len < ETH_HLEN)
3521 return -EINVAL;
3522
3523 if (skb->encapsulation)
3524 return -EALREADY;
3525
3526 mac_len = skb->network_header - skb->mac_header;
3527 inner_net = skb->network_header;
3528 if (inner_mac_len > len_diff)
3529 return -EINVAL;
3530 inner_trans = skb->transport_header;
3531 }
3532
3533 ret = bpf_skb_net_hdr_push(skb, off, len: len_diff);
3534 if (unlikely(ret < 0))
3535 return ret;
3536
3537 if (encap) {
3538 skb->inner_mac_header = inner_net - inner_mac_len;
3539 skb->inner_network_header = inner_net;
3540 skb->inner_transport_header = inner_trans;
3541
3542 if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH)
3543 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
3544 else
3545 skb_set_inner_protocol(skb, protocol: skb->protocol);
3546
3547 skb->encapsulation = 1;
3548 skb_set_network_header(skb, offset: mac_len);
3549
3550 if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
3551 gso_type |= SKB_GSO_UDP_TUNNEL;
3552 else if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE)
3553 gso_type |= SKB_GSO_GRE;
3554 else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3555 gso_type |= SKB_GSO_IPXIP6;
3556 else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
3557 gso_type |= SKB_GSO_IPXIP4;
3558
3559 if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE ||
3560 flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP) {
3561 int nh_len = flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 ?
3562 sizeof(struct ipv6hdr) :
3563 sizeof(struct iphdr);
3564
3565 skb_set_transport_header(skb, offset: mac_len + nh_len);
3566 }
3567
3568 /* Match skb->protocol to new outer l3 protocol */
3569 if (skb->protocol == htons(ETH_P_IP) &&
3570 flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3571 bpf_skb_change_protocol(skb, ETH_P_IPV6);
3572 else if (skb->protocol == htons(ETH_P_IPV6) &&
3573 flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
3574 bpf_skb_change_protocol(skb, ETH_P_IP);
3575 }
3576
3577 if (skb_is_gso(skb)) {
3578 struct skb_shared_info *shinfo = skb_shinfo(skb);
3579
3580 /* Header must be checked, and gso_segs recomputed. */
3581 shinfo->gso_type |= gso_type;
3582 shinfo->gso_segs = 0;
3583
3584 /* Due to header growth, MSS needs to be downgraded.
3585 * There is a BUG_ON() when segmenting the frag_list with
3586 * head_frag true, so linearize the skb after downgrading
3587 * the MSS.
3588 */
3589 if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO)) {
3590 skb_decrease_gso_size(shinfo, decrement: len_diff);
3591 if (shinfo->frag_list)
3592 return skb_linearize(skb);
3593 }
3594 }
3595
3596 return 0;
3597}
3598
3599static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
3600 u64 flags)
3601{
3602 int ret;
3603
3604 if (unlikely(flags & ~(BPF_F_ADJ_ROOM_FIXED_GSO |
3605 BPF_F_ADJ_ROOM_DECAP_L3_MASK |
3606 BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
3607 return -EINVAL;
3608
3609 if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
3610 /* udp gso_size delineates datagrams, only allow if fixed */
3611 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
3612 !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3613 return -ENOTSUPP;
3614 }
3615
3616 ret = skb_unclone(skb, GFP_ATOMIC);
3617 if (unlikely(ret < 0))
3618 return ret;
3619
3620 ret = bpf_skb_net_hdr_pop(skb, off, len: len_diff);
3621 if (unlikely(ret < 0))
3622 return ret;
3623
3624 /* Match skb->protocol to new outer l3 protocol */
3625 if (skb->protocol == htons(ETH_P_IP) &&
3626 flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV6)
3627 bpf_skb_change_protocol(skb, ETH_P_IPV6);
3628 else if (skb->protocol == htons(ETH_P_IPV6) &&
3629 flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV4)
3630 bpf_skb_change_protocol(skb, ETH_P_IP);
3631
3632 if (skb_is_gso(skb)) {
3633 struct skb_shared_info *shinfo = skb_shinfo(skb);
3634
3635 /* Due to header shrink, MSS can be upgraded. */
3636 if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3637 skb_increase_gso_size(shinfo, increment: len_diff);
3638
3639 /* Header must be checked, and gso_segs recomputed. */
3640 shinfo->gso_type |= SKB_GSO_DODGY;
3641 shinfo->gso_segs = 0;
3642 }
3643
3644 return 0;
3645}
3646
3647#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC
3648
3649BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3650 u32, mode, u64, flags)
3651{
3652 u32 len_diff_abs = abs(len_diff);
3653 bool shrink = len_diff < 0;
3654 int ret = 0;
3655
3656 if (unlikely(flags || mode))
3657 return -EINVAL;
3658 if (unlikely(len_diff_abs > 0xfffU))
3659 return -EFAULT;
3660
3661 if (!shrink) {
3662 ret = skb_cow(skb, headroom: len_diff);
3663 if (unlikely(ret < 0))
3664 return ret;
3665 __skb_push(skb, len: len_diff_abs);
3666 memset(s: skb->data, c: 0, n: len_diff_abs);
3667 } else {
3668 if (unlikely(!pskb_may_pull(skb, len_diff_abs)))
3669 return -ENOMEM;
3670 __skb_pull(skb, len: len_diff_abs);
3671 }
3672 if (tls_sw_has_ctx_rx(sk: skb->sk)) {
3673 struct strp_msg *rxm = strp_msg(skb);
3674
3675 rxm->full_len += len_diff;
3676 }
3677 return ret;
3678}
3679
3680static const struct bpf_func_proto sk_skb_adjust_room_proto = {
3681 .func = sk_skb_adjust_room,
3682 .gpl_only = false,
3683 .ret_type = RET_INTEGER,
3684 .arg1_type = ARG_PTR_TO_CTX,
3685 .arg2_type = ARG_ANYTHING,
3686 .arg3_type = ARG_ANYTHING,
3687 .arg4_type = ARG_ANYTHING,
3688};
3689
3690BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3691 u32, mode, u64, flags)
3692{
3693 u32 len_cur, len_diff_abs = abs(len_diff);
3694 u32 len_min = bpf_skb_net_base_len(skb);
3695 u32 len_max = BPF_SKB_MAX_LEN;
3696 __be16 proto = skb->protocol;
3697 bool shrink = len_diff < 0;
3698 u32 off;
3699 int ret;
3700
3701 if (unlikely(flags & ~(BPF_F_ADJ_ROOM_MASK |
3702 BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
3703 return -EINVAL;
3704 if (unlikely(len_diff_abs > 0xfffU))
3705 return -EFAULT;
3706 if (unlikely(proto != htons(ETH_P_IP) &&
3707 proto != htons(ETH_P_IPV6)))
3708 return -ENOTSUPP;
3709
3710 off = skb_mac_header_len(skb);
3711 switch (mode) {
3712 case BPF_ADJ_ROOM_NET:
3713 off += bpf_skb_net_base_len(skb);
3714 break;
3715 case BPF_ADJ_ROOM_MAC:
3716 break;
3717 default:
3718 return -ENOTSUPP;
3719 }
3720
3721 if (flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) {
3722 if (!shrink)
3723 return -EINVAL;
3724
3725 switch (flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) {
3726 case BPF_F_ADJ_ROOM_DECAP_L3_IPV4:
3727 len_min = sizeof(struct iphdr);
3728 break;
3729 case BPF_F_ADJ_ROOM_DECAP_L3_IPV6:
3730 len_min = sizeof(struct ipv6hdr);
3731 break;
3732 default:
3733 return -EINVAL;
3734 }
3735 }
3736
3737 len_cur = skb->len - skb_network_offset(skb);
3738 if ((shrink && (len_diff_abs >= len_cur ||
3739 len_cur - len_diff_abs < len_min)) ||
3740 (!shrink && (skb->len + len_diff_abs > len_max &&
3741 !skb_is_gso(skb))))
3742 return -ENOTSUPP;
3743
3744 ret = shrink ? bpf_skb_net_shrink(skb, off, len_diff: len_diff_abs, flags) :
3745 bpf_skb_net_grow(skb, off, len_diff: len_diff_abs, flags);
3746 if (!ret && !(flags & BPF_F_ADJ_ROOM_NO_CSUM_RESET))
3747 __skb_reset_checksum_unnecessary(skb);
3748
3749 bpf_compute_data_pointers(skb);
3750 return ret;
3751}
3752
3753static const struct bpf_func_proto bpf_skb_adjust_room_proto = {
3754 .func = bpf_skb_adjust_room,
3755 .gpl_only = false,
3756 .ret_type = RET_INTEGER,
3757 .arg1_type = ARG_PTR_TO_CTX,
3758 .arg2_type = ARG_ANYTHING,
3759 .arg3_type = ARG_ANYTHING,
3760 .arg4_type = ARG_ANYTHING,
3761};
3762
3763static u32 __bpf_skb_min_len(const struct sk_buff *skb)
3764{
3765 int offset = skb_network_offset(skb);
3766 u32 min_len = 0;
3767
3768 if (offset > 0)
3769 min_len = offset;
3770 if (skb_transport_header_was_set(skb)) {
3771 offset = skb_transport_offset(skb);
3772 if (offset > 0)
3773 min_len = offset;
3774 }
3775 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3776 offset = skb_checksum_start_offset(skb) +
3777 skb->csum_offset + sizeof(__sum16);
3778 if (offset > 0)
3779 min_len = offset;
3780 }
3781 return min_len;
3782}
3783
3784static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len)
3785{
3786 unsigned int old_len = skb->len;
3787 int ret;
3788
3789 ret = __skb_grow_rcsum(skb, len: new_len);
3790 if (!ret)
3791 memset(s: skb->data + old_len, c: 0, n: new_len - old_len);
3792 return ret;
3793}
3794
3795static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len)
3796{
3797 return __skb_trim_rcsum(skb, len: new_len);
3798}
3799
3800static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,
3801 u64 flags)
3802{
3803 u32 max_len = BPF_SKB_MAX_LEN;
3804 u32 min_len = __bpf_skb_min_len(skb);
3805 int ret;
3806
3807 if (unlikely(flags || new_len > max_len || new_len < min_len))
3808 return -EINVAL;
3809 if (skb->encapsulation)
3810 return -ENOTSUPP;
3811
3812 /* The basic idea of this helper is that it's performing the
3813 * needed work to either grow or trim an skb, and eBPF program
3814 * rewrites the rest via helpers like bpf_skb_store_bytes(),
3815 * bpf_lX_csum_replace() and others rather than passing a raw
3816 * buffer here. This one is a slow path helper and intended
3817 * for replies with control messages.
3818 *
3819 * Like in bpf_skb_change_proto(), we want to keep this rather
3820 * minimal and without protocol specifics so that we are able
3821 * to separate concerns as in bpf_skb_store_bytes() should only
3822 * be the one responsible for writing buffers.
3823 *
3824 * It's really expected to be a slow path operation here for
3825 * control message replies, so we're implicitly linearizing,
3826 * uncloning and drop offloads from the skb by this.
3827 */
3828 ret = __bpf_try_make_writable(skb, write_len: skb->len);
3829 if (!ret) {
3830 if (new_len > skb->len)
3831 ret = bpf_skb_grow_rcsum(skb, new_len);
3832 else if (new_len < skb->len)
3833 ret = bpf_skb_trim_rcsum(skb, new_len);
3834 if (!ret && skb_is_gso(skb))
3835 skb_gso_reset(skb);
3836 }
3837 return ret;
3838}
3839
3840BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len,
3841 u64, flags)
3842{
3843 int ret = __bpf_skb_change_tail(skb, new_len, flags);
3844
3845 bpf_compute_data_pointers(skb);
3846 return ret;
3847}
3848
3849static const struct bpf_func_proto bpf_skb_change_tail_proto = {
3850 .func = bpf_skb_change_tail,
3851 .gpl_only = false,
3852 .ret_type = RET_INTEGER,
3853 .arg1_type = ARG_PTR_TO_CTX,
3854 .arg2_type = ARG_ANYTHING,
3855 .arg3_type = ARG_ANYTHING,
3856};
3857
3858BPF_CALL_3(sk_skb_change_tail, struct sk_buff *, skb, u32, new_len,
3859 u64, flags)
3860{
3861 return __bpf_skb_change_tail(skb, new_len, flags);
3862}
3863
3864static const struct bpf_func_proto sk_skb_change_tail_proto = {
3865 .func = sk_skb_change_tail,
3866 .gpl_only = false,
3867 .ret_type = RET_INTEGER,
3868 .arg1_type = ARG_PTR_TO_CTX,
3869 .arg2_type = ARG_ANYTHING,
3870 .arg3_type = ARG_ANYTHING,
3871};
3872
3873static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,
3874 u64 flags)
3875{
3876 u32 max_len = BPF_SKB_MAX_LEN;
3877 u32 new_len = skb->len + head_room;
3878 int ret;
3879
3880 if (unlikely(flags || (!skb_is_gso(skb) && new_len > max_len) ||
3881 new_len < skb->len))
3882 return -EINVAL;
3883
3884 ret = skb_cow(skb, headroom: head_room);
3885 if (likely(!ret)) {
3886 /* Idea for this helper is that we currently only
3887 * allow to expand on mac header. This means that
3888 * skb->protocol network header, etc, stay as is.
3889 * Compared to bpf_skb_change_tail(), we're more
3890 * flexible due to not needing to linearize or
3891 * reset GSO. Intention for this helper is to be
3892 * used by an L3 skb that needs to push mac header
3893 * for redirection into L2 device.
3894 */
3895 __skb_push(skb, len: head_room);
3896 memset(s: skb->data, c: 0, n: head_room);
3897 skb_reset_mac_header(skb);
3898 skb_reset_mac_len(skb);
3899 }
3900
3901 return ret;
3902}
3903
3904BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room,
3905 u64, flags)
3906{
3907 int ret = __bpf_skb_change_head(skb, head_room, flags);
3908
3909 bpf_compute_data_pointers(skb);
3910 return ret;
3911}
3912
3913static const struct bpf_func_proto bpf_skb_change_head_proto = {
3914 .func = bpf_skb_change_head,
3915 .gpl_only = false,
3916 .ret_type = RET_INTEGER,
3917 .arg1_type = ARG_PTR_TO_CTX,
3918 .arg2_type = ARG_ANYTHING,
3919 .arg3_type = ARG_ANYTHING,
3920};
3921
3922BPF_CALL_3(sk_skb_change_head, struct sk_buff *, skb, u32, head_room,
3923 u64, flags)
3924{
3925 return __bpf_skb_change_head(skb, head_room, flags);
3926}
3927
3928static const struct bpf_func_proto sk_skb_change_head_proto = {
3929 .func = sk_skb_change_head,
3930 .gpl_only = false,
3931 .ret_type = RET_INTEGER,
3932 .arg1_type = ARG_PTR_TO_CTX,
3933 .arg2_type = ARG_ANYTHING,
3934 .arg3_type = ARG_ANYTHING,
3935};
3936
3937BPF_CALL_1(bpf_xdp_get_buff_len, struct xdp_buff*, xdp)
3938{
3939 return xdp_get_buff_len(xdp);
3940}
3941
3942static const struct bpf_func_proto bpf_xdp_get_buff_len_proto = {
3943 .func = bpf_xdp_get_buff_len,
3944 .gpl_only = false,
3945 .ret_type = RET_INTEGER,
3946 .arg1_type = ARG_PTR_TO_CTX,
3947};
3948
3949BTF_ID_LIST_SINGLE(bpf_xdp_get_buff_len_bpf_ids, struct, xdp_buff)
3950
3951const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto = {
3952 .func = bpf_xdp_get_buff_len,
3953 .gpl_only = false,
3954 .arg1_type = ARG_PTR_TO_BTF_ID,
3955 .arg1_btf_id = &bpf_xdp_get_buff_len_bpf_ids[0],
3956};
3957
3958static unsigned long xdp_get_metalen(const struct xdp_buff *xdp)
3959{
3960 return xdp_data_meta_unsupported(xdp) ? 0 :
3961 xdp->data - xdp->data_meta;
3962}
3963
3964BPF_CALL_2(bpf_xdp_adjust_head, struct xdp_buff *, xdp, int, offset)
3965{
3966 void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
3967 unsigned long metalen = xdp_get_metalen(xdp);
3968 void *data_start = xdp_frame_end + metalen;
3969 void *data = xdp->data + offset;
3970
3971 if (unlikely(data < data_start ||
3972 data > xdp->data_end - ETH_HLEN))
3973 return -EINVAL;
3974
3975 if (metalen)
3976 memmove(dest: xdp->data_meta + offset,
3977 src: xdp->data_meta, count: metalen);
3978 xdp->data_meta += offset;
3979 xdp->data = data;
3980
3981 return 0;
3982}
3983
3984static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
3985 .func = bpf_xdp_adjust_head,
3986 .gpl_only = false,
3987 .ret_type = RET_INTEGER,
3988 .arg1_type = ARG_PTR_TO_CTX,
3989 .arg2_type = ARG_ANYTHING,
3990};
3991
3992void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
3993 void *buf, unsigned long len, bool flush)
3994{
3995 unsigned long ptr_len, ptr_off = 0;
3996 skb_frag_t *next_frag, *end_frag;
3997 struct skb_shared_info *sinfo;
3998 void *src, *dst;
3999 u8 *ptr_buf;
4000
4001 if (likely(xdp->data_end - xdp->data >= off + len)) {
4002 src = flush ? buf : xdp->data + off;
4003 dst = flush ? xdp->data + off : buf;
4004 memcpy(to: dst, from: src, len);
4005 return;
4006 }
4007
4008 sinfo = xdp_get_shared_info_from_buff(xdp);
4009 end_frag = &sinfo->frags[sinfo->nr_frags];
4010 next_frag = &sinfo->frags[0];
4011
4012 ptr_len = xdp->data_end - xdp->data;
4013 ptr_buf = xdp->data;
4014
4015 while (true) {
4016 if (off < ptr_off + ptr_len) {
4017 unsigned long copy_off = off - ptr_off;
4018 unsigned long copy_len = min(len, ptr_len - copy_off);
4019
4020 src = flush ? buf : ptr_buf + copy_off;
4021 dst = flush ? ptr_buf + copy_off : buf;
4022 memcpy(to: dst, from: src, len: copy_len);
4023
4024 off += copy_len;
4025 len -= copy_len;
4026 buf += copy_len;
4027 }
4028
4029 if (!len || next_frag == end_frag)
4030 break;
4031
4032 ptr_off += ptr_len;
4033 ptr_buf = skb_frag_address(frag: next_frag);
4034 ptr_len = skb_frag_size(frag: next_frag);
4035 next_frag++;
4036 }
4037}
4038
4039void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
4040{
4041 u32 size = xdp->data_end - xdp->data;
4042 struct skb_shared_info *sinfo;
4043 void *addr = xdp->data;
4044 int i;
4045
4046 if (unlikely(offset > 0xffff || len > 0xffff))
4047 return ERR_PTR(error: -EFAULT);
4048
4049 if (unlikely(offset + len > xdp_get_buff_len(xdp)))
4050 return ERR_PTR(error: -EINVAL);
4051
4052 if (likely(offset < size)) /* linear area */
4053 goto out;
4054
4055 sinfo = xdp_get_shared_info_from_buff(xdp);
4056 offset -= size;
4057 for (i = 0; i < sinfo->nr_frags; i++) { /* paged area */
4058 u32 frag_size = skb_frag_size(frag: &sinfo->frags[i]);
4059
4060 if (offset < frag_size) {
4061 addr = skb_frag_address(frag: &sinfo->frags[i]);
4062 size = frag_size;
4063 break;
4064 }
4065 offset -= frag_size;
4066 }
4067out:
4068 return offset + len <= size ? addr + offset : NULL;
4069}
4070
4071BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
4072 void *, buf, u32, len)
4073{
4074 void *ptr;
4075
4076 ptr = bpf_xdp_pointer(xdp, offset, len);
4077 if (IS_ERR(ptr))
4078 return PTR_ERR(ptr);
4079
4080 if (!ptr)
4081 bpf_xdp_copy_buf(xdp, off: offset, buf, len, flush: false);
4082 else
4083 memcpy(to: buf, from: ptr, len);
4084
4085 return 0;
4086}
4087
4088static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {
4089 .func = bpf_xdp_load_bytes,
4090 .gpl_only = false,
4091 .ret_type = RET_INTEGER,
4092 .arg1_type = ARG_PTR_TO_CTX,
4093 .arg2_type = ARG_ANYTHING,
4094 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
4095 .arg4_type = ARG_CONST_SIZE,
4096};
4097
4098int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
4099{
4100 return ____bpf_xdp_load_bytes(xdp, offset, buf, len);
4101}
4102
4103BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
4104 void *, buf, u32, len)
4105{
4106 void *ptr;
4107
4108 ptr = bpf_xdp_pointer(xdp, offset, len);
4109 if (IS_ERR(ptr))
4110 return PTR_ERR(ptr);
4111
4112 if (!ptr)
4113 bpf_xdp_copy_buf(xdp, off: offset, buf, len, flush: true);
4114 else
4115 memcpy(to: ptr, from: buf, len);
4116
4117 return 0;
4118}
4119
4120static const struct bpf_func_proto bpf_xdp_store_bytes_proto = {
4121 .func = bpf_xdp_store_bytes,
4122 .gpl_only = false,
4123 .ret_type = RET_INTEGER,
4124 .arg1_type = ARG_PTR_TO_CTX,
4125 .arg2_type = ARG_ANYTHING,
4126 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
4127 .arg4_type = ARG_CONST_SIZE,
4128};
4129
4130int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
4131{
4132 return ____bpf_xdp_store_bytes(xdp, offset, buf, len);
4133}
4134
4135static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
4136{
4137 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
4138 skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags - 1];
4139 struct xdp_rxq_info *rxq = xdp->rxq;
4140 unsigned int tailroom;
4141
4142 if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz)
4143 return -EOPNOTSUPP;
4144
4145 tailroom = rxq->frag_size - skb_frag_size(frag) - skb_frag_off(frag);
4146 if (unlikely(offset > tailroom))
4147 return -EINVAL;
4148
4149 memset(s: skb_frag_address(frag) + skb_frag_size(frag), c: 0, n: offset);
4150 skb_frag_size_add(frag, delta: offset);
4151 sinfo->xdp_frags_size += offset;
4152 if (rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
4153 xsk_buff_get_tail(first: xdp)->data_end += offset;
4154
4155 return 0;
4156}
4157
4158static struct xdp_buff *bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
4159 bool tail, bool release)
4160{
4161 struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(first: xdp) :
4162 xsk_buff_get_head(first: xdp);
4163
4164 if (release) {
4165 xsk_buff_del_frag(xdp: zc_frag);
4166 } else {
4167 if (tail)
4168 zc_frag->data_end -= shrink;
4169 else
4170 zc_frag->data += shrink;
4171 }
4172
4173 return zc_frag;
4174}
4175
4176static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
4177 int shrink, bool tail)
4178{
4179 enum xdp_mem_type mem_type = xdp->rxq->mem.type;
4180 bool release = skb_frag_size(frag) == shrink;
4181 netmem_ref netmem = skb_frag_netmem(frag);
4182 struct xdp_buff *zc_frag = NULL;
4183
4184 if (mem_type == MEM_TYPE_XSK_BUFF_POOL) {
4185 netmem = 0;
4186 zc_frag = bpf_xdp_shrink_data_zc(xdp, shrink, tail, release);
4187 }
4188
4189 if (release) {
4190 __xdp_return(netmem, mem_type, napi_direct: false, xdp: zc_frag);
4191 } else {
4192 if (!tail)
4193 skb_frag_off_add(frag, delta: shrink);
4194 skb_frag_size_sub(frag, delta: shrink);
4195 }
4196
4197 return release;
4198}
4199
4200static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset)
4201{
4202 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
4203 int i, n_frags_free = 0, len_free = 0;
4204
4205 if (unlikely(offset > (int)xdp_get_buff_len(xdp) - ETH_HLEN))
4206 return -EINVAL;
4207
4208 for (i = sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) {
4209 skb_frag_t *frag = &sinfo->frags[i];
4210 int shrink = min_t(int, offset, skb_frag_size(frag));
4211
4212 len_free += shrink;
4213 offset -= shrink;
4214 if (bpf_xdp_shrink_data(xdp, frag, shrink, tail: true))
4215 n_frags_free++;
4216 }
4217 sinfo->nr_frags -= n_frags_free;
4218 sinfo->xdp_frags_size -= len_free;
4219
4220 if (unlikely(!sinfo->nr_frags)) {
4221 xdp_buff_clear_frags_flag(xdp);
4222 xdp_buff_clear_frag_pfmemalloc(xdp);
4223 xdp->data_end -= offset;
4224 }
4225
4226 return 0;
4227}
4228
4229BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset)
4230{
4231 void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */
4232 void *data_end = xdp->data_end + offset;
4233
4234 if (unlikely(xdp_buff_has_frags(xdp))) { /* non-linear xdp buff */
4235 if (offset < 0)
4236 return bpf_xdp_frags_shrink_tail(xdp, offset: -offset);
4237
4238 return bpf_xdp_frags_increase_tail(xdp, offset);
4239 }
4240
4241 /* Notice that xdp_data_hard_end have reserved some tailroom */
4242 if (unlikely(data_end > data_hard_end))
4243 return -EINVAL;
4244
4245 if (unlikely(data_end < xdp->data + ETH_HLEN))
4246 return -EINVAL;
4247
4248 /* Clear memory area on grow, can contain uninit kernel memory */
4249 if (offset > 0)
4250 memset(s: xdp->data_end, c: 0, n: offset);
4251
4252 xdp->data_end = data_end;
4253
4254 return 0;
4255}
4256
4257static const struct bpf_func_proto bpf_xdp_adjust_tail_proto = {
4258 .func = bpf_xdp_adjust_tail,
4259 .gpl_only = false,
4260 .ret_type = RET_INTEGER,
4261 .arg1_type = ARG_PTR_TO_CTX,
4262 .arg2_type = ARG_ANYTHING,
4263};
4264
4265BPF_CALL_2(bpf_xdp_adjust_meta, struct xdp_buff *, xdp, int, offset)
4266{
4267 void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
4268 void *meta = xdp->data_meta + offset;
4269 unsigned long metalen = xdp->data - meta;
4270
4271 if (xdp_data_meta_unsupported(xdp))
4272 return -ENOTSUPP;
4273 if (unlikely(meta < xdp_frame_end ||
4274 meta > xdp->data))
4275 return -EINVAL;
4276 if (unlikely(xdp_metalen_invalid(metalen)))
4277 return -EACCES;
4278
4279 xdp->data_meta = meta;
4280
4281 return 0;
4282}
4283
4284static const struct bpf_func_proto bpf_xdp_adjust_meta_proto = {
4285 .func = bpf_xdp_adjust_meta,
4286 .gpl_only = false,
4287 .ret_type = RET_INTEGER,
4288 .arg1_type = ARG_PTR_TO_CTX,
4289 .arg2_type = ARG_ANYTHING,
4290};
4291
4292/**
4293 * DOC: xdp redirect
4294 *
4295 * XDP_REDIRECT works by a three-step process, implemented in the functions
4296 * below:
4297 *
4298 * 1. The bpf_redirect() and bpf_redirect_map() helpers will lookup the target
4299 * of the redirect and store it (along with some other metadata) in a per-CPU
4300 * struct bpf_redirect_info.
4301 *
4302 * 2. When the program returns the XDP_REDIRECT return code, the driver will
4303 * call xdp_do_redirect() which will use the information in struct
4304 * bpf_redirect_info to actually enqueue the frame into a map type-specific
4305 * bulk queue structure.
4306 *
4307 * 3. Before exiting its NAPI poll loop, the driver will call
4308 * xdp_do_flush(), which will flush all the different bulk queues,
4309 * thus completing the redirect. Note that xdp_do_flush() must be
4310 * called before napi_complete_done() in the driver, as the
4311 * XDP_REDIRECT logic relies on being inside a single NAPI instance
4312 * through to the xdp_do_flush() call for RCU protection of all
4313 * in-kernel data structures.
4314 */
4315/*
4316 * Pointers to the map entries will be kept around for this whole sequence of
4317 * steps, protected by RCU. However, there is no top-level rcu_read_lock() in
4318 * the core code; instead, the RCU protection relies on everything happening
4319 * inside a single NAPI poll sequence, which means it's between a pair of calls
4320 * to local_bh_disable()/local_bh_enable().
4321 *
4322 * The map entries are marked as __rcu and the map code makes sure to
4323 * dereference those pointers with rcu_dereference_check() in a way that works
4324 * for both sections that to hold an rcu_read_lock() and sections that are
4325 * called from NAPI without a separate rcu_read_lock(). The code below does not
4326 * use RCU annotations, but relies on those in the map code.
4327 */
4328void xdp_do_flush(void)
4329{
4330 struct list_head *lh_map, *lh_dev, *lh_xsk;
4331
4332 bpf_net_ctx_get_all_used_flush_lists(lh_map: &lh_map, lh_dev: &lh_dev, lh_xsk: &lh_xsk);
4333 if (lh_dev)
4334 __dev_flush(flush_list: lh_dev);
4335 if (lh_map)
4336 __cpu_map_flush(flush_list: lh_map);
4337 if (lh_xsk)
4338 __xsk_map_flush(flush_list: lh_xsk);
4339}
4340EXPORT_SYMBOL_GPL(xdp_do_flush);
4341
4342#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
4343void xdp_do_check_flushed(struct napi_struct *napi)
4344{
4345 struct list_head *lh_map, *lh_dev, *lh_xsk;
4346 bool missed = false;
4347
4348 bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
4349 if (lh_dev) {
4350 __dev_flush(lh_dev);
4351 missed = true;
4352 }
4353 if (lh_map) {
4354 __cpu_map_flush(lh_map);
4355 missed = true;
4356 }
4357 if (lh_xsk) {
4358 __xsk_map_flush(lh_xsk);
4359 missed = true;
4360 }
4361
4362 WARN_ONCE(missed, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
4363 napi->poll);
4364}
4365#endif
4366
4367DEFINE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key);
4368EXPORT_SYMBOL_GPL(bpf_master_redirect_enabled_key);
4369
4370u32 xdp_master_redirect(struct xdp_buff *xdp)
4371{
4372 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4373 struct net_device *master, *slave;
4374
4375 master = netdev_master_upper_dev_get_rcu(dev: xdp->rxq->dev);
4376 slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp);
4377 if (slave && slave != xdp->rxq->dev) {
4378 /* The target device is different from the receiving device, so
4379 * redirect it to the new device.
4380 * Using XDP_REDIRECT gets the correct behaviour from XDP enabled
4381 * drivers to unmap the packet from their rx ring.
4382 */
4383 ri->tgt_index = slave->ifindex;
4384 ri->map_id = INT_MAX;
4385 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4386 return XDP_REDIRECT;
4387 }
4388 return XDP_TX;
4389}
4390EXPORT_SYMBOL_GPL(xdp_master_redirect);
4391
4392static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri,
4393 const struct net_device *dev,
4394 struct xdp_buff *xdp,
4395 const struct bpf_prog *xdp_prog)
4396{
4397 enum bpf_map_type map_type = ri->map_type;
4398 void *fwd = ri->tgt_value;
4399 u32 map_id = ri->map_id;
4400 int err;
4401
4402 ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
4403 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4404
4405 err = __xsk_map_redirect(xs: fwd, xdp);
4406 if (unlikely(err))
4407 goto err;
4408
4409 _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4410 return 0;
4411err:
4412 _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4413 return err;
4414}
4415
4416static __always_inline int
4417__xdp_do_redirect_frame(struct bpf_redirect_info *ri, struct net_device *dev,
4418 struct xdp_frame *xdpf,
4419 const struct bpf_prog *xdp_prog)
4420{
4421 enum bpf_map_type map_type = ri->map_type;
4422 void *fwd = ri->tgt_value;
4423 u32 map_id = ri->map_id;
4424 u32 flags = ri->flags;
4425 struct bpf_map *map;
4426 int err;
4427
4428 ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
4429 ri->flags = 0;
4430 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4431
4432 if (unlikely(!xdpf)) {
4433 err = -EOVERFLOW;
4434 goto err;
4435 }
4436
4437 switch (map_type) {
4438 case BPF_MAP_TYPE_DEVMAP:
4439 fallthrough;
4440 case BPF_MAP_TYPE_DEVMAP_HASH:
4441 if (unlikely(flags & BPF_F_BROADCAST)) {
4442 map = READ_ONCE(ri->map);
4443
4444 /* The map pointer is cleared when the map is being torn
4445 * down by dev_map_free()
4446 */
4447 if (unlikely(!map)) {
4448 err = -ENOENT;
4449 break;
4450 }
4451
4452 WRITE_ONCE(ri->map, NULL);
4453 err = dev_map_enqueue_multi(xdpf, dev_rx: dev, map,
4454 exclude_ingress: flags & BPF_F_EXCLUDE_INGRESS);
4455 } else {
4456 err = dev_map_enqueue(dst: fwd, xdpf, dev_rx: dev);
4457 }
4458 break;
4459 case BPF_MAP_TYPE_CPUMAP:
4460 err = cpu_map_enqueue(rcpu: fwd, xdpf, dev_rx: dev);
4461 break;
4462 case BPF_MAP_TYPE_UNSPEC:
4463 if (map_id == INT_MAX) {
4464 fwd = dev_get_by_index_rcu(net: dev_net(dev), ifindex: ri->tgt_index);
4465 if (unlikely(!fwd)) {
4466 err = -EINVAL;
4467 break;
4468 }
4469 err = dev_xdp_enqueue(dev: fwd, xdpf, dev_rx: dev);
4470 break;
4471 }
4472 fallthrough;
4473 default:
4474 err = -EBADRQC;
4475 }
4476
4477 if (unlikely(err))
4478 goto err;
4479
4480 _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4481 return 0;
4482err:
4483 _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4484 return err;
4485}
4486
4487int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
4488 const struct bpf_prog *xdp_prog)
4489{
4490 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4491 enum bpf_map_type map_type = ri->map_type;
4492
4493 if (map_type == BPF_MAP_TYPE_XSKMAP)
4494 return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog);
4495
4496 return __xdp_do_redirect_frame(ri, dev, xdpf: xdp_convert_buff_to_frame(xdp),
4497 xdp_prog);
4498}
4499EXPORT_SYMBOL_GPL(xdp_do_redirect);
4500
4501int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp,
4502 struct xdp_frame *xdpf,
4503 const struct bpf_prog *xdp_prog)
4504{
4505 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4506 enum bpf_map_type map_type = ri->map_type;
4507
4508 if (map_type == BPF_MAP_TYPE_XSKMAP)
4509 return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog);
4510
4511 return __xdp_do_redirect_frame(ri, dev, xdpf, xdp_prog);
4512}
4513EXPORT_SYMBOL_GPL(xdp_do_redirect_frame);
4514
4515static int xdp_do_generic_redirect_map(struct net_device *dev,
4516 struct sk_buff *skb,
4517 struct xdp_buff *xdp,
4518 const struct bpf_prog *xdp_prog,
4519 void *fwd, enum bpf_map_type map_type,
4520 u32 map_id, u32 flags)
4521{
4522 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4523 struct bpf_map *map;
4524 int err;
4525
4526 switch (map_type) {
4527 case BPF_MAP_TYPE_DEVMAP:
4528 fallthrough;
4529 case BPF_MAP_TYPE_DEVMAP_HASH:
4530 if (unlikely(flags & BPF_F_BROADCAST)) {
4531 map = READ_ONCE(ri->map);
4532
4533 /* The map pointer is cleared when the map is being torn
4534 * down by dev_map_free()
4535 */
4536 if (unlikely(!map)) {
4537 err = -ENOENT;
4538 break;
4539 }
4540
4541 WRITE_ONCE(ri->map, NULL);
4542 err = dev_map_redirect_multi(dev, skb, xdp_prog, map,
4543 exclude_ingress: flags & BPF_F_EXCLUDE_INGRESS);
4544 } else {
4545 err = dev_map_generic_redirect(dst: fwd, skb, xdp_prog);
4546 }
4547 if (unlikely(err))
4548 goto err;
4549 break;
4550 case BPF_MAP_TYPE_XSKMAP:
4551 err = xsk_generic_rcv(xs: fwd, xdp);
4552 if (err)
4553 goto err;
4554 consume_skb(skb);
4555 break;
4556 case BPF_MAP_TYPE_CPUMAP:
4557 err = cpu_map_generic_redirect(rcpu: fwd, skb);
4558 if (unlikely(err))
4559 goto err;
4560 break;
4561 default:
4562 err = -EBADRQC;
4563 goto err;
4564 }
4565
4566 _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4567 return 0;
4568err:
4569 _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4570 return err;
4571}
4572
4573int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
4574 struct xdp_buff *xdp,
4575 const struct bpf_prog *xdp_prog)
4576{
4577 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4578 enum bpf_map_type map_type = ri->map_type;
4579 void *fwd = ri->tgt_value;
4580 u32 map_id = ri->map_id;
4581 u32 flags = ri->flags;
4582 int err;
4583
4584 ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
4585 ri->flags = 0;
4586 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4587
4588 if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) {
4589 fwd = dev_get_by_index_rcu(net: dev_net(dev), ifindex: ri->tgt_index);
4590 if (unlikely(!fwd)) {
4591 err = -EINVAL;
4592 goto err;
4593 }
4594
4595 err = xdp_ok_fwd_dev(fwd, pktlen: skb->len);
4596 if (unlikely(err))
4597 goto err;
4598
4599 skb->dev = fwd;
4600 _trace_xdp_redirect(dev, xdp_prog, ri->tgt_index);
4601 generic_xdp_tx(skb, xdp_prog);
4602 return 0;
4603 }
4604
4605 return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog, fwd, map_type, map_id, flags);
4606err:
4607 _trace_xdp_redirect_err(dev, xdp_prog, ri->tgt_index, err);
4608 return err;
4609}
4610
4611BPF_CALL_2(bpf_xdp_redirect, u32, ifindex, u64, flags)
4612{
4613 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4614
4615 if (unlikely(flags))
4616 return XDP_ABORTED;
4617
4618 /* NB! Map type UNSPEC and map_id == INT_MAX (never generated
4619 * by map_idr) is used for ifindex based XDP redirect.
4620 */
4621 ri->tgt_index = ifindex;
4622 ri->map_id = INT_MAX;
4623 ri->map_type = BPF_MAP_TYPE_UNSPEC;
4624
4625 return XDP_REDIRECT;
4626}
4627
4628static const struct bpf_func_proto bpf_xdp_redirect_proto = {
4629 .func = bpf_xdp_redirect,
4630 .gpl_only = false,
4631 .ret_type = RET_INTEGER,
4632 .arg1_type = ARG_ANYTHING,
4633 .arg2_type = ARG_ANYTHING,
4634};
4635
4636BPF_CALL_3(bpf_xdp_redirect_map, struct bpf_map *, map, u64, key,
4637 u64, flags)
4638{
4639 return map->ops->map_redirect(map, key, flags);
4640}
4641
4642static const struct bpf_func_proto bpf_xdp_redirect_map_proto = {
4643 .func = bpf_xdp_redirect_map,
4644 .gpl_only = false,
4645 .ret_type = RET_INTEGER,
4646 .arg1_type = ARG_CONST_MAP_PTR,
4647 .arg2_type = ARG_ANYTHING,
4648 .arg3_type = ARG_ANYTHING,
4649};
4650
4651static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
4652 unsigned long off, unsigned long len)
4653{
4654 void *ptr = skb_header_pointer(skb, offset: off, len, buffer: dst_buff);
4655
4656 if (unlikely(!ptr))
4657 return len;
4658 if (ptr != dst_buff)
4659 memcpy(to: dst_buff, from: ptr, len);
4660
4661 return 0;
4662}
4663
4664BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map,
4665 u64, flags, void *, meta, u64, meta_size)
4666{
4667 u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
4668
4669 if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
4670 return -EINVAL;
4671 if (unlikely(!skb || skb_size > skb->len))
4672 return -EFAULT;
4673
4674 return bpf_event_output(map, flags, meta, meta_size, ctx: skb, ctx_size: skb_size,
4675 ctx_copy: bpf_skb_copy);
4676}
4677
4678static const struct bpf_func_proto bpf_skb_event_output_proto = {
4679 .func = bpf_skb_event_output,
4680 .gpl_only = true,
4681 .ret_type = RET_INTEGER,
4682 .arg1_type = ARG_PTR_TO_CTX,
4683 .arg2_type = ARG_CONST_MAP_PTR,
4684 .arg3_type = ARG_ANYTHING,
4685 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
4686 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
4687};
4688
4689BTF_ID_LIST_SINGLE(bpf_skb_output_btf_ids, struct, sk_buff)
4690
4691const struct bpf_func_proto bpf_skb_output_proto = {
4692 .func = bpf_skb_event_output,
4693 .gpl_only = true,
4694 .ret_type = RET_INTEGER,
4695 .arg1_type = ARG_PTR_TO_BTF_ID,
4696 .arg1_btf_id = &bpf_skb_output_btf_ids[0],
4697 .arg2_type = ARG_CONST_MAP_PTR,
4698 .arg3_type = ARG_ANYTHING,
4699 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
4700 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
4701};
4702
4703static unsigned short bpf_tunnel_key_af(u64 flags)
4704{
4705 return flags & BPF_F_TUNINFO_IPV6 ? AF_INET6 : AF_INET;
4706}
4707
4708BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key *, to,
4709 u32, size, u64, flags)
4710{
4711 const struct ip_tunnel_info *info = skb_tunnel_info(skb);
4712 u8 compat[sizeof(struct bpf_tunnel_key)];
4713 void *to_orig = to;
4714 int err;
4715
4716 if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6 |
4717 BPF_F_TUNINFO_FLAGS)))) {
4718 err = -EINVAL;
4719 goto err_clear;
4720 }
4721 if (ip_tunnel_info_af(tun_info: info) != bpf_tunnel_key_af(flags)) {
4722 err = -EPROTO;
4723 goto err_clear;
4724 }
4725 if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
4726 err = -EINVAL;
4727 switch (size) {
4728 case offsetof(struct bpf_tunnel_key, local_ipv6[0]):
4729 case offsetof(struct bpf_tunnel_key, tunnel_label):
4730 case offsetof(struct bpf_tunnel_key, tunnel_ext):
4731 goto set_compat;
4732 case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
4733 /* Fixup deprecated structure layouts here, so we have
4734 * a common path later on.
4735 */
4736 if (ip_tunnel_info_af(tun_info: info) != AF_INET)
4737 goto err_clear;
4738set_compat:
4739 to = (struct bpf_tunnel_key *)compat;
4740 break;
4741 default:
4742 goto err_clear;
4743 }
4744 }
4745
4746 to->tunnel_id = be64_to_cpu(info->key.tun_id);
4747 to->tunnel_tos = info->key.tos;
4748 to->tunnel_ttl = info->key.ttl;
4749 if (flags & BPF_F_TUNINFO_FLAGS)
4750 to->tunnel_flags = ip_tunnel_flags_to_be16(flags: info->key.tun_flags);
4751 else
4752 to->tunnel_ext = 0;
4753
4754 if (flags & BPF_F_TUNINFO_IPV6) {
4755 memcpy(to: to->remote_ipv6, from: &info->key.u.ipv6.src,
4756 len: sizeof(to->remote_ipv6));
4757 memcpy(to: to->local_ipv6, from: &info->key.u.ipv6.dst,
4758 len: sizeof(to->local_ipv6));
4759 to->tunnel_label = be32_to_cpu(info->key.label);
4760 } else {
4761 to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src);
4762 memset(s: &to->remote_ipv6[1], c: 0, n: sizeof(__u32) * 3);
4763 to->local_ipv4 = be32_to_cpu(info->key.u.ipv4.dst);
4764 memset(s: &to->local_ipv6[1], c: 0, n: sizeof(__u32) * 3);
4765 to->tunnel_label = 0;
4766 }
4767
4768 if (unlikely(size != sizeof(struct bpf_tunnel_key)))
4769 memcpy(to: to_orig, from: to, len: size);
4770
4771 return 0;
4772err_clear:
4773 memset(s: to_orig, c: 0, n: size);
4774 return err;
4775}
4776
4777static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = {
4778 .func = bpf_skb_get_tunnel_key,
4779 .gpl_only = false,
4780 .ret_type = RET_INTEGER,
4781 .arg1_type = ARG_PTR_TO_CTX,
4782 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
4783 .arg3_type = ARG_CONST_SIZE,
4784 .arg4_type = ARG_ANYTHING,
4785};
4786
4787BPF_CALL_3(bpf_skb_get_tunnel_opt, struct sk_buff *, skb, u8 *, to, u32, size)
4788{
4789 const struct ip_tunnel_info *info = skb_tunnel_info(skb);
4790 int err;
4791
4792 if (unlikely(!info ||
4793 !ip_tunnel_is_options_present(info->key.tun_flags))) {
4794 err = -ENOENT;
4795 goto err_clear;
4796 }
4797 if (unlikely(size < info->options_len)) {
4798 err = -ENOMEM;
4799 goto err_clear;
4800 }
4801
4802 ip_tunnel_info_opts_get(to, info);
4803 if (size > info->options_len)
4804 memset(s: to + info->options_len, c: 0, n: size - info->options_len);
4805
4806 return info->options_len;
4807err_clear:
4808 memset(s: to, c: 0, n: size);
4809 return err;
4810}
4811
4812static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = {
4813 .func = bpf_skb_get_tunnel_opt,
4814 .gpl_only = false,
4815 .ret_type = RET_INTEGER,
4816 .arg1_type = ARG_PTR_TO_CTX,
4817 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
4818 .arg3_type = ARG_CONST_SIZE,
4819};
4820
4821static struct metadata_dst __percpu *md_dst;
4822
4823BPF_CALL_4(bpf_skb_set_tunnel_key, struct sk_buff *, skb,
4824 const struct bpf_tunnel_key *, from, u32, size, u64, flags)
4825{
4826 struct metadata_dst *md = this_cpu_ptr(md_dst);
4827 u8 compat[sizeof(struct bpf_tunnel_key)];
4828 struct ip_tunnel_info *info;
4829
4830 if (unlikely(flags & ~(BPF_F_TUNINFO_IPV6 | BPF_F_ZERO_CSUM_TX |
4831 BPF_F_DONT_FRAGMENT | BPF_F_SEQ_NUMBER |
4832 BPF_F_NO_TUNNEL_KEY)))
4833 return -EINVAL;
4834 if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
4835 switch (size) {
4836 case offsetof(struct bpf_tunnel_key, local_ipv6[0]):
4837 case offsetof(struct bpf_tunnel_key, tunnel_label):
4838 case offsetof(struct bpf_tunnel_key, tunnel_ext):
4839 case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
4840 /* Fixup deprecated structure layouts here, so we have
4841 * a common path later on.
4842 */
4843 memcpy(to: compat, from, len: size);
4844 memset(s: compat + size, c: 0, n: sizeof(compat) - size);
4845 from = (const struct bpf_tunnel_key *) compat;
4846 break;
4847 default:
4848 return -EINVAL;
4849 }
4850 }
4851 if (unlikely((!(flags & BPF_F_TUNINFO_IPV6) && from->tunnel_label) ||
4852 from->tunnel_ext))
4853 return -EINVAL;
4854
4855 skb_dst_drop(skb);
4856 dst_hold(dst: (struct dst_entry *) md);
4857 skb_dst_set(skb, dst: (struct dst_entry *) md);
4858
4859 info = &md->u.tun_info;
4860 memset(s: info, c: 0, n: sizeof(*info));
4861 info->mode = IP_TUNNEL_INFO_TX;
4862
4863 __set_bit(IP_TUNNEL_NOCACHE_BIT, info->key.tun_flags);
4864 __assign_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, info->key.tun_flags,
4865 flags & BPF_F_DONT_FRAGMENT);
4866 __assign_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags,
4867 !(flags & BPF_F_ZERO_CSUM_TX));
4868 __assign_bit(IP_TUNNEL_SEQ_BIT, info->key.tun_flags,
4869 flags & BPF_F_SEQ_NUMBER);
4870 __assign_bit(IP_TUNNEL_KEY_BIT, info->key.tun_flags,
4871 !(flags & BPF_F_NO_TUNNEL_KEY));
4872
4873 info->key.tun_id = cpu_to_be64(from->tunnel_id);
4874 info->key.tos = from->tunnel_tos;
4875 info->key.ttl = from->tunnel_ttl;
4876
4877 if (flags & BPF_F_TUNINFO_IPV6) {
4878 info->mode |= IP_TUNNEL_INFO_IPV6;
4879 memcpy(to: &info->key.u.ipv6.dst, from: from->remote_ipv6,
4880 len: sizeof(from->remote_ipv6));
4881 memcpy(to: &info->key.u.ipv6.src, from: from->local_ipv6,
4882 len: sizeof(from->local_ipv6));
4883 info->key.label = cpu_to_be32(from->tunnel_label) &
4884 IPV6_FLOWLABEL_MASK;
4885 } else {
4886 info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4);
4887 info->key.u.ipv4.src = cpu_to_be32(from->local_ipv4);
4888 info->key.flow_flags = FLOWI_FLAG_ANYSRC;
4889 }
4890
4891 return 0;
4892}
4893
4894static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = {
4895 .func = bpf_skb_set_tunnel_key,
4896 .gpl_only = false,
4897 .ret_type = RET_INTEGER,
4898 .arg1_type = ARG_PTR_TO_CTX,
4899 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
4900 .arg3_type = ARG_CONST_SIZE,
4901 .arg4_type = ARG_ANYTHING,
4902};
4903
4904BPF_CALL_3(bpf_skb_set_tunnel_opt, struct sk_buff *, skb,
4905 const u8 *, from, u32, size)
4906{
4907 struct ip_tunnel_info *info = skb_tunnel_info(skb);
4908 const struct metadata_dst *md = this_cpu_ptr(md_dst);
4909 IP_TUNNEL_DECLARE_FLAGS(present) = { };
4910
4911 if (unlikely(info != &md->u.tun_info || (size & (sizeof(u32) - 1))))
4912 return -EINVAL;
4913 if (unlikely(size > IP_TUNNEL_OPTS_MAX))
4914 return -ENOMEM;
4915
4916 ip_tunnel_set_options_present(flags: present);
4917 ip_tunnel_info_opts_set(info, from, len: size, flags: present);
4918
4919 return 0;
4920}
4921
4922static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = {
4923 .func = bpf_skb_set_tunnel_opt,
4924 .gpl_only = false,
4925 .ret_type = RET_INTEGER,
4926 .arg1_type = ARG_PTR_TO_CTX,
4927 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
4928 .arg3_type = ARG_CONST_SIZE,
4929};
4930
4931static const struct bpf_func_proto *
4932bpf_get_skb_set_tunnel_proto(enum bpf_func_id which)
4933{
4934 if (!md_dst) {
4935 struct metadata_dst __percpu *tmp;
4936
4937 tmp = metadata_dst_alloc_percpu(IP_TUNNEL_OPTS_MAX,
4938 type: METADATA_IP_TUNNEL,
4939 GFP_KERNEL);
4940 if (!tmp)
4941 return NULL;
4942 if (cmpxchg(&md_dst, NULL, tmp))
4943 metadata_dst_free_percpu(md_dst: tmp);
4944 }
4945
4946 switch (which) {
4947 case BPF_FUNC_skb_set_tunnel_key:
4948 return &bpf_skb_set_tunnel_key_proto;
4949 case BPF_FUNC_skb_set_tunnel_opt:
4950 return &bpf_skb_set_tunnel_opt_proto;
4951 default:
4952 return NULL;
4953 }
4954}
4955
4956BPF_CALL_3(bpf_skb_under_cgroup, struct sk_buff *, skb, struct bpf_map *, map,
4957 u32, idx)
4958{
4959 struct bpf_array *array = container_of(map, struct bpf_array, map);
4960 struct cgroup *cgrp;
4961 struct sock *sk;
4962
4963 sk = skb_to_full_sk(skb);
4964 if (!sk || !sk_fullsock(sk))
4965 return -ENOENT;
4966 if (unlikely(idx >= array->map.max_entries))
4967 return -E2BIG;
4968
4969 cgrp = READ_ONCE(array->ptrs[idx]);
4970 if (unlikely(!cgrp))
4971 return -EAGAIN;
4972
4973 return sk_under_cgroup_hierarchy(sk, ancestor: cgrp);
4974}
4975
4976static const struct bpf_func_proto bpf_skb_under_cgroup_proto = {
4977 .func = bpf_skb_under_cgroup,
4978 .gpl_only = false,
4979 .ret_type = RET_INTEGER,
4980 .arg1_type = ARG_PTR_TO_CTX,
4981 .arg2_type = ARG_CONST_MAP_PTR,
4982 .arg3_type = ARG_ANYTHING,
4983};
4984
4985#ifdef CONFIG_SOCK_CGROUP_DATA
4986static inline u64 __bpf_sk_cgroup_id(struct sock *sk)
4987{
4988 struct cgroup *cgrp;
4989
4990 sk = sk_to_full_sk(sk);
4991 if (!sk || !sk_fullsock(sk))
4992 return 0;
4993
4994 cgrp = sock_cgroup_ptr(skcd: &sk->sk_cgrp_data);
4995 return cgroup_id(cgrp);
4996}
4997
4998BPF_CALL_1(bpf_skb_cgroup_id, const struct sk_buff *, skb)
4999{
5000 return __bpf_sk_cgroup_id(sk: skb->sk);
5001}
5002
5003static const struct bpf_func_proto bpf_skb_cgroup_id_proto = {
5004 .func = bpf_skb_cgroup_id,
5005 .gpl_only = false,
5006 .ret_type = RET_INTEGER,
5007 .arg1_type = ARG_PTR_TO_CTX,
5008};
5009
5010static inline u64 __bpf_sk_ancestor_cgroup_id(struct sock *sk,
5011 int ancestor_level)
5012{
5013 struct cgroup *ancestor;
5014 struct cgroup *cgrp;
5015
5016 sk = sk_to_full_sk(sk);
5017 if (!sk || !sk_fullsock(sk))
5018 return 0;
5019
5020 cgrp = sock_cgroup_ptr(skcd: &sk->sk_cgrp_data);
5021 ancestor = cgroup_ancestor(cgrp, ancestor_level);
5022 if (!ancestor)
5023 return 0;
5024
5025 return cgroup_id(cgrp: ancestor);
5026}
5027
5028BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int,
5029 ancestor_level)
5030{
5031 return __bpf_sk_ancestor_cgroup_id(sk: skb->sk, ancestor_level);
5032}
5033
5034static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = {
5035 .func = bpf_skb_ancestor_cgroup_id,
5036 .gpl_only = false,
5037 .ret_type = RET_INTEGER,
5038 .arg1_type = ARG_PTR_TO_CTX,
5039 .arg2_type = ARG_ANYTHING,
5040};
5041
5042BPF_CALL_1(bpf_sk_cgroup_id, struct sock *, sk)
5043{
5044 return __bpf_sk_cgroup_id(sk);
5045}
5046
5047static const struct bpf_func_proto bpf_sk_cgroup_id_proto = {
5048 .func = bpf_sk_cgroup_id,
5049 .gpl_only = false,
5050 .ret_type = RET_INTEGER,
5051 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5052};
5053
5054BPF_CALL_2(bpf_sk_ancestor_cgroup_id, struct sock *, sk, int, ancestor_level)
5055{
5056 return __bpf_sk_ancestor_cgroup_id(sk, ancestor_level);
5057}
5058
5059static const struct bpf_func_proto bpf_sk_ancestor_cgroup_id_proto = {
5060 .func = bpf_sk_ancestor_cgroup_id,
5061 .gpl_only = false,
5062 .ret_type = RET_INTEGER,
5063 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5064 .arg2_type = ARG_ANYTHING,
5065};
5066#endif
5067
5068static unsigned long bpf_xdp_copy(void *dst, const void *ctx,
5069 unsigned long off, unsigned long len)
5070{
5071 struct xdp_buff *xdp = (struct xdp_buff *)ctx;
5072
5073 bpf_xdp_copy_buf(xdp, off, buf: dst, len, flush: false);
5074 return 0;
5075}
5076
5077BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,
5078 u64, flags, void *, meta, u64, meta_size)
5079{
5080 u64 xdp_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
5081
5082 if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
5083 return -EINVAL;
5084
5085 if (unlikely(!xdp || xdp_size > xdp_get_buff_len(xdp)))
5086 return -EFAULT;
5087
5088 return bpf_event_output(map, flags, meta, meta_size, ctx: xdp,
5089 ctx_size: xdp_size, ctx_copy: bpf_xdp_copy);
5090}
5091
5092static const struct bpf_func_proto bpf_xdp_event_output_proto = {
5093 .func = bpf_xdp_event_output,
5094 .gpl_only = true,
5095 .ret_type = RET_INTEGER,
5096 .arg1_type = ARG_PTR_TO_CTX,
5097 .arg2_type = ARG_CONST_MAP_PTR,
5098 .arg3_type = ARG_ANYTHING,
5099 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5100 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
5101};
5102
5103BTF_ID_LIST_SINGLE(bpf_xdp_output_btf_ids, struct, xdp_buff)
5104
5105const struct bpf_func_proto bpf_xdp_output_proto = {
5106 .func = bpf_xdp_event_output,
5107 .gpl_only = true,
5108 .ret_type = RET_INTEGER,
5109 .arg1_type = ARG_PTR_TO_BTF_ID,
5110 .arg1_btf_id = &bpf_xdp_output_btf_ids[0],
5111 .arg2_type = ARG_CONST_MAP_PTR,
5112 .arg3_type = ARG_ANYTHING,
5113 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5114 .arg5_type = ARG_CONST_SIZE_OR_ZERO,
5115};
5116
5117BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb)
5118{
5119 return skb->sk ? __sock_gen_cookie(sk: skb->sk) : 0;
5120}
5121
5122static const struct bpf_func_proto bpf_get_socket_cookie_proto = {
5123 .func = bpf_get_socket_cookie,
5124 .gpl_only = false,
5125 .ret_type = RET_INTEGER,
5126 .arg1_type = ARG_PTR_TO_CTX,
5127};
5128
5129BPF_CALL_1(bpf_get_socket_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
5130{
5131 return __sock_gen_cookie(sk: ctx->sk);
5132}
5133
5134static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = {
5135 .func = bpf_get_socket_cookie_sock_addr,
5136 .gpl_only = false,
5137 .ret_type = RET_INTEGER,
5138 .arg1_type = ARG_PTR_TO_CTX,
5139};
5140
5141BPF_CALL_1(bpf_get_socket_cookie_sock, struct sock *, ctx)
5142{
5143 return __sock_gen_cookie(sk: ctx);
5144}
5145
5146static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = {
5147 .func = bpf_get_socket_cookie_sock,
5148 .gpl_only = false,
5149 .ret_type = RET_INTEGER,
5150 .arg1_type = ARG_PTR_TO_CTX,
5151};
5152
5153BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk)
5154{
5155 return sk ? sock_gen_cookie(sk) : 0;
5156}
5157
5158const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = {
5159 .func = bpf_get_socket_ptr_cookie,
5160 .gpl_only = false,
5161 .ret_type = RET_INTEGER,
5162 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON | PTR_MAYBE_NULL,
5163};
5164
5165BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
5166{
5167 return __sock_gen_cookie(sk: ctx->sk);
5168}
5169
5170static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = {
5171 .func = bpf_get_socket_cookie_sock_ops,
5172 .gpl_only = false,
5173 .ret_type = RET_INTEGER,
5174 .arg1_type = ARG_PTR_TO_CTX,
5175};
5176
5177static u64 __bpf_get_netns_cookie(struct sock *sk)
5178{
5179 const struct net *net = sk ? sock_net(sk) : &init_net;
5180
5181 return net->net_cookie;
5182}
5183
5184BPF_CALL_1(bpf_get_netns_cookie, struct sk_buff *, skb)
5185{
5186 return __bpf_get_netns_cookie(sk: skb && skb->sk ? skb->sk : NULL);
5187}
5188
5189static const struct bpf_func_proto bpf_get_netns_cookie_proto = {
5190 .func = bpf_get_netns_cookie,
5191 .ret_type = RET_INTEGER,
5192 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
5193};
5194
5195BPF_CALL_1(bpf_get_netns_cookie_sock, struct sock *, ctx)
5196{
5197 return __bpf_get_netns_cookie(sk: ctx);
5198}
5199
5200static const struct bpf_func_proto bpf_get_netns_cookie_sock_proto = {
5201 .func = bpf_get_netns_cookie_sock,
5202 .gpl_only = false,
5203 .ret_type = RET_INTEGER,
5204 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
5205};
5206
5207BPF_CALL_1(bpf_get_netns_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
5208{
5209 return __bpf_get_netns_cookie(sk: ctx ? ctx->sk : NULL);
5210}
5211
5212static const struct bpf_func_proto bpf_get_netns_cookie_sock_addr_proto = {
5213 .func = bpf_get_netns_cookie_sock_addr,
5214 .gpl_only = false,
5215 .ret_type = RET_INTEGER,
5216 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
5217};
5218
5219BPF_CALL_1(bpf_get_netns_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
5220{
5221 return __bpf_get_netns_cookie(sk: ctx ? ctx->sk : NULL);
5222}
5223
5224static const struct bpf_func_proto bpf_get_netns_cookie_sock_ops_proto = {
5225 .func = bpf_get_netns_cookie_sock_ops,
5226 .gpl_only = false,
5227 .ret_type = RET_INTEGER,
5228 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
5229};
5230
5231BPF_CALL_1(bpf_get_netns_cookie_sk_msg, struct sk_msg *, ctx)
5232{
5233 return __bpf_get_netns_cookie(sk: ctx ? ctx->sk : NULL);
5234}
5235
5236static const struct bpf_func_proto bpf_get_netns_cookie_sk_msg_proto = {
5237 .func = bpf_get_netns_cookie_sk_msg,
5238 .gpl_only = false,
5239 .ret_type = RET_INTEGER,
5240 .arg1_type = ARG_PTR_TO_CTX_OR_NULL,
5241};
5242
5243BPF_CALL_1(bpf_get_socket_uid, struct sk_buff *, skb)
5244{
5245 struct sock *sk = sk_to_full_sk(sk: skb->sk);
5246 kuid_t kuid;
5247
5248 if (!sk || !sk_fullsock(sk))
5249 return overflowuid;
5250 kuid = sock_net_uid(net: sock_net(sk), sk);
5251 return from_kuid_munged(to: sock_net(sk)->user_ns, kuid);
5252}
5253
5254static const struct bpf_func_proto bpf_get_socket_uid_proto = {
5255 .func = bpf_get_socket_uid,
5256 .gpl_only = false,
5257 .ret_type = RET_INTEGER,
5258 .arg1_type = ARG_PTR_TO_CTX,
5259};
5260
5261static int sk_bpf_set_get_cb_flags(struct sock *sk, char *optval, bool getopt)
5262{
5263 u32 sk_bpf_cb_flags;
5264
5265 if (getopt) {
5266 *(u32 *)optval = sk->sk_bpf_cb_flags;
5267 return 0;
5268 }
5269
5270 sk_bpf_cb_flags = *(u32 *)optval;
5271
5272 if (sk_bpf_cb_flags & ~SK_BPF_CB_MASK)
5273 return -EINVAL;
5274
5275 sk->sk_bpf_cb_flags = sk_bpf_cb_flags;
5276
5277 return 0;
5278}
5279
5280static int sol_socket_sockopt(struct sock *sk, int optname,
5281 char *optval, int *optlen,
5282 bool getopt)
5283{
5284 switch (optname) {
5285 case SO_REUSEADDR:
5286 case SO_SNDBUF:
5287 case SO_RCVBUF:
5288 case SO_KEEPALIVE:
5289 case SO_PRIORITY:
5290 case SO_REUSEPORT:
5291 case SO_RCVLOWAT:
5292 case SO_MARK:
5293 case SO_MAX_PACING_RATE:
5294 case SO_BINDTOIFINDEX:
5295 case SO_TXREHASH:
5296 case SK_BPF_CB_FLAGS:
5297 if (*optlen != sizeof(int))
5298 return -EINVAL;
5299 break;
5300 case SO_BINDTODEVICE:
5301 break;
5302 default:
5303 return -EINVAL;
5304 }
5305
5306 if (optname == SK_BPF_CB_FLAGS)
5307 return sk_bpf_set_get_cb_flags(sk, optval, getopt);
5308
5309 if (getopt) {
5310 if (optname == SO_BINDTODEVICE)
5311 return -EINVAL;
5312 return sk_getsockopt(sk, SOL_SOCKET, optname,
5313 optval: KERNEL_SOCKPTR(p: optval),
5314 optlen: KERNEL_SOCKPTR(p: optlen));
5315 }
5316
5317 return sk_setsockopt(sk, SOL_SOCKET, optname,
5318 optval: KERNEL_SOCKPTR(p: optval), optlen: *optlen);
5319}
5320
5321static int bpf_sol_tcp_getsockopt(struct sock *sk, int optname,
5322 char *optval, int optlen)
5323{
5324 if (optlen != sizeof(int))
5325 return -EINVAL;
5326
5327 switch (optname) {
5328 case TCP_BPF_SOCK_OPS_CB_FLAGS: {
5329 int cb_flags = tcp_sk(sk)->bpf_sock_ops_cb_flags;
5330
5331 memcpy(to: optval, from: &cb_flags, len: optlen);
5332 break;
5333 }
5334 case TCP_BPF_RTO_MIN: {
5335 int rto_min_us = jiffies_to_usecs(inet_csk(sk)->icsk_rto_min);
5336
5337 memcpy(to: optval, from: &rto_min_us, len: optlen);
5338 break;
5339 }
5340 case TCP_BPF_DELACK_MAX: {
5341 int delack_max_us = jiffies_to_usecs(inet_csk(sk)->icsk_delack_max);
5342
5343 memcpy(to: optval, from: &delack_max_us, len: optlen);
5344 break;
5345 }
5346 default:
5347 return -EINVAL;
5348 }
5349
5350 return 0;
5351}
5352
5353static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname,
5354 char *optval, int optlen)
5355{
5356 struct tcp_sock *tp = tcp_sk(sk);
5357 unsigned long timeout;
5358 int val;
5359
5360 if (optlen != sizeof(int))
5361 return -EINVAL;
5362
5363 val = *(int *)optval;
5364
5365 /* Only some options are supported */
5366 switch (optname) {
5367 case TCP_BPF_IW:
5368 if (val <= 0 || tp->data_segs_out > tp->syn_data)
5369 return -EINVAL;
5370 tcp_snd_cwnd_set(tp, val);
5371 break;
5372 case TCP_BPF_SNDCWND_CLAMP:
5373 if (val <= 0)
5374 return -EINVAL;
5375 tp->snd_cwnd_clamp = val;
5376 tp->snd_ssthresh = val;
5377 break;
5378 case TCP_BPF_DELACK_MAX:
5379 timeout = usecs_to_jiffies(u: val);
5380 if (timeout > TCP_DELACK_MAX ||
5381 timeout < TCP_TIMEOUT_MIN)
5382 return -EINVAL;
5383 inet_csk(sk)->icsk_delack_max = timeout;
5384 break;
5385 case TCP_BPF_RTO_MIN:
5386 timeout = usecs_to_jiffies(u: val);
5387 if (timeout > TCP_RTO_MIN ||
5388 timeout < TCP_TIMEOUT_MIN)
5389 return -EINVAL;
5390 inet_csk(sk)->icsk_rto_min = timeout;
5391 break;
5392 case TCP_BPF_SOCK_OPS_CB_FLAGS:
5393 if (val & ~(BPF_SOCK_OPS_ALL_CB_FLAGS))
5394 return -EINVAL;
5395 tp->bpf_sock_ops_cb_flags = val;
5396 break;
5397 default:
5398 return -EINVAL;
5399 }
5400
5401 return 0;
5402}
5403
5404static int sol_tcp_sockopt_congestion(struct sock *sk, char *optval,
5405 int *optlen, bool getopt)
5406{
5407 struct tcp_sock *tp;
5408 int ret;
5409
5410 if (*optlen < 2)
5411 return -EINVAL;
5412
5413 if (getopt) {
5414 if (!inet_csk(sk)->icsk_ca_ops)
5415 return -EINVAL;
5416 /* BPF expects NULL-terminated tcp-cc string */
5417 optval[--(*optlen)] = '\0';
5418 return do_tcp_getsockopt(sk, SOL_TCP, TCP_CONGESTION,
5419 optval: KERNEL_SOCKPTR(p: optval),
5420 optlen: KERNEL_SOCKPTR(p: optlen));
5421 }
5422
5423 /* "cdg" is the only cc that alloc a ptr
5424 * in inet_csk_ca area. The bpf-tcp-cc may
5425 * overwrite this ptr after switching to cdg.
5426 */
5427 if (*optlen >= sizeof("cdg") - 1 && !strncmp("cdg", optval, *optlen))
5428 return -ENOTSUPP;
5429
5430 /* It stops this looping
5431 *
5432 * .init => bpf_setsockopt(tcp_cc) => .init =>
5433 * bpf_setsockopt(tcp_cc)" => .init => ....
5434 *
5435 * The second bpf_setsockopt(tcp_cc) is not allowed
5436 * in order to break the loop when both .init
5437 * are the same bpf prog.
5438 *
5439 * This applies even the second bpf_setsockopt(tcp_cc)
5440 * does not cause a loop. This limits only the first
5441 * '.init' can call bpf_setsockopt(TCP_CONGESTION) to
5442 * pick a fallback cc (eg. peer does not support ECN)
5443 * and the second '.init' cannot fallback to
5444 * another.
5445 */
5446 tp = tcp_sk(sk);
5447 if (tp->bpf_chg_cc_inprogress)
5448 return -EBUSY;
5449
5450 tp->bpf_chg_cc_inprogress = 1;
5451 ret = do_tcp_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
5452 optval: KERNEL_SOCKPTR(p: optval), optlen: *optlen);
5453 tp->bpf_chg_cc_inprogress = 0;
5454 return ret;
5455}
5456
5457static int sol_tcp_sockopt(struct sock *sk, int optname,
5458 char *optval, int *optlen,
5459 bool getopt)
5460{
5461 if (sk->sk_protocol != IPPROTO_TCP)
5462 return -EINVAL;
5463
5464 switch (optname) {
5465 case TCP_NODELAY:
5466 case TCP_MAXSEG:
5467 case TCP_KEEPIDLE:
5468 case TCP_KEEPINTVL:
5469 case TCP_KEEPCNT:
5470 case TCP_SYNCNT:
5471 case TCP_WINDOW_CLAMP:
5472 case TCP_THIN_LINEAR_TIMEOUTS:
5473 case TCP_USER_TIMEOUT:
5474 case TCP_NOTSENT_LOWAT:
5475 case TCP_SAVE_SYN:
5476 case TCP_RTO_MAX_MS:
5477 if (*optlen != sizeof(int))
5478 return -EINVAL;
5479 break;
5480 case TCP_CONGESTION:
5481 return sol_tcp_sockopt_congestion(sk, optval, optlen, getopt);
5482 case TCP_SAVED_SYN:
5483 if (*optlen < 1)
5484 return -EINVAL;
5485 break;
5486 default:
5487 if (getopt)
5488 return bpf_sol_tcp_getsockopt(sk, optname, optval, optlen: *optlen);
5489 return bpf_sol_tcp_setsockopt(sk, optname, optval, optlen: *optlen);
5490 }
5491
5492 if (getopt) {
5493 if (optname == TCP_SAVED_SYN) {
5494 struct tcp_sock *tp = tcp_sk(sk);
5495
5496 if (!tp->saved_syn ||
5497 *optlen > tcp_saved_syn_len(saved_syn: tp->saved_syn))
5498 return -EINVAL;
5499 memcpy(to: optval, from: tp->saved_syn->data, len: *optlen);
5500 /* It cannot free tp->saved_syn here because it
5501 * does not know if the user space still needs it.
5502 */
5503 return 0;
5504 }
5505
5506 return do_tcp_getsockopt(sk, SOL_TCP, optname,
5507 optval: KERNEL_SOCKPTR(p: optval),
5508 optlen: KERNEL_SOCKPTR(p: optlen));
5509 }
5510
5511 return do_tcp_setsockopt(sk, SOL_TCP, optname,
5512 optval: KERNEL_SOCKPTR(p: optval), optlen: *optlen);
5513}
5514
5515static int sol_ip_sockopt(struct sock *sk, int optname,
5516 char *optval, int *optlen,
5517 bool getopt)
5518{
5519 if (sk->sk_family != AF_INET)
5520 return -EINVAL;
5521
5522 switch (optname) {
5523 case IP_TOS:
5524 if (*optlen != sizeof(int))
5525 return -EINVAL;
5526 break;
5527 default:
5528 return -EINVAL;
5529 }
5530
5531 if (getopt)
5532 return do_ip_getsockopt(sk, SOL_IP, optname,
5533 optval: KERNEL_SOCKPTR(p: optval),
5534 optlen: KERNEL_SOCKPTR(p: optlen));
5535
5536 return do_ip_setsockopt(sk, SOL_IP, optname,
5537 optval: KERNEL_SOCKPTR(p: optval), optlen: *optlen);
5538}
5539
5540static int sol_ipv6_sockopt(struct sock *sk, int optname,
5541 char *optval, int *optlen,
5542 bool getopt)
5543{
5544 if (sk->sk_family != AF_INET6)
5545 return -EINVAL;
5546
5547 switch (optname) {
5548 case IPV6_TCLASS:
5549 case IPV6_AUTOFLOWLABEL:
5550 if (*optlen != sizeof(int))
5551 return -EINVAL;
5552 break;
5553 default:
5554 return -EINVAL;
5555 }
5556
5557 if (getopt)
5558 return ipv6_bpf_stub->ipv6_getsockopt(sk, SOL_IPV6, optname,
5559 KERNEL_SOCKPTR(p: optval),
5560 KERNEL_SOCKPTR(p: optlen));
5561
5562 return ipv6_bpf_stub->ipv6_setsockopt(sk, SOL_IPV6, optname,
5563 KERNEL_SOCKPTR(p: optval), *optlen);
5564}
5565
5566static int __bpf_setsockopt(struct sock *sk, int level, int optname,
5567 char *optval, int optlen)
5568{
5569 if (!sk_fullsock(sk))
5570 return -EINVAL;
5571
5572 if (level == SOL_SOCKET)
5573 return sol_socket_sockopt(sk, optname, optval, optlen: &optlen, getopt: false);
5574 else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
5575 return sol_ip_sockopt(sk, optname, optval, optlen: &optlen, getopt: false);
5576 else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
5577 return sol_ipv6_sockopt(sk, optname, optval, optlen: &optlen, getopt: false);
5578 else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
5579 return sol_tcp_sockopt(sk, optname, optval, optlen: &optlen, getopt: false);
5580
5581 return -EINVAL;
5582}
5583
5584static bool is_locked_tcp_sock_ops(struct bpf_sock_ops_kern *bpf_sock)
5585{
5586 return bpf_sock->op <= BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
5587}
5588
5589static int _bpf_setsockopt(struct sock *sk, int level, int optname,
5590 char *optval, int optlen)
5591{
5592 if (sk_fullsock(sk))
5593 sock_owned_by_me(sk);
5594 return __bpf_setsockopt(sk, level, optname, optval, optlen);
5595}
5596
5597static int __bpf_getsockopt(struct sock *sk, int level, int optname,
5598 char *optval, int optlen)
5599{
5600 int err, saved_optlen = optlen;
5601
5602 if (!sk_fullsock(sk)) {
5603 err = -EINVAL;
5604 goto done;
5605 }
5606
5607 if (level == SOL_SOCKET)
5608 err = sol_socket_sockopt(sk, optname, optval, optlen: &optlen, getopt: true);
5609 else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
5610 err = sol_tcp_sockopt(sk, optname, optval, optlen: &optlen, getopt: true);
5611 else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
5612 err = sol_ip_sockopt(sk, optname, optval, optlen: &optlen, getopt: true);
5613 else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
5614 err = sol_ipv6_sockopt(sk, optname, optval, optlen: &optlen, getopt: true);
5615 else
5616 err = -EINVAL;
5617
5618done:
5619 if (err)
5620 optlen = 0;
5621 if (optlen < saved_optlen)
5622 memset(s: optval + optlen, c: 0, n: saved_optlen - optlen);
5623 return err;
5624}
5625
5626static int _bpf_getsockopt(struct sock *sk, int level, int optname,
5627 char *optval, int optlen)
5628{
5629 if (sk_fullsock(sk))
5630 sock_owned_by_me(sk);
5631 return __bpf_getsockopt(sk, level, optname, optval, optlen);
5632}
5633
5634BPF_CALL_5(bpf_sk_setsockopt, struct sock *, sk, int, level,
5635 int, optname, char *, optval, int, optlen)
5636{
5637 return _bpf_setsockopt(sk, level, optname, optval, optlen);
5638}
5639
5640const struct bpf_func_proto bpf_sk_setsockopt_proto = {
5641 .func = bpf_sk_setsockopt,
5642 .gpl_only = false,
5643 .ret_type = RET_INTEGER,
5644 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5645 .arg2_type = ARG_ANYTHING,
5646 .arg3_type = ARG_ANYTHING,
5647 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5648 .arg5_type = ARG_CONST_SIZE,
5649};
5650
5651BPF_CALL_5(bpf_sk_getsockopt, struct sock *, sk, int, level,
5652 int, optname, char *, optval, int, optlen)
5653{
5654 return _bpf_getsockopt(sk, level, optname, optval, optlen);
5655}
5656
5657const struct bpf_func_proto bpf_sk_getsockopt_proto = {
5658 .func = bpf_sk_getsockopt,
5659 .gpl_only = false,
5660 .ret_type = RET_INTEGER,
5661 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5662 .arg2_type = ARG_ANYTHING,
5663 .arg3_type = ARG_ANYTHING,
5664 .arg4_type = ARG_PTR_TO_UNINIT_MEM,
5665 .arg5_type = ARG_CONST_SIZE,
5666};
5667
5668BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level,
5669 int, optname, char *, optval, int, optlen)
5670{
5671 return __bpf_setsockopt(sk, level, optname, optval, optlen);
5672}
5673
5674const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto = {
5675 .func = bpf_unlocked_sk_setsockopt,
5676 .gpl_only = false,
5677 .ret_type = RET_INTEGER,
5678 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5679 .arg2_type = ARG_ANYTHING,
5680 .arg3_type = ARG_ANYTHING,
5681 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5682 .arg5_type = ARG_CONST_SIZE,
5683};
5684
5685BPF_CALL_5(bpf_unlocked_sk_getsockopt, struct sock *, sk, int, level,
5686 int, optname, char *, optval, int, optlen)
5687{
5688 return __bpf_getsockopt(sk, level, optname, optval, optlen);
5689}
5690
5691const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto = {
5692 .func = bpf_unlocked_sk_getsockopt,
5693 .gpl_only = false,
5694 .ret_type = RET_INTEGER,
5695 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5696 .arg2_type = ARG_ANYTHING,
5697 .arg3_type = ARG_ANYTHING,
5698 .arg4_type = ARG_PTR_TO_UNINIT_MEM,
5699 .arg5_type = ARG_CONST_SIZE,
5700};
5701
5702BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx,
5703 int, level, int, optname, char *, optval, int, optlen)
5704{
5705 return _bpf_setsockopt(sk: ctx->sk, level, optname, optval, optlen);
5706}
5707
5708static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = {
5709 .func = bpf_sock_addr_setsockopt,
5710 .gpl_only = false,
5711 .ret_type = RET_INTEGER,
5712 .arg1_type = ARG_PTR_TO_CTX,
5713 .arg2_type = ARG_ANYTHING,
5714 .arg3_type = ARG_ANYTHING,
5715 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5716 .arg5_type = ARG_CONST_SIZE,
5717};
5718
5719BPF_CALL_5(bpf_sock_addr_getsockopt, struct bpf_sock_addr_kern *, ctx,
5720 int, level, int, optname, char *, optval, int, optlen)
5721{
5722 return _bpf_getsockopt(sk: ctx->sk, level, optname, optval, optlen);
5723}
5724
5725static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = {
5726 .func = bpf_sock_addr_getsockopt,
5727 .gpl_only = false,
5728 .ret_type = RET_INTEGER,
5729 .arg1_type = ARG_PTR_TO_CTX,
5730 .arg2_type = ARG_ANYTHING,
5731 .arg3_type = ARG_ANYTHING,
5732 .arg4_type = ARG_PTR_TO_UNINIT_MEM,
5733 .arg5_type = ARG_CONST_SIZE,
5734};
5735
5736BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
5737 int, level, int, optname, char *, optval, int, optlen)
5738{
5739 if (!is_locked_tcp_sock_ops(bpf_sock))
5740 return -EOPNOTSUPP;
5741
5742 return _bpf_setsockopt(sk: bpf_sock->sk, level, optname, optval, optlen);
5743}
5744
5745static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = {
5746 .func = bpf_sock_ops_setsockopt,
5747 .gpl_only = false,
5748 .ret_type = RET_INTEGER,
5749 .arg1_type = ARG_PTR_TO_CTX,
5750 .arg2_type = ARG_ANYTHING,
5751 .arg3_type = ARG_ANYTHING,
5752 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5753 .arg5_type = ARG_CONST_SIZE,
5754};
5755
5756static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock,
5757 int optname, const u8 **start)
5758{
5759 struct sk_buff *syn_skb = bpf_sock->syn_skb;
5760 const u8 *hdr_start;
5761 int ret;
5762
5763 if (syn_skb) {
5764 /* sk is a request_sock here */
5765
5766 if (optname == TCP_BPF_SYN) {
5767 hdr_start = syn_skb->data;
5768 ret = tcp_hdrlen(skb: syn_skb);
5769 } else if (optname == TCP_BPF_SYN_IP) {
5770 hdr_start = skb_network_header(skb: syn_skb);
5771 ret = skb_network_header_len(skb: syn_skb) +
5772 tcp_hdrlen(skb: syn_skb);
5773 } else {
5774 /* optname == TCP_BPF_SYN_MAC */
5775 hdr_start = skb_mac_header(skb: syn_skb);
5776 ret = skb_mac_header_len(skb: syn_skb) +
5777 skb_network_header_len(skb: syn_skb) +
5778 tcp_hdrlen(skb: syn_skb);
5779 }
5780 } else {
5781 struct sock *sk = bpf_sock->sk;
5782 struct saved_syn *saved_syn;
5783
5784 if (sk->sk_state == TCP_NEW_SYN_RECV)
5785 /* synack retransmit. bpf_sock->syn_skb will
5786 * not be available. It has to resort to
5787 * saved_syn (if it is saved).
5788 */
5789 saved_syn = inet_reqsk(sk)->saved_syn;
5790 else
5791 saved_syn = tcp_sk(sk)->saved_syn;
5792
5793 if (!saved_syn)
5794 return -ENOENT;
5795
5796 if (optname == TCP_BPF_SYN) {
5797 hdr_start = saved_syn->data +
5798 saved_syn->mac_hdrlen +
5799 saved_syn->network_hdrlen;
5800 ret = saved_syn->tcp_hdrlen;
5801 } else if (optname == TCP_BPF_SYN_IP) {
5802 hdr_start = saved_syn->data +
5803 saved_syn->mac_hdrlen;
5804 ret = saved_syn->network_hdrlen +
5805 saved_syn->tcp_hdrlen;
5806 } else {
5807 /* optname == TCP_BPF_SYN_MAC */
5808
5809 /* TCP_SAVE_SYN may not have saved the mac hdr */
5810 if (!saved_syn->mac_hdrlen)
5811 return -ENOENT;
5812
5813 hdr_start = saved_syn->data;
5814 ret = saved_syn->mac_hdrlen +
5815 saved_syn->network_hdrlen +
5816 saved_syn->tcp_hdrlen;
5817 }
5818 }
5819
5820 *start = hdr_start;
5821 return ret;
5822}
5823
5824BPF_CALL_5(bpf_sock_ops_getsockopt, struct bpf_sock_ops_kern *, bpf_sock,
5825 int, level, int, optname, char *, optval, int, optlen)
5826{
5827 if (!is_locked_tcp_sock_ops(bpf_sock))
5828 return -EOPNOTSUPP;
5829
5830 if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP &&
5831 optname >= TCP_BPF_SYN && optname <= TCP_BPF_SYN_MAC) {
5832 int ret, copy_len = 0;
5833 const u8 *start;
5834
5835 ret = bpf_sock_ops_get_syn(bpf_sock, optname, start: &start);
5836 if (ret > 0) {
5837 copy_len = ret;
5838 if (optlen < copy_len) {
5839 copy_len = optlen;
5840 ret = -ENOSPC;
5841 }
5842
5843 memcpy(to: optval, from: start, len: copy_len);
5844 }
5845
5846 /* Zero out unused buffer at the end */
5847 memset(s: optval + copy_len, c: 0, n: optlen - copy_len);
5848
5849 return ret;
5850 }
5851
5852 return _bpf_getsockopt(sk: bpf_sock->sk, level, optname, optval, optlen);
5853}
5854
5855static const struct bpf_func_proto bpf_sock_ops_getsockopt_proto = {
5856 .func = bpf_sock_ops_getsockopt,
5857 .gpl_only = false,
5858 .ret_type = RET_INTEGER,
5859 .arg1_type = ARG_PTR_TO_CTX,
5860 .arg2_type = ARG_ANYTHING,
5861 .arg3_type = ARG_ANYTHING,
5862 .arg4_type = ARG_PTR_TO_UNINIT_MEM,
5863 .arg5_type = ARG_CONST_SIZE,
5864};
5865
5866BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock,
5867 int, argval)
5868{
5869 struct sock *sk = bpf_sock->sk;
5870 int val = argval & BPF_SOCK_OPS_ALL_CB_FLAGS;
5871
5872 if (!is_locked_tcp_sock_ops(bpf_sock))
5873 return -EOPNOTSUPP;
5874
5875 if (!IS_ENABLED(CONFIG_INET) || !sk_fullsock(sk))
5876 return -EINVAL;
5877
5878 tcp_sk(sk)->bpf_sock_ops_cb_flags = val;
5879
5880 return argval & (~BPF_SOCK_OPS_ALL_CB_FLAGS);
5881}
5882
5883static const struct bpf_func_proto bpf_sock_ops_cb_flags_set_proto = {
5884 .func = bpf_sock_ops_cb_flags_set,
5885 .gpl_only = false,
5886 .ret_type = RET_INTEGER,
5887 .arg1_type = ARG_PTR_TO_CTX,
5888 .arg2_type = ARG_ANYTHING,
5889};
5890
5891const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly;
5892EXPORT_SYMBOL_GPL(ipv6_bpf_stub);
5893
5894BPF_CALL_3(bpf_bind, struct bpf_sock_addr_kern *, ctx, struct sockaddr *, addr,
5895 int, addr_len)
5896{
5897#ifdef CONFIG_INET
5898 struct sock *sk = ctx->sk;
5899 u32 flags = BIND_FROM_BPF;
5900 int err;
5901
5902 err = -EINVAL;
5903 if (addr_len < offsetofend(struct sockaddr, sa_family))
5904 return err;
5905 if (addr->sa_family == AF_INET) {
5906 if (addr_len < sizeof(struct sockaddr_in))
5907 return err;
5908 if (((struct sockaddr_in *)addr)->sin_port == htons(0))
5909 flags |= BIND_FORCE_ADDRESS_NO_PORT;
5910 return __inet_bind(sk, uaddr: addr, addr_len, flags);
5911#if IS_ENABLED(CONFIG_IPV6)
5912 } else if (addr->sa_family == AF_INET6) {
5913 if (addr_len < SIN6_LEN_RFC2133)
5914 return err;
5915 if (((struct sockaddr_in6 *)addr)->sin6_port == htons(0))
5916 flags |= BIND_FORCE_ADDRESS_NO_PORT;
5917 /* ipv6_bpf_stub cannot be NULL, since it's called from
5918 * bpf_cgroup_inet6_connect hook and ipv6 is already loaded
5919 */
5920 return ipv6_bpf_stub->inet6_bind(sk, addr, addr_len, flags);
5921#endif /* CONFIG_IPV6 */
5922 }
5923#endif /* CONFIG_INET */
5924
5925 return -EAFNOSUPPORT;
5926}
5927
5928static const struct bpf_func_proto bpf_bind_proto = {
5929 .func = bpf_bind,
5930 .gpl_only = false,
5931 .ret_type = RET_INTEGER,
5932 .arg1_type = ARG_PTR_TO_CTX,
5933 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
5934 .arg3_type = ARG_CONST_SIZE,
5935};
5936
5937#ifdef CONFIG_XFRM
5938
5939#if (IS_BUILTIN(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
5940 (IS_MODULE(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
5941
5942struct metadata_dst __percpu *xfrm_bpf_md_dst;
5943EXPORT_SYMBOL_GPL(xfrm_bpf_md_dst);
5944
5945#endif
5946
5947BPF_CALL_5(bpf_skb_get_xfrm_state, struct sk_buff *, skb, u32, index,
5948 struct bpf_xfrm_state *, to, u32, size, u64, flags)
5949{
5950 const struct sec_path *sp = skb_sec_path(skb);
5951 const struct xfrm_state *x;
5952
5953 if (!sp || unlikely(index >= sp->len || flags))
5954 goto err_clear;
5955
5956 x = sp->xvec[index];
5957
5958 if (unlikely(size != sizeof(struct bpf_xfrm_state)))
5959 goto err_clear;
5960
5961 to->reqid = x->props.reqid;
5962 to->spi = x->id.spi;
5963 to->family = x->props.family;
5964 to->ext = 0;
5965
5966 if (to->family == AF_INET6) {
5967 memcpy(to: to->remote_ipv6, from: x->props.saddr.a6,
5968 len: sizeof(to->remote_ipv6));
5969 } else {
5970 to->remote_ipv4 = x->props.saddr.a4;
5971 memset(s: &to->remote_ipv6[1], c: 0, n: sizeof(__u32) * 3);
5972 }
5973
5974 return 0;
5975err_clear:
5976 memset(s: to, c: 0, n: size);
5977 return -EINVAL;
5978}
5979
5980static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = {
5981 .func = bpf_skb_get_xfrm_state,
5982 .gpl_only = false,
5983 .ret_type = RET_INTEGER,
5984 .arg1_type = ARG_PTR_TO_CTX,
5985 .arg2_type = ARG_ANYTHING,
5986 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
5987 .arg4_type = ARG_CONST_SIZE,
5988 .arg5_type = ARG_ANYTHING,
5989};
5990#endif
5991
5992#if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)
5993static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params, u32 mtu)
5994{
5995 params->h_vlan_TCI = 0;
5996 params->h_vlan_proto = 0;
5997 if (mtu)
5998 params->mtu_result = mtu; /* union with tot_len */
5999
6000 return 0;
6001}
6002#endif
6003
6004#if IS_ENABLED(CONFIG_INET)
6005static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
6006 u32 flags, bool check_mtu)
6007{
6008 struct fib_nh_common *nhc;
6009 struct in_device *in_dev;
6010 struct neighbour *neigh;
6011 struct net_device *dev;
6012 struct fib_result res;
6013 struct flowi4 fl4;
6014 u32 mtu = 0;
6015 int err;
6016
6017 dev = dev_get_by_index_rcu(net, ifindex: params->ifindex);
6018 if (unlikely(!dev))
6019 return -ENODEV;
6020
6021 /* verify forwarding is enabled on this interface */
6022 in_dev = __in_dev_get_rcu(dev);
6023 if (unlikely(!in_dev || !IN_DEV_FORWARD(in_dev)))
6024 return BPF_FIB_LKUP_RET_FWD_DISABLED;
6025
6026 if (flags & BPF_FIB_LOOKUP_OUTPUT) {
6027 fl4.flowi4_iif = 1;
6028 fl4.flowi4_oif = params->ifindex;
6029 } else {
6030 fl4.flowi4_iif = params->ifindex;
6031 fl4.flowi4_oif = 0;
6032 }
6033 fl4.flowi4_dscp = inet_dsfield_to_dscp(dsfield: params->tos);
6034 fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
6035 fl4.flowi4_flags = 0;
6036
6037 fl4.flowi4_proto = params->l4_protocol;
6038 fl4.daddr = params->ipv4_dst;
6039 fl4.saddr = params->ipv4_src;
6040 fl4.fl4_sport = params->sport;
6041 fl4.fl4_dport = params->dport;
6042 fl4.flowi4_multipath_hash = 0;
6043
6044 if (flags & BPF_FIB_LOOKUP_DIRECT) {
6045 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
6046 struct fib_table *tb;
6047
6048 if (flags & BPF_FIB_LOOKUP_TBID) {
6049 tbid = params->tbid;
6050 /* zero out for vlan output */
6051 params->tbid = 0;
6052 }
6053
6054 tb = fib_get_table(net, id: tbid);
6055 if (unlikely(!tb))
6056 return BPF_FIB_LKUP_RET_NOT_FWDED;
6057
6058 err = fib_table_lookup(tb, flp: &fl4, res: &res, FIB_LOOKUP_NOREF);
6059 } else {
6060 if (flags & BPF_FIB_LOOKUP_MARK)
6061 fl4.flowi4_mark = params->mark;
6062 else
6063 fl4.flowi4_mark = 0;
6064 fl4.flowi4_secid = 0;
6065 fl4.flowi4_tun_key.tun_id = 0;
6066 fl4.flowi4_uid = sock_net_uid(net, NULL);
6067
6068 err = fib_lookup(net, flp: &fl4, res: &res, FIB_LOOKUP_NOREF);
6069 }
6070
6071 if (err) {
6072 /* map fib lookup errors to RTN_ type */
6073 if (err == -EINVAL)
6074 return BPF_FIB_LKUP_RET_BLACKHOLE;
6075 if (err == -EHOSTUNREACH)
6076 return BPF_FIB_LKUP_RET_UNREACHABLE;
6077 if (err == -EACCES)
6078 return BPF_FIB_LKUP_RET_PROHIBIT;
6079
6080 return BPF_FIB_LKUP_RET_NOT_FWDED;
6081 }
6082
6083 if (res.type != RTN_UNICAST)
6084 return BPF_FIB_LKUP_RET_NOT_FWDED;
6085
6086 if (fib_info_num_path(fi: res.fi) > 1)
6087 fib_select_path(net, res: &res, fl4: &fl4, NULL);
6088
6089 if (check_mtu) {
6090 mtu = ip_mtu_from_fib_result(res: &res, daddr: params->ipv4_dst);
6091 if (params->tot_len > mtu) {
6092 params->mtu_result = mtu; /* union with tot_len */
6093 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
6094 }
6095 }
6096
6097 nhc = res.nhc;
6098
6099 /* do not handle lwt encaps right now */
6100 if (nhc->nhc_lwtstate)
6101 return BPF_FIB_LKUP_RET_UNSUPP_LWT;
6102
6103 dev = nhc->nhc_dev;
6104
6105 params->rt_metric = res.fi->fib_priority;
6106 params->ifindex = dev->ifindex;
6107
6108 if (flags & BPF_FIB_LOOKUP_SRC)
6109 params->ipv4_src = fib_result_prefsrc(net, res: &res);
6110
6111 /* xdp and cls_bpf programs are run in RCU-bh so
6112 * rcu_read_lock_bh is not needed here
6113 */
6114 if (likely(nhc->nhc_gw_family != AF_INET6)) {
6115 if (nhc->nhc_gw_family)
6116 params->ipv4_dst = nhc->nhc_gw.ipv4;
6117 } else {
6118 struct in6_addr *dst = (struct in6_addr *)params->ipv6_dst;
6119
6120 params->family = AF_INET6;
6121 *dst = nhc->nhc_gw.ipv6;
6122 }
6123
6124 if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH)
6125 goto set_fwd_params;
6126
6127 if (likely(nhc->nhc_gw_family != AF_INET6))
6128 neigh = __ipv4_neigh_lookup_noref(dev,
6129 key: (__force u32)params->ipv4_dst);
6130 else
6131 neigh = __ipv6_neigh_lookup_noref_stub(dev, pkey: params->ipv6_dst);
6132
6133 if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
6134 return BPF_FIB_LKUP_RET_NO_NEIGH;
6135 memcpy(to: params->dmac, from: neigh->ha, ETH_ALEN);
6136 memcpy(to: params->smac, from: dev->dev_addr, ETH_ALEN);
6137
6138set_fwd_params:
6139 return bpf_fib_set_fwd_params(params, mtu);
6140}
6141#endif
6142
6143#if IS_ENABLED(CONFIG_IPV6)
6144static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
6145 u32 flags, bool check_mtu)
6146{
6147 struct in6_addr *src = (struct in6_addr *) params->ipv6_src;
6148 struct in6_addr *dst = (struct in6_addr *) params->ipv6_dst;
6149 struct fib6_result res = {};
6150 struct neighbour *neigh;
6151 struct net_device *dev;
6152 struct inet6_dev *idev;
6153 struct flowi6 fl6;
6154 int strict = 0;
6155 int oif, err;
6156 u32 mtu = 0;
6157
6158 /* link local addresses are never forwarded */
6159 if (rt6_need_strict(daddr: dst) || rt6_need_strict(daddr: src))
6160 return BPF_FIB_LKUP_RET_NOT_FWDED;
6161
6162 dev = dev_get_by_index_rcu(net, ifindex: params->ifindex);
6163 if (unlikely(!dev))
6164 return -ENODEV;
6165
6166 idev = __in6_dev_get_safely(dev);
6167 if (unlikely(!idev || !READ_ONCE(idev->cnf.forwarding)))
6168 return BPF_FIB_LKUP_RET_FWD_DISABLED;
6169
6170 if (flags & BPF_FIB_LOOKUP_OUTPUT) {
6171 fl6.flowi6_iif = 1;
6172 oif = fl6.flowi6_oif = params->ifindex;
6173 } else {
6174 oif = fl6.flowi6_iif = params->ifindex;
6175 fl6.flowi6_oif = 0;
6176 strict = RT6_LOOKUP_F_HAS_SADDR;
6177 }
6178 fl6.flowlabel = params->flowinfo;
6179 fl6.flowi6_scope = 0;
6180 fl6.flowi6_flags = 0;
6181 fl6.mp_hash = 0;
6182
6183 fl6.flowi6_proto = params->l4_protocol;
6184 fl6.daddr = *dst;
6185 fl6.saddr = *src;
6186 fl6.fl6_sport = params->sport;
6187 fl6.fl6_dport = params->dport;
6188
6189 if (flags & BPF_FIB_LOOKUP_DIRECT) {
6190 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
6191 struct fib6_table *tb;
6192
6193 if (flags & BPF_FIB_LOOKUP_TBID) {
6194 tbid = params->tbid;
6195 /* zero out for vlan output */
6196 params->tbid = 0;
6197 }
6198
6199 tb = ipv6_stub->fib6_get_table(net, tbid);
6200 if (unlikely(!tb))
6201 return BPF_FIB_LKUP_RET_NOT_FWDED;
6202
6203 err = ipv6_stub->fib6_table_lookup(net, tb, oif, &fl6, &res,
6204 strict);
6205 } else {
6206 if (flags & BPF_FIB_LOOKUP_MARK)
6207 fl6.flowi6_mark = params->mark;
6208 else
6209 fl6.flowi6_mark = 0;
6210 fl6.flowi6_secid = 0;
6211 fl6.flowi6_tun_key.tun_id = 0;
6212 fl6.flowi6_uid = sock_net_uid(net, NULL);
6213
6214 err = ipv6_stub->fib6_lookup(net, oif, &fl6, &res, strict);
6215 }
6216
6217 if (unlikely(err || IS_ERR_OR_NULL(res.f6i) ||
6218 res.f6i == net->ipv6.fib6_null_entry))
6219 return BPF_FIB_LKUP_RET_NOT_FWDED;
6220
6221 switch (res.fib6_type) {
6222 /* only unicast is forwarded */
6223 case RTN_UNICAST:
6224 break;
6225 case RTN_BLACKHOLE:
6226 return BPF_FIB_LKUP_RET_BLACKHOLE;
6227 case RTN_UNREACHABLE:
6228 return BPF_FIB_LKUP_RET_UNREACHABLE;
6229 case RTN_PROHIBIT:
6230 return BPF_FIB_LKUP_RET_PROHIBIT;
6231 default:
6232 return BPF_FIB_LKUP_RET_NOT_FWDED;
6233 }
6234
6235 ipv6_stub->fib6_select_path(net, &res, &fl6, fl6.flowi6_oif,
6236 fl6.flowi6_oif != 0, NULL, strict);
6237
6238 if (check_mtu) {
6239 mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src);
6240 if (params->tot_len > mtu) {
6241 params->mtu_result = mtu; /* union with tot_len */
6242 return BPF_FIB_LKUP_RET_FRAG_NEEDED;
6243 }
6244 }
6245
6246 if (res.nh->fib_nh_lws)
6247 return BPF_FIB_LKUP_RET_UNSUPP_LWT;
6248
6249 if (res.nh->fib_nh_gw_family)
6250 *dst = res.nh->fib_nh_gw6;
6251
6252 dev = res.nh->fib_nh_dev;
6253 params->rt_metric = res.f6i->fib6_metric;
6254 params->ifindex = dev->ifindex;
6255
6256 if (flags & BPF_FIB_LOOKUP_SRC) {
6257 if (res.f6i->fib6_prefsrc.plen) {
6258 *src = res.f6i->fib6_prefsrc.addr;
6259 } else {
6260 err = ipv6_bpf_stub->ipv6_dev_get_saddr(net, dev,
6261 &fl6.daddr, 0,
6262 src);
6263 if (err)
6264 return BPF_FIB_LKUP_RET_NO_SRC_ADDR;
6265 }
6266 }
6267
6268 if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH)
6269 goto set_fwd_params;
6270
6271 /* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is
6272 * not needed here.
6273 */
6274 neigh = __ipv6_neigh_lookup_noref_stub(dev, pkey: dst);
6275 if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
6276 return BPF_FIB_LKUP_RET_NO_NEIGH;
6277 memcpy(to: params->dmac, from: neigh->ha, ETH_ALEN);
6278 memcpy(to: params->smac, from: dev->dev_addr, ETH_ALEN);
6279
6280set_fwd_params:
6281 return bpf_fib_set_fwd_params(params, mtu);
6282}
6283#endif
6284
6285#define BPF_FIB_LOOKUP_MASK (BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT | \
6286 BPF_FIB_LOOKUP_SKIP_NEIGH | BPF_FIB_LOOKUP_TBID | \
6287 BPF_FIB_LOOKUP_SRC | BPF_FIB_LOOKUP_MARK)
6288
6289BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx,
6290 struct bpf_fib_lookup *, params, int, plen, u32, flags)
6291{
6292 if (plen < sizeof(*params))
6293 return -EINVAL;
6294
6295 if (flags & ~BPF_FIB_LOOKUP_MASK)
6296 return -EINVAL;
6297
6298 switch (params->family) {
6299#if IS_ENABLED(CONFIG_INET)
6300 case AF_INET:
6301 return bpf_ipv4_fib_lookup(net: dev_net(dev: ctx->rxq->dev), params,
6302 flags, check_mtu: true);
6303#endif
6304#if IS_ENABLED(CONFIG_IPV6)
6305 case AF_INET6:
6306 return bpf_ipv6_fib_lookup(net: dev_net(dev: ctx->rxq->dev), params,
6307 flags, check_mtu: true);
6308#endif
6309 }
6310 return -EAFNOSUPPORT;
6311}
6312
6313static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = {
6314 .func = bpf_xdp_fib_lookup,
6315 .gpl_only = true,
6316 .ret_type = RET_INTEGER,
6317 .arg1_type = ARG_PTR_TO_CTX,
6318 .arg2_type = ARG_PTR_TO_MEM,
6319 .arg3_type = ARG_CONST_SIZE,
6320 .arg4_type = ARG_ANYTHING,
6321};
6322
6323BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
6324 struct bpf_fib_lookup *, params, int, plen, u32, flags)
6325{
6326 struct net *net = dev_net(dev: skb->dev);
6327 int rc = -EAFNOSUPPORT;
6328 bool check_mtu = false;
6329
6330 if (plen < sizeof(*params))
6331 return -EINVAL;
6332
6333 if (flags & ~BPF_FIB_LOOKUP_MASK)
6334 return -EINVAL;
6335
6336 if (params->tot_len)
6337 check_mtu = true;
6338
6339 switch (params->family) {
6340#if IS_ENABLED(CONFIG_INET)
6341 case AF_INET:
6342 rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
6343 break;
6344#endif
6345#if IS_ENABLED(CONFIG_IPV6)
6346 case AF_INET6:
6347 rc = bpf_ipv6_fib_lookup(net, params, flags, check_mtu);
6348 break;
6349#endif
6350 }
6351
6352 if (rc == BPF_FIB_LKUP_RET_SUCCESS && !check_mtu) {
6353 struct net_device *dev;
6354
6355 /* When tot_len isn't provided by user, check skb
6356 * against MTU of FIB lookup resulting net_device
6357 */
6358 dev = dev_get_by_index_rcu(net, ifindex: params->ifindex);
6359 if (!is_skb_forwardable(dev, skb))
6360 rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
6361
6362 params->mtu_result = dev->mtu; /* union with tot_len */
6363 }
6364
6365 return rc;
6366}
6367
6368static const struct bpf_func_proto bpf_skb_fib_lookup_proto = {
6369 .func = bpf_skb_fib_lookup,
6370 .gpl_only = true,
6371 .ret_type = RET_INTEGER,
6372 .arg1_type = ARG_PTR_TO_CTX,
6373 .arg2_type = ARG_PTR_TO_MEM,
6374 .arg3_type = ARG_CONST_SIZE,
6375 .arg4_type = ARG_ANYTHING,
6376};
6377
6378static struct net_device *__dev_via_ifindex(struct net_device *dev_curr,
6379 u32 ifindex)
6380{
6381 struct net *netns = dev_net(dev: dev_curr);
6382
6383 /* Non-redirect use-cases can use ifindex=0 and save ifindex lookup */
6384 if (ifindex == 0)
6385 return dev_curr;
6386
6387 return dev_get_by_index_rcu(net: netns, ifindex);
6388}
6389
6390BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb,
6391 u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
6392{
6393 int ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
6394 struct net_device *dev = skb->dev;
6395 int mtu, dev_len, skb_len;
6396
6397 if (unlikely(flags & ~(BPF_MTU_CHK_SEGS)))
6398 return -EINVAL;
6399 if (unlikely(flags & BPF_MTU_CHK_SEGS && (len_diff || *mtu_len)))
6400 return -EINVAL;
6401
6402 dev = __dev_via_ifindex(dev_curr: dev, ifindex);
6403 if (unlikely(!dev))
6404 return -ENODEV;
6405
6406 mtu = READ_ONCE(dev->mtu);
6407 dev_len = mtu + dev->hard_header_len;
6408
6409 /* If set use *mtu_len as input, L3 as iph->tot_len (like fib_lookup) */
6410 skb_len = *mtu_len ? *mtu_len + dev->hard_header_len : skb->len;
6411
6412 skb_len += len_diff; /* minus result pass check */
6413 if (skb_len <= dev_len) {
6414 ret = BPF_MTU_CHK_RET_SUCCESS;
6415 goto out;
6416 }
6417 /* At this point, skb->len exceed MTU, but as it include length of all
6418 * segments, it can still be below MTU. The SKB can possibly get
6419 * re-segmented in transmit path (see validate_xmit_skb). Thus, user
6420 * must choose if segs are to be MTU checked.
6421 */
6422 if (skb_is_gso(skb)) {
6423 ret = BPF_MTU_CHK_RET_SUCCESS;
6424 if (flags & BPF_MTU_CHK_SEGS &&
6425 !skb_gso_validate_network_len(skb, mtu))
6426 ret = BPF_MTU_CHK_RET_SEGS_TOOBIG;
6427 }
6428out:
6429 *mtu_len = mtu;
6430 return ret;
6431}
6432
6433BPF_CALL_5(bpf_xdp_check_mtu, struct xdp_buff *, xdp,
6434 u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
6435{
6436 struct net_device *dev = xdp->rxq->dev;
6437 int xdp_len = xdp->data_end - xdp->data;
6438 int ret = BPF_MTU_CHK_RET_SUCCESS;
6439 int mtu, dev_len;
6440
6441 /* XDP variant doesn't support multi-buffer segment check (yet) */
6442 if (unlikely(flags))
6443 return -EINVAL;
6444
6445 dev = __dev_via_ifindex(dev_curr: dev, ifindex);
6446 if (unlikely(!dev))
6447 return -ENODEV;
6448
6449 mtu = READ_ONCE(dev->mtu);
6450 dev_len = mtu + dev->hard_header_len;
6451
6452 /* Use *mtu_len as input, L3 as iph->tot_len (like fib_lookup) */
6453 if (*mtu_len)
6454 xdp_len = *mtu_len + dev->hard_header_len;
6455
6456 xdp_len += len_diff; /* minus result pass check */
6457 if (xdp_len > dev_len)
6458 ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
6459
6460 *mtu_len = mtu;
6461 return ret;
6462}
6463
6464static const struct bpf_func_proto bpf_skb_check_mtu_proto = {
6465 .func = bpf_skb_check_mtu,
6466 .gpl_only = true,
6467 .ret_type = RET_INTEGER,
6468 .arg1_type = ARG_PTR_TO_CTX,
6469 .arg2_type = ARG_ANYTHING,
6470 .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_WRITE | MEM_ALIGNED,
6471 .arg3_size = sizeof(u32),
6472 .arg4_type = ARG_ANYTHING,
6473 .arg5_type = ARG_ANYTHING,
6474};
6475
6476static const struct bpf_func_proto bpf_xdp_check_mtu_proto = {
6477 .func = bpf_xdp_check_mtu,
6478 .gpl_only = true,
6479 .ret_type = RET_INTEGER,
6480 .arg1_type = ARG_PTR_TO_CTX,
6481 .arg2_type = ARG_ANYTHING,
6482 .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_WRITE | MEM_ALIGNED,
6483 .arg3_size = sizeof(u32),
6484 .arg4_type = ARG_ANYTHING,
6485 .arg5_type = ARG_ANYTHING,
6486};
6487
6488#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
6489static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)
6490{
6491 int err;
6492 struct ipv6_sr_hdr *srh = (struct ipv6_sr_hdr *)hdr;
6493
6494 if (!seg6_validate_srh(srh, len, false))
6495 return -EINVAL;
6496
6497 switch (type) {
6498 case BPF_LWT_ENCAP_SEG6_INLINE:
6499 if (skb->protocol != htons(ETH_P_IPV6))
6500 return -EBADMSG;
6501
6502 err = seg6_do_srh_inline(skb, srh);
6503 break;
6504 case BPF_LWT_ENCAP_SEG6:
6505 skb_reset_inner_headers(skb);
6506 skb->encapsulation = 1;
6507 err = seg6_do_srh_encap(skb, srh, IPPROTO_IPV6);
6508 break;
6509 default:
6510 return -EINVAL;
6511 }
6512
6513 bpf_compute_data_pointers(skb);
6514 if (err)
6515 return err;
6516
6517 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
6518
6519 return seg6_lookup_nexthop(skb, NULL, 0);
6520}
6521#endif /* CONFIG_IPV6_SEG6_BPF */
6522
6523#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
6524static int bpf_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len,
6525 bool ingress)
6526{
6527 return bpf_lwt_push_ip_encap(skb, hdr, len, ingress);
6528}
6529#endif
6530
6531BPF_CALL_4(bpf_lwt_in_push_encap, struct sk_buff *, skb, u32, type, void *, hdr,
6532 u32, len)
6533{
6534 switch (type) {
6535#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
6536 case BPF_LWT_ENCAP_SEG6:
6537 case BPF_LWT_ENCAP_SEG6_INLINE:
6538 return bpf_push_seg6_encap(skb, type, hdr, len);
6539#endif
6540#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
6541 case BPF_LWT_ENCAP_IP:
6542 return bpf_push_ip_encap(skb, hdr, len, true /* ingress */);
6543#endif
6544 default:
6545 return -EINVAL;
6546 }
6547}
6548
6549BPF_CALL_4(bpf_lwt_xmit_push_encap, struct sk_buff *, skb, u32, type,
6550 void *, hdr, u32, len)
6551{
6552 switch (type) {
6553#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
6554 case BPF_LWT_ENCAP_IP:
6555 return bpf_push_ip_encap(skb, hdr, len, false /* egress */);
6556#endif
6557 default:
6558 return -EINVAL;
6559 }
6560}
6561
6562static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = {
6563 .func = bpf_lwt_in_push_encap,
6564 .gpl_only = false,
6565 .ret_type = RET_INTEGER,
6566 .arg1_type = ARG_PTR_TO_CTX,
6567 .arg2_type = ARG_ANYTHING,
6568 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6569 .arg4_type = ARG_CONST_SIZE
6570};
6571
6572static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = {
6573 .func = bpf_lwt_xmit_push_encap,
6574 .gpl_only = false,
6575 .ret_type = RET_INTEGER,
6576 .arg1_type = ARG_PTR_TO_CTX,
6577 .arg2_type = ARG_ANYTHING,
6578 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6579 .arg4_type = ARG_CONST_SIZE
6580};
6581
6582#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
6583BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
6584 const void *, from, u32, len)
6585{
6586 struct seg6_bpf_srh_state *srh_state =
6587 this_cpu_ptr(&seg6_bpf_srh_states);
6588 struct ipv6_sr_hdr *srh = srh_state->srh;
6589 void *srh_tlvs, *srh_end, *ptr;
6590 int srhoff = 0;
6591
6592 lockdep_assert_held(&srh_state->bh_lock);
6593 if (srh == NULL)
6594 return -EINVAL;
6595
6596 srh_tlvs = (void *)((char *)srh + ((srh->first_segment + 1) << 4));
6597 srh_end = (void *)((char *)srh + sizeof(*srh) + srh_state->hdrlen);
6598
6599 ptr = skb->data + offset;
6600 if (ptr >= srh_tlvs && ptr + len <= srh_end)
6601 srh_state->valid = false;
6602 else if (ptr < (void *)&srh->flags ||
6603 ptr + len > (void *)&srh->segments)
6604 return -EFAULT;
6605
6606 if (unlikely(bpf_try_make_writable(skb, offset + len)))
6607 return -EFAULT;
6608 if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
6609 return -EINVAL;
6610 srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
6611
6612 memcpy(skb->data + offset, from, len);
6613 return 0;
6614}
6615
6616static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = {
6617 .func = bpf_lwt_seg6_store_bytes,
6618 .gpl_only = false,
6619 .ret_type = RET_INTEGER,
6620 .arg1_type = ARG_PTR_TO_CTX,
6621 .arg2_type = ARG_ANYTHING,
6622 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6623 .arg4_type = ARG_CONST_SIZE
6624};
6625
6626static void bpf_update_srh_state(struct sk_buff *skb)
6627{
6628 struct seg6_bpf_srh_state *srh_state =
6629 this_cpu_ptr(&seg6_bpf_srh_states);
6630 int srhoff = 0;
6631
6632 if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) {
6633 srh_state->srh = NULL;
6634 } else {
6635 srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
6636 srh_state->hdrlen = srh_state->srh->hdrlen << 3;
6637 srh_state->valid = true;
6638 }
6639}
6640
6641BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb,
6642 u32, action, void *, param, u32, param_len)
6643{
6644 struct seg6_bpf_srh_state *srh_state =
6645 this_cpu_ptr(&seg6_bpf_srh_states);
6646 int hdroff = 0;
6647 int err;
6648
6649 lockdep_assert_held(&srh_state->bh_lock);
6650 switch (action) {
6651 case SEG6_LOCAL_ACTION_END_X:
6652 if (!seg6_bpf_has_valid_srh(skb))
6653 return -EBADMSG;
6654 if (param_len != sizeof(struct in6_addr))
6655 return -EINVAL;
6656 return seg6_lookup_nexthop(skb, (struct in6_addr *)param, 0);
6657 case SEG6_LOCAL_ACTION_END_T:
6658 if (!seg6_bpf_has_valid_srh(skb))
6659 return -EBADMSG;
6660 if (param_len != sizeof(int))
6661 return -EINVAL;
6662 return seg6_lookup_nexthop(skb, NULL, *(int *)param);
6663 case SEG6_LOCAL_ACTION_END_DT6:
6664 if (!seg6_bpf_has_valid_srh(skb))
6665 return -EBADMSG;
6666 if (param_len != sizeof(int))
6667 return -EINVAL;
6668
6669 if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0)
6670 return -EBADMSG;
6671 if (!pskb_pull(skb, hdroff))
6672 return -EBADMSG;
6673
6674 skb_postpull_rcsum(skb, skb_network_header(skb), hdroff);
6675 skb_reset_network_header(skb);
6676 skb_reset_transport_header(skb);
6677 skb->encapsulation = 0;
6678
6679 bpf_compute_data_pointers(skb);
6680 bpf_update_srh_state(skb);
6681 return seg6_lookup_nexthop(skb, NULL, *(int *)param);
6682 case SEG6_LOCAL_ACTION_END_B6:
6683 if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
6684 return -EBADMSG;
6685 err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE,
6686 param, param_len);
6687 if (!err)
6688 bpf_update_srh_state(skb);
6689
6690 return err;
6691 case SEG6_LOCAL_ACTION_END_B6_ENCAP:
6692 if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
6693 return -EBADMSG;
6694 err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6,
6695 param, param_len);
6696 if (!err)
6697 bpf_update_srh_state(skb);
6698
6699 return err;
6700 default:
6701 return -EINVAL;
6702 }
6703}
6704
6705static const struct bpf_func_proto bpf_lwt_seg6_action_proto = {
6706 .func = bpf_lwt_seg6_action,
6707 .gpl_only = false,
6708 .ret_type = RET_INTEGER,
6709 .arg1_type = ARG_PTR_TO_CTX,
6710 .arg2_type = ARG_ANYTHING,
6711 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6712 .arg4_type = ARG_CONST_SIZE
6713};
6714
6715BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
6716 s32, len)
6717{
6718 struct seg6_bpf_srh_state *srh_state =
6719 this_cpu_ptr(&seg6_bpf_srh_states);
6720 struct ipv6_sr_hdr *srh = srh_state->srh;
6721 void *srh_end, *srh_tlvs, *ptr;
6722 struct ipv6hdr *hdr;
6723 int srhoff = 0;
6724 int ret;
6725
6726 lockdep_assert_held(&srh_state->bh_lock);
6727 if (unlikely(srh == NULL))
6728 return -EINVAL;
6729
6730 srh_tlvs = (void *)((unsigned char *)srh + sizeof(*srh) +
6731 ((srh->first_segment + 1) << 4));
6732 srh_end = (void *)((unsigned char *)srh + sizeof(*srh) +
6733 srh_state->hdrlen);
6734 ptr = skb->data + offset;
6735
6736 if (unlikely(ptr < srh_tlvs || ptr > srh_end))
6737 return -EFAULT;
6738 if (unlikely(len < 0 && (void *)((char *)ptr - len) > srh_end))
6739 return -EFAULT;
6740
6741 if (len > 0) {
6742 ret = skb_cow_head(skb, len);
6743 if (unlikely(ret < 0))
6744 return ret;
6745
6746 ret = bpf_skb_net_hdr_push(skb, offset, len);
6747 } else {
6748 ret = bpf_skb_net_hdr_pop(skb, offset, -1 * len);
6749 }
6750
6751 bpf_compute_data_pointers(skb);
6752 if (unlikely(ret < 0))
6753 return ret;
6754
6755 hdr = (struct ipv6hdr *)skb->data;
6756 hdr->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
6757
6758 if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
6759 return -EINVAL;
6760 srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
6761 srh_state->hdrlen += len;
6762 srh_state->valid = false;
6763 return 0;
6764}
6765
6766static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
6767 .func = bpf_lwt_seg6_adjust_srh,
6768 .gpl_only = false,
6769 .ret_type = RET_INTEGER,
6770 .arg1_type = ARG_PTR_TO_CTX,
6771 .arg2_type = ARG_ANYTHING,
6772 .arg3_type = ARG_ANYTHING,
6773};
6774#endif /* CONFIG_IPV6_SEG6_BPF */
6775
6776#ifdef CONFIG_INET
6777static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
6778 int dif, int sdif, u8 family, u8 proto)
6779{
6780 bool refcounted = false;
6781 struct sock *sk = NULL;
6782
6783 if (family == AF_INET) {
6784 __be32 src4 = tuple->ipv4.saddr;
6785 __be32 dst4 = tuple->ipv4.daddr;
6786
6787 if (proto == IPPROTO_TCP)
6788 sk = __inet_lookup(net, NULL, doff: 0,
6789 saddr: src4, sport: tuple->ipv4.sport,
6790 daddr: dst4, dport: tuple->ipv4.dport,
6791 dif, sdif, refcounted: &refcounted);
6792 else
6793 sk = __udp4_lib_lookup(net, saddr: src4, sport: tuple->ipv4.sport,
6794 daddr: dst4, dport: tuple->ipv4.dport,
6795 dif, sdif, tbl: net->ipv4.udp_table, NULL);
6796#if IS_ENABLED(CONFIG_IPV6)
6797 } else {
6798 struct in6_addr *src6 = (struct in6_addr *)&tuple->ipv6.saddr;
6799 struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr;
6800
6801 if (proto == IPPROTO_TCP)
6802 sk = __inet6_lookup(net, NULL, doff: 0,
6803 saddr: src6, sport: tuple->ipv6.sport,
6804 daddr: dst6, ntohs(tuple->ipv6.dport),
6805 dif, sdif, refcounted: &refcounted);
6806 else if (likely(ipv6_bpf_stub))
6807 sk = ipv6_bpf_stub->udp6_lib_lookup(net,
6808 src6, tuple->ipv6.sport,
6809 dst6, tuple->ipv6.dport,
6810 dif, sdif,
6811 net->ipv4.udp_table, NULL);
6812#endif
6813 }
6814
6815 if (unlikely(sk && !refcounted && !sock_flag(sk, SOCK_RCU_FREE))) {
6816 WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
6817 sk = NULL;
6818 }
6819 return sk;
6820}
6821
6822/* bpf_skc_lookup performs the core lookup for different types of sockets,
6823 * taking a reference on the socket if it doesn't have the flag SOCK_RCU_FREE.
6824 */
6825static struct sock *
6826__bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6827 struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
6828 u64 flags, int sdif)
6829{
6830 struct sock *sk = NULL;
6831 struct net *net;
6832 u8 family;
6833
6834 if (len == sizeof(tuple->ipv4))
6835 family = AF_INET;
6836 else if (len == sizeof(tuple->ipv6))
6837 family = AF_INET6;
6838 else
6839 return NULL;
6840
6841 if (unlikely(flags || !((s32)netns_id < 0 || netns_id <= S32_MAX)))
6842 goto out;
6843
6844 if (sdif < 0) {
6845 if (family == AF_INET)
6846 sdif = inet_sdif(skb);
6847 else
6848 sdif = inet6_sdif(skb);
6849 }
6850
6851 if ((s32)netns_id < 0) {
6852 net = caller_net;
6853 sk = sk_lookup(net, tuple, dif: ifindex, sdif, family, proto);
6854 } else {
6855 net = get_net_ns_by_id(net: caller_net, id: netns_id);
6856 if (unlikely(!net))
6857 goto out;
6858 sk = sk_lookup(net, tuple, dif: ifindex, sdif, family, proto);
6859 put_net(net);
6860 }
6861
6862out:
6863 return sk;
6864}
6865
6866static struct sock *
6867__bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6868 struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
6869 u64 flags, int sdif)
6870{
6871 struct sock *sk = __bpf_skc_lookup(skb, tuple, len, caller_net,
6872 ifindex, proto, netns_id, flags,
6873 sdif);
6874
6875 if (sk) {
6876 struct sock *sk2 = sk_to_full_sk(sk);
6877
6878 /* sk_to_full_sk() may return (sk)->rsk_listener, so make sure the original sk
6879 * sock refcnt is decremented to prevent a request_sock leak.
6880 */
6881 if (sk2 != sk) {
6882 sock_gen_put(sk);
6883 /* Ensure there is no need to bump sk2 refcnt */
6884 if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) {
6885 WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
6886 return NULL;
6887 }
6888 sk = sk2;
6889 }
6890 }
6891
6892 return sk;
6893}
6894
6895static struct sock *
6896bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6897 u8 proto, u64 netns_id, u64 flags)
6898{
6899 struct net *caller_net;
6900 int ifindex;
6901
6902 if (skb->dev) {
6903 caller_net = dev_net(dev: skb->dev);
6904 ifindex = skb->dev->ifindex;
6905 } else {
6906 caller_net = sock_net(sk: skb->sk);
6907 ifindex = 0;
6908 }
6909
6910 return __bpf_skc_lookup(skb, tuple, len, caller_net, ifindex, proto,
6911 netns_id, flags, sdif: -1);
6912}
6913
6914static struct sock *
6915bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
6916 u8 proto, u64 netns_id, u64 flags)
6917{
6918 struct sock *sk = bpf_skc_lookup(skb, tuple, len, proto, netns_id,
6919 flags);
6920
6921 if (sk) {
6922 struct sock *sk2 = sk_to_full_sk(sk);
6923
6924 /* sk_to_full_sk() may return (sk)->rsk_listener, so make sure the original sk
6925 * sock refcnt is decremented to prevent a request_sock leak.
6926 */
6927 if (sk2 != sk) {
6928 sock_gen_put(sk);
6929 /* Ensure there is no need to bump sk2 refcnt */
6930 if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) {
6931 WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
6932 return NULL;
6933 }
6934 sk = sk2;
6935 }
6936 }
6937
6938 return sk;
6939}
6940
6941BPF_CALL_5(bpf_skc_lookup_tcp, struct sk_buff *, skb,
6942 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6943{
6944 return (unsigned long)bpf_skc_lookup(skb, tuple, len, IPPROTO_TCP,
6945 netns_id, flags);
6946}
6947
6948static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = {
6949 .func = bpf_skc_lookup_tcp,
6950 .gpl_only = false,
6951 .pkt_access = true,
6952 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
6953 .arg1_type = ARG_PTR_TO_CTX,
6954 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6955 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
6956 .arg4_type = ARG_ANYTHING,
6957 .arg5_type = ARG_ANYTHING,
6958};
6959
6960BPF_CALL_5(bpf_sk_lookup_tcp, struct sk_buff *, skb,
6961 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6962{
6963 return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_TCP,
6964 netns_id, flags);
6965}
6966
6967static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = {
6968 .func = bpf_sk_lookup_tcp,
6969 .gpl_only = false,
6970 .pkt_access = true,
6971 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6972 .arg1_type = ARG_PTR_TO_CTX,
6973 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6974 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
6975 .arg4_type = ARG_ANYTHING,
6976 .arg5_type = ARG_ANYTHING,
6977};
6978
6979BPF_CALL_5(bpf_sk_lookup_udp, struct sk_buff *, skb,
6980 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
6981{
6982 return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_UDP,
6983 netns_id, flags);
6984}
6985
6986static const struct bpf_func_proto bpf_sk_lookup_udp_proto = {
6987 .func = bpf_sk_lookup_udp,
6988 .gpl_only = false,
6989 .pkt_access = true,
6990 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
6991 .arg1_type = ARG_PTR_TO_CTX,
6992 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
6993 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
6994 .arg4_type = ARG_ANYTHING,
6995 .arg5_type = ARG_ANYTHING,
6996};
6997
6998BPF_CALL_5(bpf_tc_skc_lookup_tcp, struct sk_buff *, skb,
6999 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7000{
7001 struct net_device *dev = skb->dev;
7002 int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7003 struct net *caller_net = dev_net(dev);
7004
7005 return (unsigned long)__bpf_skc_lookup(skb, tuple, len, caller_net,
7006 ifindex, IPPROTO_TCP, netns_id,
7007 flags, sdif);
7008}
7009
7010static const struct bpf_func_proto bpf_tc_skc_lookup_tcp_proto = {
7011 .func = bpf_tc_skc_lookup_tcp,
7012 .gpl_only = false,
7013 .pkt_access = true,
7014 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
7015 .arg1_type = ARG_PTR_TO_CTX,
7016 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7017 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7018 .arg4_type = ARG_ANYTHING,
7019 .arg5_type = ARG_ANYTHING,
7020};
7021
7022BPF_CALL_5(bpf_tc_sk_lookup_tcp, struct sk_buff *, skb,
7023 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7024{
7025 struct net_device *dev = skb->dev;
7026 int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7027 struct net *caller_net = dev_net(dev);
7028
7029 return (unsigned long)__bpf_sk_lookup(skb, tuple, len, caller_net,
7030 ifindex, IPPROTO_TCP, netns_id,
7031 flags, sdif);
7032}
7033
7034static const struct bpf_func_proto bpf_tc_sk_lookup_tcp_proto = {
7035 .func = bpf_tc_sk_lookup_tcp,
7036 .gpl_only = false,
7037 .pkt_access = true,
7038 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7039 .arg1_type = ARG_PTR_TO_CTX,
7040 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7041 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7042 .arg4_type = ARG_ANYTHING,
7043 .arg5_type = ARG_ANYTHING,
7044};
7045
7046BPF_CALL_5(bpf_tc_sk_lookup_udp, struct sk_buff *, skb,
7047 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7048{
7049 struct net_device *dev = skb->dev;
7050 int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7051 struct net *caller_net = dev_net(dev);
7052
7053 return (unsigned long)__bpf_sk_lookup(skb, tuple, len, caller_net,
7054 ifindex, IPPROTO_UDP, netns_id,
7055 flags, sdif);
7056}
7057
7058static const struct bpf_func_proto bpf_tc_sk_lookup_udp_proto = {
7059 .func = bpf_tc_sk_lookup_udp,
7060 .gpl_only = false,
7061 .pkt_access = true,
7062 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7063 .arg1_type = ARG_PTR_TO_CTX,
7064 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7065 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7066 .arg4_type = ARG_ANYTHING,
7067 .arg5_type = ARG_ANYTHING,
7068};
7069
7070BPF_CALL_1(bpf_sk_release, struct sock *, sk)
7071{
7072 if (sk && sk_is_refcounted(sk))
7073 sock_gen_put(sk);
7074 return 0;
7075}
7076
7077static const struct bpf_func_proto bpf_sk_release_proto = {
7078 .func = bpf_sk_release,
7079 .gpl_only = false,
7080 .ret_type = RET_INTEGER,
7081 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON | OBJ_RELEASE,
7082};
7083
7084BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx,
7085 struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
7086{
7087 struct net_device *dev = ctx->rxq->dev;
7088 int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7089 struct net *caller_net = dev_net(dev);
7090
7091 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
7092 ifindex, IPPROTO_UDP, netns_id,
7093 flags, sdif);
7094}
7095
7096static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = {
7097 .func = bpf_xdp_sk_lookup_udp,
7098 .gpl_only = false,
7099 .pkt_access = true,
7100 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7101 .arg1_type = ARG_PTR_TO_CTX,
7102 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7103 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7104 .arg4_type = ARG_ANYTHING,
7105 .arg5_type = ARG_ANYTHING,
7106};
7107
7108BPF_CALL_5(bpf_xdp_skc_lookup_tcp, struct xdp_buff *, ctx,
7109 struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
7110{
7111 struct net_device *dev = ctx->rxq->dev;
7112 int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7113 struct net *caller_net = dev_net(dev);
7114
7115 return (unsigned long)__bpf_skc_lookup(NULL, tuple, len, caller_net,
7116 ifindex, IPPROTO_TCP, netns_id,
7117 flags, sdif);
7118}
7119
7120static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = {
7121 .func = bpf_xdp_skc_lookup_tcp,
7122 .gpl_only = false,
7123 .pkt_access = true,
7124 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
7125 .arg1_type = ARG_PTR_TO_CTX,
7126 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7127 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7128 .arg4_type = ARG_ANYTHING,
7129 .arg5_type = ARG_ANYTHING,
7130};
7131
7132BPF_CALL_5(bpf_xdp_sk_lookup_tcp, struct xdp_buff *, ctx,
7133 struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
7134{
7135 struct net_device *dev = ctx->rxq->dev;
7136 int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7137 struct net *caller_net = dev_net(dev);
7138
7139 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
7140 ifindex, IPPROTO_TCP, netns_id,
7141 flags, sdif);
7142}
7143
7144static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = {
7145 .func = bpf_xdp_sk_lookup_tcp,
7146 .gpl_only = false,
7147 .pkt_access = true,
7148 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7149 .arg1_type = ARG_PTR_TO_CTX,
7150 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7151 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7152 .arg4_type = ARG_ANYTHING,
7153 .arg5_type = ARG_ANYTHING,
7154};
7155
7156BPF_CALL_5(bpf_sock_addr_skc_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
7157 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7158{
7159 return (unsigned long)__bpf_skc_lookup(NULL, tuple, len,
7160 caller_net: sock_net(sk: ctx->sk), ifindex: 0,
7161 IPPROTO_TCP, netns_id, flags,
7162 sdif: -1);
7163}
7164
7165static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = {
7166 .func = bpf_sock_addr_skc_lookup_tcp,
7167 .gpl_only = false,
7168 .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL,
7169 .arg1_type = ARG_PTR_TO_CTX,
7170 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7171 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7172 .arg4_type = ARG_ANYTHING,
7173 .arg5_type = ARG_ANYTHING,
7174};
7175
7176BPF_CALL_5(bpf_sock_addr_sk_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
7177 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7178{
7179 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
7180 caller_net: sock_net(sk: ctx->sk), ifindex: 0, IPPROTO_TCP,
7181 netns_id, flags, sdif: -1);
7182}
7183
7184static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = {
7185 .func = bpf_sock_addr_sk_lookup_tcp,
7186 .gpl_only = false,
7187 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7188 .arg1_type = ARG_PTR_TO_CTX,
7189 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7190 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7191 .arg4_type = ARG_ANYTHING,
7192 .arg5_type = ARG_ANYTHING,
7193};
7194
7195BPF_CALL_5(bpf_sock_addr_sk_lookup_udp, struct bpf_sock_addr_kern *, ctx,
7196 struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7197{
7198 return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
7199 caller_net: sock_net(sk: ctx->sk), ifindex: 0, IPPROTO_UDP,
7200 netns_id, flags, sdif: -1);
7201}
7202
7203static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = {
7204 .func = bpf_sock_addr_sk_lookup_udp,
7205 .gpl_only = false,
7206 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7207 .arg1_type = ARG_PTR_TO_CTX,
7208 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7209 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7210 .arg4_type = ARG_ANYTHING,
7211 .arg5_type = ARG_ANYTHING,
7212};
7213
7214bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
7215 struct bpf_insn_access_aux *info)
7216{
7217 if (off < 0 || off >= offsetofend(struct bpf_tcp_sock,
7218 icsk_retransmits))
7219 return false;
7220
7221 if (off % size != 0)
7222 return false;
7223
7224 switch (off) {
7225 case offsetof(struct bpf_tcp_sock, bytes_received):
7226 case offsetof(struct bpf_tcp_sock, bytes_acked):
7227 return size == sizeof(__u64);
7228 default:
7229 return size == sizeof(__u32);
7230 }
7231}
7232
7233u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
7234 const struct bpf_insn *si,
7235 struct bpf_insn *insn_buf,
7236 struct bpf_prog *prog, u32 *target_size)
7237{
7238 struct bpf_insn *insn = insn_buf;
7239
7240#define BPF_TCP_SOCK_GET_COMMON(FIELD) \
7241 do { \
7242 BUILD_BUG_ON(sizeof_field(struct tcp_sock, FIELD) > \
7243 sizeof_field(struct bpf_tcp_sock, FIELD)); \
7244 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_sock, FIELD),\
7245 si->dst_reg, si->src_reg, \
7246 offsetof(struct tcp_sock, FIELD)); \
7247 } while (0)
7248
7249#define BPF_INET_SOCK_GET_COMMON(FIELD) \
7250 do { \
7251 BUILD_BUG_ON(sizeof_field(struct inet_connection_sock, \
7252 FIELD) > \
7253 sizeof_field(struct bpf_tcp_sock, FIELD)); \
7254 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
7255 struct inet_connection_sock, \
7256 FIELD), \
7257 si->dst_reg, si->src_reg, \
7258 offsetof( \
7259 struct inet_connection_sock, \
7260 FIELD)); \
7261 } while (0)
7262
7263 BTF_TYPE_EMIT(struct bpf_tcp_sock);
7264
7265 switch (si->off) {
7266 case offsetof(struct bpf_tcp_sock, rtt_min):
7267 BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
7268 sizeof(struct minmax));
7269 BUILD_BUG_ON(sizeof(struct minmax) <
7270 sizeof(struct minmax_sample));
7271
7272 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
7273 offsetof(struct tcp_sock, rtt_min) +
7274 offsetof(struct minmax_sample, v));
7275 break;
7276 case offsetof(struct bpf_tcp_sock, snd_cwnd):
7277 BPF_TCP_SOCK_GET_COMMON(snd_cwnd);
7278 break;
7279 case offsetof(struct bpf_tcp_sock, srtt_us):
7280 BPF_TCP_SOCK_GET_COMMON(srtt_us);
7281 break;
7282 case offsetof(struct bpf_tcp_sock, snd_ssthresh):
7283 BPF_TCP_SOCK_GET_COMMON(snd_ssthresh);
7284 break;
7285 case offsetof(struct bpf_tcp_sock, rcv_nxt):
7286 BPF_TCP_SOCK_GET_COMMON(rcv_nxt);
7287 break;
7288 case offsetof(struct bpf_tcp_sock, snd_nxt):
7289 BPF_TCP_SOCK_GET_COMMON(snd_nxt);
7290 break;
7291 case offsetof(struct bpf_tcp_sock, snd_una):
7292 BPF_TCP_SOCK_GET_COMMON(snd_una);
7293 break;
7294 case offsetof(struct bpf_tcp_sock, mss_cache):
7295 BPF_TCP_SOCK_GET_COMMON(mss_cache);
7296 break;
7297 case offsetof(struct bpf_tcp_sock, ecn_flags):
7298 BPF_TCP_SOCK_GET_COMMON(ecn_flags);
7299 break;
7300 case offsetof(struct bpf_tcp_sock, rate_delivered):
7301 BPF_TCP_SOCK_GET_COMMON(rate_delivered);
7302 break;
7303 case offsetof(struct bpf_tcp_sock, rate_interval_us):
7304 BPF_TCP_SOCK_GET_COMMON(rate_interval_us);
7305 break;
7306 case offsetof(struct bpf_tcp_sock, packets_out):
7307 BPF_TCP_SOCK_GET_COMMON(packets_out);
7308 break;
7309 case offsetof(struct bpf_tcp_sock, retrans_out):
7310 BPF_TCP_SOCK_GET_COMMON(retrans_out);
7311 break;
7312 case offsetof(struct bpf_tcp_sock, total_retrans):
7313 BPF_TCP_SOCK_GET_COMMON(total_retrans);
7314 break;
7315 case offsetof(struct bpf_tcp_sock, segs_in):
7316 BPF_TCP_SOCK_GET_COMMON(segs_in);
7317 break;
7318 case offsetof(struct bpf_tcp_sock, data_segs_in):
7319 BPF_TCP_SOCK_GET_COMMON(data_segs_in);
7320 break;
7321 case offsetof(struct bpf_tcp_sock, segs_out):
7322 BPF_TCP_SOCK_GET_COMMON(segs_out);
7323 break;
7324 case offsetof(struct bpf_tcp_sock, data_segs_out):
7325 BPF_TCP_SOCK_GET_COMMON(data_segs_out);
7326 break;
7327 case offsetof(struct bpf_tcp_sock, lost_out):
7328 BPF_TCP_SOCK_GET_COMMON(lost_out);
7329 break;
7330 case offsetof(struct bpf_tcp_sock, sacked_out):
7331 BPF_TCP_SOCK_GET_COMMON(sacked_out);
7332 break;
7333 case offsetof(struct bpf_tcp_sock, bytes_received):
7334 BPF_TCP_SOCK_GET_COMMON(bytes_received);
7335 break;
7336 case offsetof(struct bpf_tcp_sock, bytes_acked):
7337 BPF_TCP_SOCK_GET_COMMON(bytes_acked);
7338 break;
7339 case offsetof(struct bpf_tcp_sock, dsack_dups):
7340 BPF_TCP_SOCK_GET_COMMON(dsack_dups);
7341 break;
7342 case offsetof(struct bpf_tcp_sock, delivered):
7343 BPF_TCP_SOCK_GET_COMMON(delivered);
7344 break;
7345 case offsetof(struct bpf_tcp_sock, delivered_ce):
7346 BPF_TCP_SOCK_GET_COMMON(delivered_ce);
7347 break;
7348 case offsetof(struct bpf_tcp_sock, icsk_retransmits):
7349 BPF_INET_SOCK_GET_COMMON(icsk_retransmits);
7350 break;
7351 }
7352
7353 return insn - insn_buf;
7354}
7355
7356BPF_CALL_1(bpf_tcp_sock, struct sock *, sk)
7357{
7358 if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
7359 return (unsigned long)sk;
7360
7361 return (unsigned long)NULL;
7362}
7363
7364const struct bpf_func_proto bpf_tcp_sock_proto = {
7365 .func = bpf_tcp_sock,
7366 .gpl_only = false,
7367 .ret_type = RET_PTR_TO_TCP_SOCK_OR_NULL,
7368 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
7369};
7370
7371BPF_CALL_1(bpf_get_listener_sock, struct sock *, sk)
7372{
7373 sk = sk_to_full_sk(sk);
7374
7375 if (sk && sk->sk_state == TCP_LISTEN && sock_flag(sk, flag: SOCK_RCU_FREE))
7376 return (unsigned long)sk;
7377
7378 return (unsigned long)NULL;
7379}
7380
7381static const struct bpf_func_proto bpf_get_listener_sock_proto = {
7382 .func = bpf_get_listener_sock,
7383 .gpl_only = false,
7384 .ret_type = RET_PTR_TO_SOCKET_OR_NULL,
7385 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
7386};
7387
7388BPF_CALL_1(bpf_skb_ecn_set_ce, struct sk_buff *, skb)
7389{
7390 unsigned int iphdr_len;
7391
7392 switch (skb_protocol(skb, skip_vlan: true)) {
7393 case cpu_to_be16(ETH_P_IP):
7394 iphdr_len = sizeof(struct iphdr);
7395 break;
7396 case cpu_to_be16(ETH_P_IPV6):
7397 iphdr_len = sizeof(struct ipv6hdr);
7398 break;
7399 default:
7400 return 0;
7401 }
7402
7403 if (skb_headlen(skb) < iphdr_len)
7404 return 0;
7405
7406 if (skb_cloned(skb) && !skb_clone_writable(skb, len: iphdr_len))
7407 return 0;
7408
7409 return INET_ECN_set_ce(skb);
7410}
7411
7412bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
7413 struct bpf_insn_access_aux *info)
7414{
7415 if (off < 0 || off >= offsetofend(struct bpf_xdp_sock, queue_id))
7416 return false;
7417
7418 if (off % size != 0)
7419 return false;
7420
7421 switch (off) {
7422 default:
7423 return size == sizeof(__u32);
7424 }
7425}
7426
7427u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
7428 const struct bpf_insn *si,
7429 struct bpf_insn *insn_buf,
7430 struct bpf_prog *prog, u32 *target_size)
7431{
7432 struct bpf_insn *insn = insn_buf;
7433
7434#define BPF_XDP_SOCK_GET(FIELD) \
7435 do { \
7436 BUILD_BUG_ON(sizeof_field(struct xdp_sock, FIELD) > \
7437 sizeof_field(struct bpf_xdp_sock, FIELD)); \
7438 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_sock, FIELD),\
7439 si->dst_reg, si->src_reg, \
7440 offsetof(struct xdp_sock, FIELD)); \
7441 } while (0)
7442
7443 BTF_TYPE_EMIT(struct bpf_xdp_sock);
7444
7445 switch (si->off) {
7446 case offsetof(struct bpf_xdp_sock, queue_id):
7447 BPF_XDP_SOCK_GET(queue_id);
7448 break;
7449 }
7450
7451 return insn - insn_buf;
7452}
7453
7454static const struct bpf_func_proto bpf_skb_ecn_set_ce_proto = {
7455 .func = bpf_skb_ecn_set_ce,
7456 .gpl_only = false,
7457 .ret_type = RET_INTEGER,
7458 .arg1_type = ARG_PTR_TO_CTX,
7459};
7460
7461BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
7462 struct tcphdr *, th, u32, th_len)
7463{
7464#ifdef CONFIG_SYN_COOKIES
7465 int ret;
7466
7467 if (unlikely(!sk || th_len < sizeof(*th)))
7468 return -EINVAL;
7469
7470 /* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */
7471 if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
7472 return -EINVAL;
7473
7474 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies))
7475 return -EINVAL;
7476
7477 if (!th->ack || th->rst || th->syn)
7478 return -ENOENT;
7479
7480 if (unlikely(iph_len < sizeof(struct iphdr)))
7481 return -EINVAL;
7482
7483 if (tcp_synq_no_recent_overflow(sk))
7484 return -ENOENT;
7485
7486 /* Both struct iphdr and struct ipv6hdr have the version field at the
7487 * same offset so we can cast to the shorter header (struct iphdr).
7488 */
7489 switch (((struct iphdr *)iph)->version) {
7490 case 4:
7491 if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
7492 return -EINVAL;
7493
7494 ret = __cookie_v4_check(iph: (struct iphdr *)iph, th);
7495 break;
7496
7497#if IS_BUILTIN(CONFIG_IPV6)
7498 case 6:
7499 if (unlikely(iph_len < sizeof(struct ipv6hdr)))
7500 return -EINVAL;
7501
7502 if (sk->sk_family != AF_INET6)
7503 return -EINVAL;
7504
7505 ret = __cookie_v6_check(iph: (struct ipv6hdr *)iph, th);
7506 break;
7507#endif /* CONFIG_IPV6 */
7508
7509 default:
7510 return -EPROTONOSUPPORT;
7511 }
7512
7513 if (ret > 0)
7514 return 0;
7515
7516 return -ENOENT;
7517#else
7518 return -ENOTSUPP;
7519#endif
7520}
7521
7522static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = {
7523 .func = bpf_tcp_check_syncookie,
7524 .gpl_only = true,
7525 .pkt_access = true,
7526 .ret_type = RET_INTEGER,
7527 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
7528 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7529 .arg3_type = ARG_CONST_SIZE,
7530 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7531 .arg5_type = ARG_CONST_SIZE,
7532};
7533
7534BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
7535 struct tcphdr *, th, u32, th_len)
7536{
7537#ifdef CONFIG_SYN_COOKIES
7538 u32 cookie;
7539 u16 mss;
7540
7541 if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4))
7542 return -EINVAL;
7543
7544 if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
7545 return -EINVAL;
7546
7547 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies))
7548 return -ENOENT;
7549
7550 if (!th->syn || th->ack || th->fin || th->rst)
7551 return -EINVAL;
7552
7553 if (unlikely(iph_len < sizeof(struct iphdr)))
7554 return -EINVAL;
7555
7556 /* Both struct iphdr and struct ipv6hdr have the version field at the
7557 * same offset so we can cast to the shorter header (struct iphdr).
7558 */
7559 switch (((struct iphdr *)iph)->version) {
7560 case 4:
7561 if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
7562 return -EINVAL;
7563
7564 mss = tcp_v4_get_syncookie(sk, iph, th, cookie: &cookie);
7565 break;
7566
7567#if IS_BUILTIN(CONFIG_IPV6)
7568 case 6:
7569 if (unlikely(iph_len < sizeof(struct ipv6hdr)))
7570 return -EINVAL;
7571
7572 if (sk->sk_family != AF_INET6)
7573 return -EINVAL;
7574
7575 mss = tcp_v6_get_syncookie(sk, iph, th, cookie: &cookie);
7576 break;
7577#endif /* CONFIG_IPV6 */
7578
7579 default:
7580 return -EPROTONOSUPPORT;
7581 }
7582 if (mss == 0)
7583 return -ENOENT;
7584
7585 return cookie | ((u64)mss << 32);
7586#else
7587 return -EOPNOTSUPP;
7588#endif /* CONFIG_SYN_COOKIES */
7589}
7590
7591static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = {
7592 .func = bpf_tcp_gen_syncookie,
7593 .gpl_only = true, /* __cookie_v*_init_sequence() is GPL */
7594 .pkt_access = true,
7595 .ret_type = RET_INTEGER,
7596 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
7597 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7598 .arg3_type = ARG_CONST_SIZE,
7599 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7600 .arg5_type = ARG_CONST_SIZE,
7601};
7602
7603BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags)
7604{
7605 if (!sk || flags != 0)
7606 return -EINVAL;
7607 if (!skb_at_tc_ingress(skb))
7608 return -EOPNOTSUPP;
7609 if (unlikely(dev_net(skb->dev) != sock_net(sk)))
7610 return -ENETUNREACH;
7611 if (sk_unhashed(sk))
7612 return -EOPNOTSUPP;
7613 if (sk_is_refcounted(sk) &&
7614 unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
7615 return -ENOENT;
7616
7617 skb_orphan(skb);
7618 skb->sk = sk;
7619 skb->destructor = sock_pfree;
7620
7621 return 0;
7622}
7623
7624static const struct bpf_func_proto bpf_sk_assign_proto = {
7625 .func = bpf_sk_assign,
7626 .gpl_only = false,
7627 .ret_type = RET_INTEGER,
7628 .arg1_type = ARG_PTR_TO_CTX,
7629 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
7630 .arg3_type = ARG_ANYTHING,
7631};
7632
7633static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend,
7634 u8 search_kind, const u8 *magic,
7635 u8 magic_len, bool *eol)
7636{
7637 u8 kind, kind_len;
7638
7639 *eol = false;
7640
7641 while (op < opend) {
7642 kind = op[0];
7643
7644 if (kind == TCPOPT_EOL) {
7645 *eol = true;
7646 return ERR_PTR(error: -ENOMSG);
7647 } else if (kind == TCPOPT_NOP) {
7648 op++;
7649 continue;
7650 }
7651
7652 if (opend - op < 2 || opend - op < op[1] || op[1] < 2)
7653 /* Something is wrong in the received header.
7654 * Follow the TCP stack's tcp_parse_options()
7655 * and just bail here.
7656 */
7657 return ERR_PTR(error: -EFAULT);
7658
7659 kind_len = op[1];
7660 if (search_kind == kind) {
7661 if (!magic_len)
7662 return op;
7663
7664 if (magic_len > kind_len - 2)
7665 return ERR_PTR(error: -ENOMSG);
7666
7667 if (!memcmp(&op[2], magic, magic_len))
7668 return op;
7669 }
7670
7671 op += kind_len;
7672 }
7673
7674 return ERR_PTR(error: -ENOMSG);
7675}
7676
7677BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
7678 void *, search_res, u32, len, u64, flags)
7679{
7680 bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN;
7681 const u8 *op, *opend, *magic, *search = search_res;
7682 u8 search_kind, search_len, copy_len, magic_len;
7683 int ret;
7684
7685 if (!is_locked_tcp_sock_ops(bpf_sock))
7686 return -EOPNOTSUPP;
7687
7688 /* 2 byte is the minimal option len except TCPOPT_NOP and
7689 * TCPOPT_EOL which are useless for the bpf prog to learn
7690 * and this helper disallow loading them also.
7691 */
7692 if (len < 2 || flags & ~BPF_LOAD_HDR_OPT_TCP_SYN)
7693 return -EINVAL;
7694
7695 search_kind = search[0];
7696 search_len = search[1];
7697
7698 if (search_len > len || search_kind == TCPOPT_NOP ||
7699 search_kind == TCPOPT_EOL)
7700 return -EINVAL;
7701
7702 if (search_kind == TCPOPT_EXP || search_kind == 253) {
7703 /* 16 or 32 bit magic. +2 for kind and kind length */
7704 if (search_len != 4 && search_len != 6)
7705 return -EINVAL;
7706 magic = &search[2];
7707 magic_len = search_len - 2;
7708 } else {
7709 if (search_len)
7710 return -EINVAL;
7711 magic = NULL;
7712 magic_len = 0;
7713 }
7714
7715 if (load_syn) {
7716 ret = bpf_sock_ops_get_syn(bpf_sock, optname: TCP_BPF_SYN, start: &op);
7717 if (ret < 0)
7718 return ret;
7719
7720 opend = op + ret;
7721 op += sizeof(struct tcphdr);
7722 } else {
7723 if (!bpf_sock->skb ||
7724 bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB)
7725 /* This bpf_sock->op cannot call this helper */
7726 return -EPERM;
7727
7728 opend = bpf_sock->skb_data_end;
7729 op = bpf_sock->skb->data + sizeof(struct tcphdr);
7730 }
7731
7732 op = bpf_search_tcp_opt(op, opend, search_kind, magic, magic_len,
7733 eol: &eol);
7734 if (IS_ERR(ptr: op))
7735 return PTR_ERR(ptr: op);
7736
7737 copy_len = op[1];
7738 ret = copy_len;
7739 if (copy_len > len) {
7740 ret = -ENOSPC;
7741 copy_len = len;
7742 }
7743
7744 memcpy(to: search_res, from: op, len: copy_len);
7745 return ret;
7746}
7747
7748static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = {
7749 .func = bpf_sock_ops_load_hdr_opt,
7750 .gpl_only = false,
7751 .ret_type = RET_INTEGER,
7752 .arg1_type = ARG_PTR_TO_CTX,
7753 .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE,
7754 .arg3_type = ARG_CONST_SIZE,
7755 .arg4_type = ARG_ANYTHING,
7756};
7757
7758BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
7759 const void *, from, u32, len, u64, flags)
7760{
7761 u8 new_kind, new_kind_len, magic_len = 0, *opend;
7762 const u8 *op, *new_op, *magic = NULL;
7763 struct sk_buff *skb;
7764 bool eol;
7765
7766 if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB)
7767 return -EPERM;
7768
7769 if (len < 2 || flags)
7770 return -EINVAL;
7771
7772 new_op = from;
7773 new_kind = new_op[0];
7774 new_kind_len = new_op[1];
7775
7776 if (new_kind_len > len || new_kind == TCPOPT_NOP ||
7777 new_kind == TCPOPT_EOL)
7778 return -EINVAL;
7779
7780 if (new_kind_len > bpf_sock->remaining_opt_len)
7781 return -ENOSPC;
7782
7783 /* 253 is another experimental kind */
7784 if (new_kind == TCPOPT_EXP || new_kind == 253) {
7785 if (new_kind_len < 4)
7786 return -EINVAL;
7787 /* Match for the 2 byte magic also.
7788 * RFC 6994: the magic could be 2 or 4 bytes.
7789 * Hence, matching by 2 byte only is on the
7790 * conservative side but it is the right
7791 * thing to do for the 'search-for-duplication'
7792 * purpose.
7793 */
7794 magic = &new_op[2];
7795 magic_len = 2;
7796 }
7797
7798 /* Check for duplication */
7799 skb = bpf_sock->skb;
7800 op = skb->data + sizeof(struct tcphdr);
7801 opend = bpf_sock->skb_data_end;
7802
7803 op = bpf_search_tcp_opt(op, opend, search_kind: new_kind, magic, magic_len,
7804 eol: &eol);
7805 if (!IS_ERR(ptr: op))
7806 return -EEXIST;
7807
7808 if (PTR_ERR(ptr: op) != -ENOMSG)
7809 return PTR_ERR(ptr: op);
7810
7811 if (eol)
7812 /* The option has been ended. Treat it as no more
7813 * header option can be written.
7814 */
7815 return -ENOSPC;
7816
7817 /* No duplication found. Store the header option. */
7818 memcpy(to: opend, from, len: new_kind_len);
7819
7820 bpf_sock->remaining_opt_len -= new_kind_len;
7821 bpf_sock->skb_data_end += new_kind_len;
7822
7823 return 0;
7824}
7825
7826static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = {
7827 .func = bpf_sock_ops_store_hdr_opt,
7828 .gpl_only = false,
7829 .ret_type = RET_INTEGER,
7830 .arg1_type = ARG_PTR_TO_CTX,
7831 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
7832 .arg3_type = ARG_CONST_SIZE,
7833 .arg4_type = ARG_ANYTHING,
7834};
7835
7836BPF_CALL_3(bpf_sock_ops_reserve_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
7837 u32, len, u64, flags)
7838{
7839 if (bpf_sock->op != BPF_SOCK_OPS_HDR_OPT_LEN_CB)
7840 return -EPERM;
7841
7842 if (flags || len < 2)
7843 return -EINVAL;
7844
7845 if (len > bpf_sock->remaining_opt_len)
7846 return -ENOSPC;
7847
7848 bpf_sock->remaining_opt_len -= len;
7849
7850 return 0;
7851}
7852
7853static const struct bpf_func_proto bpf_sock_ops_reserve_hdr_opt_proto = {
7854 .func = bpf_sock_ops_reserve_hdr_opt,
7855 .gpl_only = false,
7856 .ret_type = RET_INTEGER,
7857 .arg1_type = ARG_PTR_TO_CTX,
7858 .arg2_type = ARG_ANYTHING,
7859 .arg3_type = ARG_ANYTHING,
7860};
7861
7862BPF_CALL_3(bpf_skb_set_tstamp, struct sk_buff *, skb,
7863 u64, tstamp, u32, tstamp_type)
7864{
7865 /* skb_clear_delivery_time() is done for inet protocol */
7866 if (skb->protocol != htons(ETH_P_IP) &&
7867 skb->protocol != htons(ETH_P_IPV6))
7868 return -EOPNOTSUPP;
7869
7870 switch (tstamp_type) {
7871 case BPF_SKB_CLOCK_REALTIME:
7872 skb->tstamp = tstamp;
7873 skb->tstamp_type = SKB_CLOCK_REALTIME;
7874 break;
7875 case BPF_SKB_CLOCK_MONOTONIC:
7876 if (!tstamp)
7877 return -EINVAL;
7878 skb->tstamp = tstamp;
7879 skb->tstamp_type = SKB_CLOCK_MONOTONIC;
7880 break;
7881 case BPF_SKB_CLOCK_TAI:
7882 if (!tstamp)
7883 return -EINVAL;
7884 skb->tstamp = tstamp;
7885 skb->tstamp_type = SKB_CLOCK_TAI;
7886 break;
7887 default:
7888 return -EINVAL;
7889 }
7890
7891 return 0;
7892}
7893
7894static const struct bpf_func_proto bpf_skb_set_tstamp_proto = {
7895 .func = bpf_skb_set_tstamp,
7896 .gpl_only = false,
7897 .ret_type = RET_INTEGER,
7898 .arg1_type = ARG_PTR_TO_CTX,
7899 .arg2_type = ARG_ANYTHING,
7900 .arg3_type = ARG_ANYTHING,
7901};
7902
7903#ifdef CONFIG_SYN_COOKIES
7904BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv4, struct iphdr *, iph,
7905 struct tcphdr *, th, u32, th_len)
7906{
7907 u32 cookie;
7908 u16 mss;
7909
7910 if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
7911 return -EINVAL;
7912
7913 mss = tcp_parse_mss_option(th, user_mss: 0) ?: TCP_MSS_DEFAULT;
7914 cookie = __cookie_v4_init_sequence(iph, th, mssp: &mss);
7915
7916 return cookie | ((u64)mss << 32);
7917}
7918
7919static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = {
7920 .func = bpf_tcp_raw_gen_syncookie_ipv4,
7921 .gpl_only = true, /* __cookie_v4_init_sequence() is GPL */
7922 .pkt_access = true,
7923 .ret_type = RET_INTEGER,
7924 .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
7925 .arg1_size = sizeof(struct iphdr),
7926 .arg2_type = ARG_PTR_TO_MEM,
7927 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7928};
7929
7930BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv6, struct ipv6hdr *, iph,
7931 struct tcphdr *, th, u32, th_len)
7932{
7933#if IS_BUILTIN(CONFIG_IPV6)
7934 const u16 mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
7935 sizeof(struct ipv6hdr);
7936 u32 cookie;
7937 u16 mss;
7938
7939 if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
7940 return -EINVAL;
7941
7942 mss = tcp_parse_mss_option(th, user_mss: 0) ?: mss_clamp;
7943 cookie = __cookie_v6_init_sequence(iph, th, mssp: &mss);
7944
7945 return cookie | ((u64)mss << 32);
7946#else
7947 return -EPROTONOSUPPORT;
7948#endif
7949}
7950
7951static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = {
7952 .func = bpf_tcp_raw_gen_syncookie_ipv6,
7953 .gpl_only = true, /* __cookie_v6_init_sequence() is GPL */
7954 .pkt_access = true,
7955 .ret_type = RET_INTEGER,
7956 .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
7957 .arg1_size = sizeof(struct ipv6hdr),
7958 .arg2_type = ARG_PTR_TO_MEM,
7959 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
7960};
7961
7962BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph,
7963 struct tcphdr *, th)
7964{
7965 if (__cookie_v4_check(iph, th) > 0)
7966 return 0;
7967
7968 return -EACCES;
7969}
7970
7971static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = {
7972 .func = bpf_tcp_raw_check_syncookie_ipv4,
7973 .gpl_only = true, /* __cookie_v4_check is GPL */
7974 .pkt_access = true,
7975 .ret_type = RET_INTEGER,
7976 .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
7977 .arg1_size = sizeof(struct iphdr),
7978 .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM,
7979 .arg2_size = sizeof(struct tcphdr),
7980};
7981
7982BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv6, struct ipv6hdr *, iph,
7983 struct tcphdr *, th)
7984{
7985#if IS_BUILTIN(CONFIG_IPV6)
7986 if (__cookie_v6_check(iph, th) > 0)
7987 return 0;
7988
7989 return -EACCES;
7990#else
7991 return -EPROTONOSUPPORT;
7992#endif
7993}
7994
7995static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = {
7996 .func = bpf_tcp_raw_check_syncookie_ipv6,
7997 .gpl_only = true, /* __cookie_v6_check is GPL */
7998 .pkt_access = true,
7999 .ret_type = RET_INTEGER,
8000 .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
8001 .arg1_size = sizeof(struct ipv6hdr),
8002 .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM,
8003 .arg2_size = sizeof(struct tcphdr),
8004};
8005#endif /* CONFIG_SYN_COOKIES */
8006
8007#endif /* CONFIG_INET */
8008
8009bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id)
8010{
8011 switch (func_id) {
8012 case BPF_FUNC_clone_redirect:
8013 case BPF_FUNC_l3_csum_replace:
8014 case BPF_FUNC_l4_csum_replace:
8015 case BPF_FUNC_lwt_push_encap:
8016 case BPF_FUNC_lwt_seg6_action:
8017 case BPF_FUNC_lwt_seg6_adjust_srh:
8018 case BPF_FUNC_lwt_seg6_store_bytes:
8019 case BPF_FUNC_msg_pop_data:
8020 case BPF_FUNC_msg_pull_data:
8021 case BPF_FUNC_msg_push_data:
8022 case BPF_FUNC_skb_adjust_room:
8023 case BPF_FUNC_skb_change_head:
8024 case BPF_FUNC_skb_change_proto:
8025 case BPF_FUNC_skb_change_tail:
8026 case BPF_FUNC_skb_pull_data:
8027 case BPF_FUNC_skb_store_bytes:
8028 case BPF_FUNC_skb_vlan_pop:
8029 case BPF_FUNC_skb_vlan_push:
8030 case BPF_FUNC_store_hdr_opt:
8031 case BPF_FUNC_xdp_adjust_head:
8032 case BPF_FUNC_xdp_adjust_meta:
8033 case BPF_FUNC_xdp_adjust_tail:
8034 /* tail-called program could call any of the above */
8035 case BPF_FUNC_tail_call:
8036 return true;
8037 default:
8038 return false;
8039 }
8040}
8041
8042const struct bpf_func_proto bpf_event_output_data_proto __weak;
8043const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak;
8044
8045static const struct bpf_func_proto *
8046sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8047{
8048 const struct bpf_func_proto *func_proto;
8049
8050 func_proto = cgroup_common_func_proto(func_id, prog);
8051 if (func_proto)
8052 return func_proto;
8053
8054 switch (func_id) {
8055 case BPF_FUNC_get_socket_cookie:
8056 return &bpf_get_socket_cookie_sock_proto;
8057 case BPF_FUNC_get_netns_cookie:
8058 return &bpf_get_netns_cookie_sock_proto;
8059 case BPF_FUNC_perf_event_output:
8060 return &bpf_event_output_data_proto;
8061 case BPF_FUNC_sk_storage_get:
8062 return &bpf_sk_storage_get_cg_sock_proto;
8063 case BPF_FUNC_ktime_get_coarse_ns:
8064 return &bpf_ktime_get_coarse_ns_proto;
8065 default:
8066 return bpf_base_func_proto(func_id, prog);
8067 }
8068}
8069
8070static const struct bpf_func_proto *
8071sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8072{
8073 const struct bpf_func_proto *func_proto;
8074
8075 func_proto = cgroup_common_func_proto(func_id, prog);
8076 if (func_proto)
8077 return func_proto;
8078
8079 switch (func_id) {
8080 case BPF_FUNC_bind:
8081 switch (prog->expected_attach_type) {
8082 case BPF_CGROUP_INET4_CONNECT:
8083 case BPF_CGROUP_INET6_CONNECT:
8084 return &bpf_bind_proto;
8085 default:
8086 return NULL;
8087 }
8088 case BPF_FUNC_get_socket_cookie:
8089 return &bpf_get_socket_cookie_sock_addr_proto;
8090 case BPF_FUNC_get_netns_cookie:
8091 return &bpf_get_netns_cookie_sock_addr_proto;
8092 case BPF_FUNC_perf_event_output:
8093 return &bpf_event_output_data_proto;
8094#ifdef CONFIG_INET
8095 case BPF_FUNC_sk_lookup_tcp:
8096 return &bpf_sock_addr_sk_lookup_tcp_proto;
8097 case BPF_FUNC_sk_lookup_udp:
8098 return &bpf_sock_addr_sk_lookup_udp_proto;
8099 case BPF_FUNC_sk_release:
8100 return &bpf_sk_release_proto;
8101 case BPF_FUNC_skc_lookup_tcp:
8102 return &bpf_sock_addr_skc_lookup_tcp_proto;
8103#endif /* CONFIG_INET */
8104 case BPF_FUNC_sk_storage_get:
8105 return &bpf_sk_storage_get_proto;
8106 case BPF_FUNC_sk_storage_delete:
8107 return &bpf_sk_storage_delete_proto;
8108 case BPF_FUNC_setsockopt:
8109 switch (prog->expected_attach_type) {
8110 case BPF_CGROUP_INET4_BIND:
8111 case BPF_CGROUP_INET6_BIND:
8112 case BPF_CGROUP_INET4_CONNECT:
8113 case BPF_CGROUP_INET6_CONNECT:
8114 case BPF_CGROUP_UNIX_CONNECT:
8115 case BPF_CGROUP_UDP4_RECVMSG:
8116 case BPF_CGROUP_UDP6_RECVMSG:
8117 case BPF_CGROUP_UNIX_RECVMSG:
8118 case BPF_CGROUP_UDP4_SENDMSG:
8119 case BPF_CGROUP_UDP6_SENDMSG:
8120 case BPF_CGROUP_UNIX_SENDMSG:
8121 case BPF_CGROUP_INET4_GETPEERNAME:
8122 case BPF_CGROUP_INET6_GETPEERNAME:
8123 case BPF_CGROUP_UNIX_GETPEERNAME:
8124 case BPF_CGROUP_INET4_GETSOCKNAME:
8125 case BPF_CGROUP_INET6_GETSOCKNAME:
8126 case BPF_CGROUP_UNIX_GETSOCKNAME:
8127 return &bpf_sock_addr_setsockopt_proto;
8128 default:
8129 return NULL;
8130 }
8131 case BPF_FUNC_getsockopt:
8132 switch (prog->expected_attach_type) {
8133 case BPF_CGROUP_INET4_BIND:
8134 case BPF_CGROUP_INET6_BIND:
8135 case BPF_CGROUP_INET4_CONNECT:
8136 case BPF_CGROUP_INET6_CONNECT:
8137 case BPF_CGROUP_UNIX_CONNECT:
8138 case BPF_CGROUP_UDP4_RECVMSG:
8139 case BPF_CGROUP_UDP6_RECVMSG:
8140 case BPF_CGROUP_UNIX_RECVMSG:
8141 case BPF_CGROUP_UDP4_SENDMSG:
8142 case BPF_CGROUP_UDP6_SENDMSG:
8143 case BPF_CGROUP_UNIX_SENDMSG:
8144 case BPF_CGROUP_INET4_GETPEERNAME:
8145 case BPF_CGROUP_INET6_GETPEERNAME:
8146 case BPF_CGROUP_UNIX_GETPEERNAME:
8147 case BPF_CGROUP_INET4_GETSOCKNAME:
8148 case BPF_CGROUP_INET6_GETSOCKNAME:
8149 case BPF_CGROUP_UNIX_GETSOCKNAME:
8150 return &bpf_sock_addr_getsockopt_proto;
8151 default:
8152 return NULL;
8153 }
8154 default:
8155 return bpf_sk_base_func_proto(func_id, prog);
8156 }
8157}
8158
8159static const struct bpf_func_proto *
8160sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8161{
8162 switch (func_id) {
8163 case BPF_FUNC_skb_load_bytes:
8164 return &bpf_skb_load_bytes_proto;
8165 case BPF_FUNC_skb_load_bytes_relative:
8166 return &bpf_skb_load_bytes_relative_proto;
8167 case BPF_FUNC_get_socket_cookie:
8168 return &bpf_get_socket_cookie_proto;
8169 case BPF_FUNC_get_netns_cookie:
8170 return &bpf_get_netns_cookie_proto;
8171 case BPF_FUNC_get_socket_uid:
8172 return &bpf_get_socket_uid_proto;
8173 case BPF_FUNC_perf_event_output:
8174 return &bpf_skb_event_output_proto;
8175 default:
8176 return bpf_sk_base_func_proto(func_id, prog);
8177 }
8178}
8179
8180const struct bpf_func_proto bpf_sk_storage_get_proto __weak;
8181const struct bpf_func_proto bpf_sk_storage_delete_proto __weak;
8182
8183static const struct bpf_func_proto *
8184cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8185{
8186 const struct bpf_func_proto *func_proto;
8187
8188 func_proto = cgroup_common_func_proto(func_id, prog);
8189 if (func_proto)
8190 return func_proto;
8191
8192 switch (func_id) {
8193 case BPF_FUNC_sk_fullsock:
8194 return &bpf_sk_fullsock_proto;
8195 case BPF_FUNC_sk_storage_get:
8196 return &bpf_sk_storage_get_proto;
8197 case BPF_FUNC_sk_storage_delete:
8198 return &bpf_sk_storage_delete_proto;
8199 case BPF_FUNC_perf_event_output:
8200 return &bpf_skb_event_output_proto;
8201#ifdef CONFIG_SOCK_CGROUP_DATA
8202 case BPF_FUNC_skb_cgroup_id:
8203 return &bpf_skb_cgroup_id_proto;
8204 case BPF_FUNC_skb_ancestor_cgroup_id:
8205 return &bpf_skb_ancestor_cgroup_id_proto;
8206 case BPF_FUNC_sk_cgroup_id:
8207 return &bpf_sk_cgroup_id_proto;
8208 case BPF_FUNC_sk_ancestor_cgroup_id:
8209 return &bpf_sk_ancestor_cgroup_id_proto;
8210#endif
8211#ifdef CONFIG_INET
8212 case BPF_FUNC_sk_lookup_tcp:
8213 return &bpf_sk_lookup_tcp_proto;
8214 case BPF_FUNC_sk_lookup_udp:
8215 return &bpf_sk_lookup_udp_proto;
8216 case BPF_FUNC_sk_release:
8217 return &bpf_sk_release_proto;
8218 case BPF_FUNC_skc_lookup_tcp:
8219 return &bpf_skc_lookup_tcp_proto;
8220 case BPF_FUNC_tcp_sock:
8221 return &bpf_tcp_sock_proto;
8222 case BPF_FUNC_get_listener_sock:
8223 return &bpf_get_listener_sock_proto;
8224 case BPF_FUNC_skb_ecn_set_ce:
8225 return &bpf_skb_ecn_set_ce_proto;
8226#endif
8227 default:
8228 return sk_filter_func_proto(func_id, prog);
8229 }
8230}
8231
8232static const struct bpf_func_proto *
8233tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8234{
8235 switch (func_id) {
8236 case BPF_FUNC_skb_store_bytes:
8237 return &bpf_skb_store_bytes_proto;
8238 case BPF_FUNC_skb_load_bytes:
8239 return &bpf_skb_load_bytes_proto;
8240 case BPF_FUNC_skb_load_bytes_relative:
8241 return &bpf_skb_load_bytes_relative_proto;
8242 case BPF_FUNC_skb_pull_data:
8243 return &bpf_skb_pull_data_proto;
8244 case BPF_FUNC_csum_diff:
8245 return &bpf_csum_diff_proto;
8246 case BPF_FUNC_csum_update:
8247 return &bpf_csum_update_proto;
8248 case BPF_FUNC_csum_level:
8249 return &bpf_csum_level_proto;
8250 case BPF_FUNC_l3_csum_replace:
8251 return &bpf_l3_csum_replace_proto;
8252 case BPF_FUNC_l4_csum_replace:
8253 return &bpf_l4_csum_replace_proto;
8254 case BPF_FUNC_clone_redirect:
8255 return &bpf_clone_redirect_proto;
8256 case BPF_FUNC_get_cgroup_classid:
8257 return &bpf_get_cgroup_classid_proto;
8258 case BPF_FUNC_skb_vlan_push:
8259 return &bpf_skb_vlan_push_proto;
8260 case BPF_FUNC_skb_vlan_pop:
8261 return &bpf_skb_vlan_pop_proto;
8262 case BPF_FUNC_skb_change_proto:
8263 return &bpf_skb_change_proto_proto;
8264 case BPF_FUNC_skb_change_type:
8265 return &bpf_skb_change_type_proto;
8266 case BPF_FUNC_skb_adjust_room:
8267 return &bpf_skb_adjust_room_proto;
8268 case BPF_FUNC_skb_change_tail:
8269 return &bpf_skb_change_tail_proto;
8270 case BPF_FUNC_skb_change_head:
8271 return &bpf_skb_change_head_proto;
8272 case BPF_FUNC_skb_get_tunnel_key:
8273 return &bpf_skb_get_tunnel_key_proto;
8274 case BPF_FUNC_skb_set_tunnel_key:
8275 return bpf_get_skb_set_tunnel_proto(which: func_id);
8276 case BPF_FUNC_skb_get_tunnel_opt:
8277 return &bpf_skb_get_tunnel_opt_proto;
8278 case BPF_FUNC_skb_set_tunnel_opt:
8279 return bpf_get_skb_set_tunnel_proto(which: func_id);
8280 case BPF_FUNC_redirect:
8281 return &bpf_redirect_proto;
8282 case BPF_FUNC_redirect_neigh:
8283 return &bpf_redirect_neigh_proto;
8284 case BPF_FUNC_redirect_peer:
8285 return &bpf_redirect_peer_proto;
8286 case BPF_FUNC_get_route_realm:
8287 return &bpf_get_route_realm_proto;
8288 case BPF_FUNC_get_hash_recalc:
8289 return &bpf_get_hash_recalc_proto;
8290 case BPF_FUNC_set_hash_invalid:
8291 return &bpf_set_hash_invalid_proto;
8292 case BPF_FUNC_set_hash:
8293 return &bpf_set_hash_proto;
8294 case BPF_FUNC_perf_event_output:
8295 return &bpf_skb_event_output_proto;
8296 case BPF_FUNC_get_smp_processor_id:
8297 return &bpf_get_smp_processor_id_proto;
8298 case BPF_FUNC_skb_under_cgroup:
8299 return &bpf_skb_under_cgroup_proto;
8300 case BPF_FUNC_get_socket_cookie:
8301 return &bpf_get_socket_cookie_proto;
8302 case BPF_FUNC_get_netns_cookie:
8303 return &bpf_get_netns_cookie_proto;
8304 case BPF_FUNC_get_socket_uid:
8305 return &bpf_get_socket_uid_proto;
8306 case BPF_FUNC_fib_lookup:
8307 return &bpf_skb_fib_lookup_proto;
8308 case BPF_FUNC_check_mtu:
8309 return &bpf_skb_check_mtu_proto;
8310 case BPF_FUNC_sk_fullsock:
8311 return &bpf_sk_fullsock_proto;
8312 case BPF_FUNC_sk_storage_get:
8313 return &bpf_sk_storage_get_proto;
8314 case BPF_FUNC_sk_storage_delete:
8315 return &bpf_sk_storage_delete_proto;
8316#ifdef CONFIG_XFRM
8317 case BPF_FUNC_skb_get_xfrm_state:
8318 return &bpf_skb_get_xfrm_state_proto;
8319#endif
8320#ifdef CONFIG_CGROUP_NET_CLASSID
8321 case BPF_FUNC_skb_cgroup_classid:
8322 return &bpf_skb_cgroup_classid_proto;
8323#endif
8324#ifdef CONFIG_SOCK_CGROUP_DATA
8325 case BPF_FUNC_skb_cgroup_id:
8326 return &bpf_skb_cgroup_id_proto;
8327 case BPF_FUNC_skb_ancestor_cgroup_id:
8328 return &bpf_skb_ancestor_cgroup_id_proto;
8329#endif
8330#ifdef CONFIG_INET
8331 case BPF_FUNC_sk_lookup_tcp:
8332 return &bpf_tc_sk_lookup_tcp_proto;
8333 case BPF_FUNC_sk_lookup_udp:
8334 return &bpf_tc_sk_lookup_udp_proto;
8335 case BPF_FUNC_sk_release:
8336 return &bpf_sk_release_proto;
8337 case BPF_FUNC_tcp_sock:
8338 return &bpf_tcp_sock_proto;
8339 case BPF_FUNC_get_listener_sock:
8340 return &bpf_get_listener_sock_proto;
8341 case BPF_FUNC_skc_lookup_tcp:
8342 return &bpf_tc_skc_lookup_tcp_proto;
8343 case BPF_FUNC_tcp_check_syncookie:
8344 return &bpf_tcp_check_syncookie_proto;
8345 case BPF_FUNC_skb_ecn_set_ce:
8346 return &bpf_skb_ecn_set_ce_proto;
8347 case BPF_FUNC_tcp_gen_syncookie:
8348 return &bpf_tcp_gen_syncookie_proto;
8349 case BPF_FUNC_sk_assign:
8350 return &bpf_sk_assign_proto;
8351 case BPF_FUNC_skb_set_tstamp:
8352 return &bpf_skb_set_tstamp_proto;
8353#ifdef CONFIG_SYN_COOKIES
8354 case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
8355 return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
8356 case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
8357 return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
8358 case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
8359 return &bpf_tcp_raw_check_syncookie_ipv4_proto;
8360 case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
8361 return &bpf_tcp_raw_check_syncookie_ipv6_proto;
8362#endif
8363#endif
8364 default:
8365 return bpf_sk_base_func_proto(func_id, prog);
8366 }
8367}
8368
8369static const struct bpf_func_proto *
8370xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8371{
8372 switch (func_id) {
8373 case BPF_FUNC_perf_event_output:
8374 return &bpf_xdp_event_output_proto;
8375 case BPF_FUNC_get_smp_processor_id:
8376 return &bpf_get_smp_processor_id_proto;
8377 case BPF_FUNC_csum_diff:
8378 return &bpf_csum_diff_proto;
8379 case BPF_FUNC_xdp_adjust_head:
8380 return &bpf_xdp_adjust_head_proto;
8381 case BPF_FUNC_xdp_adjust_meta:
8382 return &bpf_xdp_adjust_meta_proto;
8383 case BPF_FUNC_redirect:
8384 return &bpf_xdp_redirect_proto;
8385 case BPF_FUNC_redirect_map:
8386 return &bpf_xdp_redirect_map_proto;
8387 case BPF_FUNC_xdp_adjust_tail:
8388 return &bpf_xdp_adjust_tail_proto;
8389 case BPF_FUNC_xdp_get_buff_len:
8390 return &bpf_xdp_get_buff_len_proto;
8391 case BPF_FUNC_xdp_load_bytes:
8392 return &bpf_xdp_load_bytes_proto;
8393 case BPF_FUNC_xdp_store_bytes:
8394 return &bpf_xdp_store_bytes_proto;
8395 case BPF_FUNC_fib_lookup:
8396 return &bpf_xdp_fib_lookup_proto;
8397 case BPF_FUNC_check_mtu:
8398 return &bpf_xdp_check_mtu_proto;
8399#ifdef CONFIG_INET
8400 case BPF_FUNC_sk_lookup_udp:
8401 return &bpf_xdp_sk_lookup_udp_proto;
8402 case BPF_FUNC_sk_lookup_tcp:
8403 return &bpf_xdp_sk_lookup_tcp_proto;
8404 case BPF_FUNC_sk_release:
8405 return &bpf_sk_release_proto;
8406 case BPF_FUNC_skc_lookup_tcp:
8407 return &bpf_xdp_skc_lookup_tcp_proto;
8408 case BPF_FUNC_tcp_check_syncookie:
8409 return &bpf_tcp_check_syncookie_proto;
8410 case BPF_FUNC_tcp_gen_syncookie:
8411 return &bpf_tcp_gen_syncookie_proto;
8412#ifdef CONFIG_SYN_COOKIES
8413 case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
8414 return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
8415 case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
8416 return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
8417 case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
8418 return &bpf_tcp_raw_check_syncookie_ipv4_proto;
8419 case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
8420 return &bpf_tcp_raw_check_syncookie_ipv6_proto;
8421#endif
8422#endif
8423 default:
8424 return bpf_sk_base_func_proto(func_id, prog);
8425 }
8426
8427#if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)
8428 /* The nf_conn___init type is used in the NF_CONNTRACK kfuncs. The
8429 * kfuncs are defined in two different modules, and we want to be able
8430 * to use them interchangeably with the same BTF type ID. Because modules
8431 * can't de-duplicate BTF IDs between each other, we need the type to be
8432 * referenced in the vmlinux BTF or the verifier will get confused about
8433 * the different types. So we add this dummy type reference which will
8434 * be included in vmlinux BTF, allowing both modules to refer to the
8435 * same type ID.
8436 */
8437 BTF_TYPE_EMIT(struct nf_conn___init);
8438#endif
8439}
8440
8441const struct bpf_func_proto bpf_sock_map_update_proto __weak;
8442const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
8443
8444static const struct bpf_func_proto *
8445sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8446{
8447 const struct bpf_func_proto *func_proto;
8448
8449 func_proto = cgroup_common_func_proto(func_id, prog);
8450 if (func_proto)
8451 return func_proto;
8452
8453 switch (func_id) {
8454 case BPF_FUNC_setsockopt:
8455 return &bpf_sock_ops_setsockopt_proto;
8456 case BPF_FUNC_getsockopt:
8457 return &bpf_sock_ops_getsockopt_proto;
8458 case BPF_FUNC_sock_ops_cb_flags_set:
8459 return &bpf_sock_ops_cb_flags_set_proto;
8460 case BPF_FUNC_sock_map_update:
8461 return &bpf_sock_map_update_proto;
8462 case BPF_FUNC_sock_hash_update:
8463 return &bpf_sock_hash_update_proto;
8464 case BPF_FUNC_get_socket_cookie:
8465 return &bpf_get_socket_cookie_sock_ops_proto;
8466 case BPF_FUNC_perf_event_output:
8467 return &bpf_event_output_data_proto;
8468 case BPF_FUNC_sk_storage_get:
8469 return &bpf_sk_storage_get_proto;
8470 case BPF_FUNC_sk_storage_delete:
8471 return &bpf_sk_storage_delete_proto;
8472 case BPF_FUNC_get_netns_cookie:
8473 return &bpf_get_netns_cookie_sock_ops_proto;
8474#ifdef CONFIG_INET
8475 case BPF_FUNC_load_hdr_opt:
8476 return &bpf_sock_ops_load_hdr_opt_proto;
8477 case BPF_FUNC_store_hdr_opt:
8478 return &bpf_sock_ops_store_hdr_opt_proto;
8479 case BPF_FUNC_reserve_hdr_opt:
8480 return &bpf_sock_ops_reserve_hdr_opt_proto;
8481 case BPF_FUNC_tcp_sock:
8482 return &bpf_tcp_sock_proto;
8483#endif /* CONFIG_INET */
8484 default:
8485 return bpf_sk_base_func_proto(func_id, prog);
8486 }
8487}
8488
8489const struct bpf_func_proto bpf_msg_redirect_map_proto __weak;
8490const struct bpf_func_proto bpf_msg_redirect_hash_proto __weak;
8491
8492static const struct bpf_func_proto *
8493sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8494{
8495 switch (func_id) {
8496 case BPF_FUNC_msg_redirect_map:
8497 return &bpf_msg_redirect_map_proto;
8498 case BPF_FUNC_msg_redirect_hash:
8499 return &bpf_msg_redirect_hash_proto;
8500 case BPF_FUNC_msg_apply_bytes:
8501 return &bpf_msg_apply_bytes_proto;
8502 case BPF_FUNC_msg_cork_bytes:
8503 return &bpf_msg_cork_bytes_proto;
8504 case BPF_FUNC_msg_pull_data:
8505 return &bpf_msg_pull_data_proto;
8506 case BPF_FUNC_msg_push_data:
8507 return &bpf_msg_push_data_proto;
8508 case BPF_FUNC_msg_pop_data:
8509 return &bpf_msg_pop_data_proto;
8510 case BPF_FUNC_perf_event_output:
8511 return &bpf_event_output_data_proto;
8512 case BPF_FUNC_sk_storage_get:
8513 return &bpf_sk_storage_get_proto;
8514 case BPF_FUNC_sk_storage_delete:
8515 return &bpf_sk_storage_delete_proto;
8516 case BPF_FUNC_get_netns_cookie:
8517 return &bpf_get_netns_cookie_sk_msg_proto;
8518 default:
8519 return bpf_sk_base_func_proto(func_id, prog);
8520 }
8521}
8522
8523const struct bpf_func_proto bpf_sk_redirect_map_proto __weak;
8524const struct bpf_func_proto bpf_sk_redirect_hash_proto __weak;
8525
8526static const struct bpf_func_proto *
8527sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8528{
8529 switch (func_id) {
8530 case BPF_FUNC_skb_store_bytes:
8531 return &bpf_skb_store_bytes_proto;
8532 case BPF_FUNC_skb_load_bytes:
8533 return &bpf_skb_load_bytes_proto;
8534 case BPF_FUNC_skb_pull_data:
8535 return &sk_skb_pull_data_proto;
8536 case BPF_FUNC_skb_change_tail:
8537 return &sk_skb_change_tail_proto;
8538 case BPF_FUNC_skb_change_head:
8539 return &sk_skb_change_head_proto;
8540 case BPF_FUNC_skb_adjust_room:
8541 return &sk_skb_adjust_room_proto;
8542 case BPF_FUNC_get_socket_cookie:
8543 return &bpf_get_socket_cookie_proto;
8544 case BPF_FUNC_get_socket_uid:
8545 return &bpf_get_socket_uid_proto;
8546 case BPF_FUNC_sk_redirect_map:
8547 return &bpf_sk_redirect_map_proto;
8548 case BPF_FUNC_sk_redirect_hash:
8549 return &bpf_sk_redirect_hash_proto;
8550 case BPF_FUNC_perf_event_output:
8551 return &bpf_skb_event_output_proto;
8552#ifdef CONFIG_INET
8553 case BPF_FUNC_sk_lookup_tcp:
8554 return &bpf_sk_lookup_tcp_proto;
8555 case BPF_FUNC_sk_lookup_udp:
8556 return &bpf_sk_lookup_udp_proto;
8557 case BPF_FUNC_sk_release:
8558 return &bpf_sk_release_proto;
8559 case BPF_FUNC_skc_lookup_tcp:
8560 return &bpf_skc_lookup_tcp_proto;
8561#endif
8562 default:
8563 return bpf_sk_base_func_proto(func_id, prog);
8564 }
8565}
8566
8567static const struct bpf_func_proto *
8568flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8569{
8570 switch (func_id) {
8571 case BPF_FUNC_skb_load_bytes:
8572 return &bpf_flow_dissector_load_bytes_proto;
8573 default:
8574 return bpf_sk_base_func_proto(func_id, prog);
8575 }
8576}
8577
8578static const struct bpf_func_proto *
8579lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8580{
8581 switch (func_id) {
8582 case BPF_FUNC_skb_load_bytes:
8583 return &bpf_skb_load_bytes_proto;
8584 case BPF_FUNC_skb_pull_data:
8585 return &bpf_skb_pull_data_proto;
8586 case BPF_FUNC_csum_diff:
8587 return &bpf_csum_diff_proto;
8588 case BPF_FUNC_get_cgroup_classid:
8589 return &bpf_get_cgroup_classid_proto;
8590 case BPF_FUNC_get_route_realm:
8591 return &bpf_get_route_realm_proto;
8592 case BPF_FUNC_get_hash_recalc:
8593 return &bpf_get_hash_recalc_proto;
8594 case BPF_FUNC_perf_event_output:
8595 return &bpf_skb_event_output_proto;
8596 case BPF_FUNC_get_smp_processor_id:
8597 return &bpf_get_smp_processor_id_proto;
8598 case BPF_FUNC_skb_under_cgroup:
8599 return &bpf_skb_under_cgroup_proto;
8600 default:
8601 return bpf_sk_base_func_proto(func_id, prog);
8602 }
8603}
8604
8605static const struct bpf_func_proto *
8606lwt_in_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8607{
8608 switch (func_id) {
8609 case BPF_FUNC_lwt_push_encap:
8610 return &bpf_lwt_in_push_encap_proto;
8611 default:
8612 return lwt_out_func_proto(func_id, prog);
8613 }
8614}
8615
8616static const struct bpf_func_proto *
8617lwt_xmit_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8618{
8619 switch (func_id) {
8620 case BPF_FUNC_skb_get_tunnel_key:
8621 return &bpf_skb_get_tunnel_key_proto;
8622 case BPF_FUNC_skb_set_tunnel_key:
8623 return bpf_get_skb_set_tunnel_proto(which: func_id);
8624 case BPF_FUNC_skb_get_tunnel_opt:
8625 return &bpf_skb_get_tunnel_opt_proto;
8626 case BPF_FUNC_skb_set_tunnel_opt:
8627 return bpf_get_skb_set_tunnel_proto(which: func_id);
8628 case BPF_FUNC_redirect:
8629 return &bpf_redirect_proto;
8630 case BPF_FUNC_clone_redirect:
8631 return &bpf_clone_redirect_proto;
8632 case BPF_FUNC_skb_change_tail:
8633 return &bpf_skb_change_tail_proto;
8634 case BPF_FUNC_skb_change_head:
8635 return &bpf_skb_change_head_proto;
8636 case BPF_FUNC_skb_store_bytes:
8637 return &bpf_skb_store_bytes_proto;
8638 case BPF_FUNC_csum_update:
8639 return &bpf_csum_update_proto;
8640 case BPF_FUNC_csum_level:
8641 return &bpf_csum_level_proto;
8642 case BPF_FUNC_l3_csum_replace:
8643 return &bpf_l3_csum_replace_proto;
8644 case BPF_FUNC_l4_csum_replace:
8645 return &bpf_l4_csum_replace_proto;
8646 case BPF_FUNC_set_hash_invalid:
8647 return &bpf_set_hash_invalid_proto;
8648 case BPF_FUNC_lwt_push_encap:
8649 return &bpf_lwt_xmit_push_encap_proto;
8650 default:
8651 return lwt_out_func_proto(func_id, prog);
8652 }
8653}
8654
8655static const struct bpf_func_proto *
8656lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8657{
8658 switch (func_id) {
8659#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
8660 case BPF_FUNC_lwt_seg6_store_bytes:
8661 return &bpf_lwt_seg6_store_bytes_proto;
8662 case BPF_FUNC_lwt_seg6_action:
8663 return &bpf_lwt_seg6_action_proto;
8664 case BPF_FUNC_lwt_seg6_adjust_srh:
8665 return &bpf_lwt_seg6_adjust_srh_proto;
8666#endif
8667 default:
8668 return lwt_out_func_proto(func_id, prog);
8669 }
8670}
8671
8672static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type,
8673 const struct bpf_prog *prog,
8674 struct bpf_insn_access_aux *info)
8675{
8676 const int size_default = sizeof(__u32);
8677
8678 if (off < 0 || off >= sizeof(struct __sk_buff))
8679 return false;
8680
8681 /* The verifier guarantees that size > 0. */
8682 if (off % size != 0)
8683 return false;
8684
8685 switch (off) {
8686 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
8687 if (off + size > offsetofend(struct __sk_buff, cb[4]))
8688 return false;
8689 break;
8690 case bpf_ctx_range(struct __sk_buff, data):
8691 case bpf_ctx_range(struct __sk_buff, data_meta):
8692 case bpf_ctx_range(struct __sk_buff, data_end):
8693 if (info->is_ldsx || size != size_default)
8694 return false;
8695 break;
8696 case bpf_ctx_range_till(struct __sk_buff, remote_ip6[0], remote_ip6[3]):
8697 case bpf_ctx_range_till(struct __sk_buff, local_ip6[0], local_ip6[3]):
8698 case bpf_ctx_range_till(struct __sk_buff, remote_ip4, remote_ip4):
8699 case bpf_ctx_range_till(struct __sk_buff, local_ip4, local_ip4):
8700 if (size != size_default)
8701 return false;
8702 break;
8703 case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
8704 return false;
8705 case bpf_ctx_range(struct __sk_buff, hwtstamp):
8706 if (type == BPF_WRITE || size != sizeof(__u64))
8707 return false;
8708 break;
8709 case bpf_ctx_range(struct __sk_buff, tstamp):
8710 if (size != sizeof(__u64))
8711 return false;
8712 break;
8713 case bpf_ctx_range_ptr(struct __sk_buff, sk):
8714 if (type == BPF_WRITE || size != sizeof(__u64))
8715 return false;
8716 info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
8717 break;
8718 case offsetof(struct __sk_buff, tstamp_type):
8719 return false;
8720 case offsetofend(struct __sk_buff, tstamp_type) ... offsetof(struct __sk_buff, hwtstamp) - 1:
8721 /* Explicitly prohibit access to padding in __sk_buff. */
8722 return false;
8723 default:
8724 /* Only narrow read access allowed for now. */
8725 if (type == BPF_WRITE) {
8726 if (size != size_default)
8727 return false;
8728 } else {
8729 bpf_ctx_record_field_size(aux: info, size: size_default);
8730 if (!bpf_ctx_narrow_access_ok(off, size, size_default))
8731 return false;
8732 }
8733 }
8734
8735 return true;
8736}
8737
8738static bool sk_filter_is_valid_access(int off, int size,
8739 enum bpf_access_type type,
8740 const struct bpf_prog *prog,
8741 struct bpf_insn_access_aux *info)
8742{
8743 switch (off) {
8744 case bpf_ctx_range(struct __sk_buff, tc_classid):
8745 case bpf_ctx_range(struct __sk_buff, data):
8746 case bpf_ctx_range(struct __sk_buff, data_meta):
8747 case bpf_ctx_range(struct __sk_buff, data_end):
8748 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
8749 case bpf_ctx_range(struct __sk_buff, tstamp):
8750 case bpf_ctx_range(struct __sk_buff, wire_len):
8751 case bpf_ctx_range(struct __sk_buff, hwtstamp):
8752 return false;
8753 }
8754
8755 if (type == BPF_WRITE) {
8756 switch (off) {
8757 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
8758 break;
8759 default:
8760 return false;
8761 }
8762 }
8763
8764 return bpf_skb_is_valid_access(off, size, type, prog, info);
8765}
8766
8767static bool cg_skb_is_valid_access(int off, int size,
8768 enum bpf_access_type type,
8769 const struct bpf_prog *prog,
8770 struct bpf_insn_access_aux *info)
8771{
8772 switch (off) {
8773 case bpf_ctx_range(struct __sk_buff, tc_classid):
8774 case bpf_ctx_range(struct __sk_buff, data_meta):
8775 case bpf_ctx_range(struct __sk_buff, wire_len):
8776 return false;
8777 case bpf_ctx_range(struct __sk_buff, data):
8778 case bpf_ctx_range(struct __sk_buff, data_end):
8779 if (!bpf_token_capable(token: prog->aux->token, CAP_BPF))
8780 return false;
8781 break;
8782 }
8783
8784 if (type == BPF_WRITE) {
8785 switch (off) {
8786 case bpf_ctx_range(struct __sk_buff, mark):
8787 case bpf_ctx_range(struct __sk_buff, priority):
8788 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
8789 break;
8790 case bpf_ctx_range(struct __sk_buff, tstamp):
8791 if (!bpf_token_capable(token: prog->aux->token, CAP_BPF))
8792 return false;
8793 break;
8794 default:
8795 return false;
8796 }
8797 }
8798
8799 switch (off) {
8800 case bpf_ctx_range(struct __sk_buff, data):
8801 info->reg_type = PTR_TO_PACKET;
8802 break;
8803 case bpf_ctx_range(struct __sk_buff, data_end):
8804 info->reg_type = PTR_TO_PACKET_END;
8805 break;
8806 }
8807
8808 return bpf_skb_is_valid_access(off, size, type, prog, info);
8809}
8810
8811static bool lwt_is_valid_access(int off, int size,
8812 enum bpf_access_type type,
8813 const struct bpf_prog *prog,
8814 struct bpf_insn_access_aux *info)
8815{
8816 switch (off) {
8817 case bpf_ctx_range(struct __sk_buff, tc_classid):
8818 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
8819 case bpf_ctx_range(struct __sk_buff, data_meta):
8820 case bpf_ctx_range(struct __sk_buff, tstamp):
8821 case bpf_ctx_range(struct __sk_buff, wire_len):
8822 case bpf_ctx_range(struct __sk_buff, hwtstamp):
8823 return false;
8824 }
8825
8826 if (type == BPF_WRITE) {
8827 switch (off) {
8828 case bpf_ctx_range(struct __sk_buff, mark):
8829 case bpf_ctx_range(struct __sk_buff, priority):
8830 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
8831 break;
8832 default:
8833 return false;
8834 }
8835 }
8836
8837 switch (off) {
8838 case bpf_ctx_range(struct __sk_buff, data):
8839 info->reg_type = PTR_TO_PACKET;
8840 break;
8841 case bpf_ctx_range(struct __sk_buff, data_end):
8842 info->reg_type = PTR_TO_PACKET_END;
8843 break;
8844 }
8845
8846 return bpf_skb_is_valid_access(off, size, type, prog, info);
8847}
8848
8849/* Attach type specific accesses */
8850static bool __sock_filter_check_attach_type(int off,
8851 enum bpf_access_type access_type,
8852 enum bpf_attach_type attach_type)
8853{
8854 switch (off) {
8855 case offsetof(struct bpf_sock, bound_dev_if):
8856 case offsetof(struct bpf_sock, mark):
8857 case offsetof(struct bpf_sock, priority):
8858 switch (attach_type) {
8859 case BPF_CGROUP_INET_SOCK_CREATE:
8860 case BPF_CGROUP_INET_SOCK_RELEASE:
8861 goto full_access;
8862 default:
8863 return false;
8864 }
8865 case bpf_ctx_range(struct bpf_sock, src_ip4):
8866 switch (attach_type) {
8867 case BPF_CGROUP_INET4_POST_BIND:
8868 goto read_only;
8869 default:
8870 return false;
8871 }
8872 case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
8873 switch (attach_type) {
8874 case BPF_CGROUP_INET6_POST_BIND:
8875 goto read_only;
8876 default:
8877 return false;
8878 }
8879 case bpf_ctx_range(struct bpf_sock, src_port):
8880 switch (attach_type) {
8881 case BPF_CGROUP_INET4_POST_BIND:
8882 case BPF_CGROUP_INET6_POST_BIND:
8883 goto read_only;
8884 default:
8885 return false;
8886 }
8887 }
8888read_only:
8889 return access_type == BPF_READ;
8890full_access:
8891 return true;
8892}
8893
8894bool bpf_sock_common_is_valid_access(int off, int size,
8895 enum bpf_access_type type,
8896 struct bpf_insn_access_aux *info)
8897{
8898 switch (off) {
8899 case bpf_ctx_range_till(struct bpf_sock, type, priority):
8900 return false;
8901 default:
8902 return bpf_sock_is_valid_access(off, size, type, info);
8903 }
8904}
8905
8906bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
8907 struct bpf_insn_access_aux *info)
8908{
8909 const int size_default = sizeof(__u32);
8910 int field_size;
8911
8912 if (off < 0 || off >= sizeof(struct bpf_sock))
8913 return false;
8914 if (off % size != 0)
8915 return false;
8916
8917 switch (off) {
8918 case offsetof(struct bpf_sock, state):
8919 case offsetof(struct bpf_sock, family):
8920 case offsetof(struct bpf_sock, type):
8921 case offsetof(struct bpf_sock, protocol):
8922 case offsetof(struct bpf_sock, src_port):
8923 case offsetof(struct bpf_sock, rx_queue_mapping):
8924 case bpf_ctx_range(struct bpf_sock, src_ip4):
8925 case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
8926 case bpf_ctx_range(struct bpf_sock, dst_ip4):
8927 case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
8928 bpf_ctx_record_field_size(aux: info, size: size_default);
8929 return bpf_ctx_narrow_access_ok(off, size, size_default);
8930 case bpf_ctx_range(struct bpf_sock, dst_port):
8931 field_size = size == size_default ?
8932 size_default : sizeof_field(struct bpf_sock, dst_port);
8933 bpf_ctx_record_field_size(aux: info, size: field_size);
8934 return bpf_ctx_narrow_access_ok(off, size, size_default: field_size);
8935 case offsetofend(struct bpf_sock, dst_port) ...
8936 offsetof(struct bpf_sock, dst_ip4) - 1:
8937 return false;
8938 }
8939
8940 return size == size_default;
8941}
8942
8943static bool sock_filter_is_valid_access(int off, int size,
8944 enum bpf_access_type type,
8945 const struct bpf_prog *prog,
8946 struct bpf_insn_access_aux *info)
8947{
8948 if (!bpf_sock_is_valid_access(off, size, type, info))
8949 return false;
8950 return __sock_filter_check_attach_type(off, access_type: type,
8951 attach_type: prog->expected_attach_type);
8952}
8953
8954static int bpf_noop_prologue(struct bpf_insn *insn_buf, bool direct_write,
8955 const struct bpf_prog *prog)
8956{
8957 /* Neither direct read nor direct write requires any preliminary
8958 * action.
8959 */
8960 return 0;
8961}
8962
8963static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write,
8964 const struct bpf_prog *prog, int drop_verdict)
8965{
8966 struct bpf_insn *insn = insn_buf;
8967
8968 if (!direct_write)
8969 return 0;
8970
8971 /* if (!skb->cloned)
8972 * goto start;
8973 *
8974 * (Fast-path, otherwise approximation that we might be
8975 * a clone, do the rest in helper.)
8976 */
8977 *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET);
8978 *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK);
8979 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7);
8980
8981 /* ret = bpf_skb_pull_data(skb, 0); */
8982 *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
8983 *insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2);
8984 *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
8985 BPF_FUNC_skb_pull_data);
8986 /* if (!ret)
8987 * goto restore;
8988 * return TC_ACT_SHOT;
8989 */
8990 *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2);
8991 *insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, drop_verdict);
8992 *insn++ = BPF_EXIT_INSN();
8993
8994 /* restore: */
8995 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
8996 /* start: */
8997 *insn++ = prog->insnsi[0];
8998
8999 return insn - insn_buf;
9000}
9001
9002static int bpf_gen_ld_abs(const struct bpf_insn *orig,
9003 struct bpf_insn *insn_buf)
9004{
9005 bool indirect = BPF_MODE(orig->code) == BPF_IND;
9006 struct bpf_insn *insn = insn_buf;
9007
9008 if (!indirect) {
9009 *insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm);
9010 } else {
9011 *insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg);
9012 if (orig->imm)
9013 *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm);
9014 }
9015 /* We're guaranteed here that CTX is in R6. */
9016 *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);
9017
9018 switch (BPF_SIZE(orig->code)) {
9019 case BPF_B:
9020 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache);
9021 break;
9022 case BPF_H:
9023 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache);
9024 break;
9025 case BPF_W:
9026 *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache);
9027 break;
9028 }
9029
9030 *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2);
9031 *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
9032 *insn++ = BPF_EXIT_INSN();
9033
9034 return insn - insn_buf;
9035}
9036
9037static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write,
9038 const struct bpf_prog *prog)
9039{
9040 return bpf_unclone_prologue(insn_buf, direct_write, prog, TC_ACT_SHOT);
9041}
9042
9043static bool tc_cls_act_is_valid_access(int off, int size,
9044 enum bpf_access_type type,
9045 const struct bpf_prog *prog,
9046 struct bpf_insn_access_aux *info)
9047{
9048 if (type == BPF_WRITE) {
9049 switch (off) {
9050 case bpf_ctx_range(struct __sk_buff, mark):
9051 case bpf_ctx_range(struct __sk_buff, tc_index):
9052 case bpf_ctx_range(struct __sk_buff, priority):
9053 case bpf_ctx_range(struct __sk_buff, tc_classid):
9054 case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
9055 case bpf_ctx_range(struct __sk_buff, tstamp):
9056 case bpf_ctx_range(struct __sk_buff, queue_mapping):
9057 break;
9058 default:
9059 return false;
9060 }
9061 }
9062
9063 switch (off) {
9064 case bpf_ctx_range(struct __sk_buff, data):
9065 info->reg_type = PTR_TO_PACKET;
9066 break;
9067 case bpf_ctx_range(struct __sk_buff, data_meta):
9068 info->reg_type = PTR_TO_PACKET_META;
9069 break;
9070 case bpf_ctx_range(struct __sk_buff, data_end):
9071 info->reg_type = PTR_TO_PACKET_END;
9072 break;
9073 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
9074 return false;
9075 case offsetof(struct __sk_buff, tstamp_type):
9076 /* The convert_ctx_access() on reading and writing
9077 * __sk_buff->tstamp depends on whether the bpf prog
9078 * has used __sk_buff->tstamp_type or not.
9079 * Thus, we need to set prog->tstamp_type_access
9080 * earlier during is_valid_access() here.
9081 */
9082 ((struct bpf_prog *)prog)->tstamp_type_access = 1;
9083 return size == sizeof(__u8);
9084 }
9085
9086 return bpf_skb_is_valid_access(off, size, type, prog, info);
9087}
9088
9089DEFINE_MUTEX(nf_conn_btf_access_lock);
9090EXPORT_SYMBOL_GPL(nf_conn_btf_access_lock);
9091
9092int (*nfct_btf_struct_access)(struct bpf_verifier_log *log,
9093 const struct bpf_reg_state *reg,
9094 int off, int size);
9095EXPORT_SYMBOL_GPL(nfct_btf_struct_access);
9096
9097static int tc_cls_act_btf_struct_access(struct bpf_verifier_log *log,
9098 const struct bpf_reg_state *reg,
9099 int off, int size)
9100{
9101 int ret = -EACCES;
9102
9103 mutex_lock(lock: &nf_conn_btf_access_lock);
9104 if (nfct_btf_struct_access)
9105 ret = nfct_btf_struct_access(log, reg, off, size);
9106 mutex_unlock(lock: &nf_conn_btf_access_lock);
9107
9108 return ret;
9109}
9110
9111static bool __is_valid_xdp_access(int off, int size)
9112{
9113 if (off < 0 || off >= sizeof(struct xdp_md))
9114 return false;
9115 if (off % size != 0)
9116 return false;
9117 if (size != sizeof(__u32))
9118 return false;
9119
9120 return true;
9121}
9122
9123static bool xdp_is_valid_access(int off, int size,
9124 enum bpf_access_type type,
9125 const struct bpf_prog *prog,
9126 struct bpf_insn_access_aux *info)
9127{
9128 if (prog->expected_attach_type != BPF_XDP_DEVMAP) {
9129 switch (off) {
9130 case offsetof(struct xdp_md, egress_ifindex):
9131 return false;
9132 }
9133 }
9134
9135 if (type == BPF_WRITE) {
9136 if (bpf_prog_is_offloaded(aux: prog->aux)) {
9137 switch (off) {
9138 case offsetof(struct xdp_md, rx_queue_index):
9139 return __is_valid_xdp_access(off, size);
9140 }
9141 }
9142 return false;
9143 } else {
9144 switch (off) {
9145 case offsetof(struct xdp_md, data_meta):
9146 case offsetof(struct xdp_md, data):
9147 case offsetof(struct xdp_md, data_end):
9148 if (info->is_ldsx)
9149 return false;
9150 }
9151 }
9152
9153 switch (off) {
9154 case offsetof(struct xdp_md, data):
9155 info->reg_type = PTR_TO_PACKET;
9156 break;
9157 case offsetof(struct xdp_md, data_meta):
9158 info->reg_type = PTR_TO_PACKET_META;
9159 break;
9160 case offsetof(struct xdp_md, data_end):
9161 info->reg_type = PTR_TO_PACKET_END;
9162 break;
9163 }
9164
9165 return __is_valid_xdp_access(off, size);
9166}
9167
9168void bpf_warn_invalid_xdp_action(const struct net_device *dev,
9169 const struct bpf_prog *prog, u32 act)
9170{
9171 const u32 act_max = XDP_REDIRECT;
9172
9173 pr_warn_once("%s XDP return value %u on prog %s (id %d) dev %s, expect packet loss!\n",
9174 act > act_max ? "Illegal" : "Driver unsupported",
9175 act, prog->aux->name, prog->aux->id, dev ? dev->name : "N/A");
9176}
9177EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
9178
9179static int xdp_btf_struct_access(struct bpf_verifier_log *log,
9180 const struct bpf_reg_state *reg,
9181 int off, int size)
9182{
9183 int ret = -EACCES;
9184
9185 mutex_lock(lock: &nf_conn_btf_access_lock);
9186 if (nfct_btf_struct_access)
9187 ret = nfct_btf_struct_access(log, reg, off, size);
9188 mutex_unlock(lock: &nf_conn_btf_access_lock);
9189
9190 return ret;
9191}
9192
9193static bool sock_addr_is_valid_access(int off, int size,
9194 enum bpf_access_type type,
9195 const struct bpf_prog *prog,
9196 struct bpf_insn_access_aux *info)
9197{
9198 const int size_default = sizeof(__u32);
9199
9200 if (off < 0 || off >= sizeof(struct bpf_sock_addr))
9201 return false;
9202 if (off % size != 0)
9203 return false;
9204
9205 /* Disallow access to fields not belonging to the attach type's address
9206 * family.
9207 */
9208 switch (off) {
9209 case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
9210 switch (prog->expected_attach_type) {
9211 case BPF_CGROUP_INET4_BIND:
9212 case BPF_CGROUP_INET4_CONNECT:
9213 case BPF_CGROUP_INET4_GETPEERNAME:
9214 case BPF_CGROUP_INET4_GETSOCKNAME:
9215 case BPF_CGROUP_UDP4_SENDMSG:
9216 case BPF_CGROUP_UDP4_RECVMSG:
9217 break;
9218 default:
9219 return false;
9220 }
9221 break;
9222 case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
9223 switch (prog->expected_attach_type) {
9224 case BPF_CGROUP_INET6_BIND:
9225 case BPF_CGROUP_INET6_CONNECT:
9226 case BPF_CGROUP_INET6_GETPEERNAME:
9227 case BPF_CGROUP_INET6_GETSOCKNAME:
9228 case BPF_CGROUP_UDP6_SENDMSG:
9229 case BPF_CGROUP_UDP6_RECVMSG:
9230 break;
9231 default:
9232 return false;
9233 }
9234 break;
9235 case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
9236 switch (prog->expected_attach_type) {
9237 case BPF_CGROUP_UDP4_SENDMSG:
9238 break;
9239 default:
9240 return false;
9241 }
9242 break;
9243 case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
9244 msg_src_ip6[3]):
9245 switch (prog->expected_attach_type) {
9246 case BPF_CGROUP_UDP6_SENDMSG:
9247 break;
9248 default:
9249 return false;
9250 }
9251 break;
9252 }
9253
9254 switch (off) {
9255 case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
9256 case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
9257 case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
9258 case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
9259 msg_src_ip6[3]):
9260 case bpf_ctx_range(struct bpf_sock_addr, user_port):
9261 if (type == BPF_READ) {
9262 bpf_ctx_record_field_size(aux: info, size: size_default);
9263
9264 if (bpf_ctx_wide_access_ok(off, size,
9265 struct bpf_sock_addr,
9266 user_ip6))
9267 return true;
9268
9269 if (bpf_ctx_wide_access_ok(off, size,
9270 struct bpf_sock_addr,
9271 msg_src_ip6))
9272 return true;
9273
9274 if (!bpf_ctx_narrow_access_ok(off, size, size_default))
9275 return false;
9276 } else {
9277 if (bpf_ctx_wide_access_ok(off, size,
9278 struct bpf_sock_addr,
9279 user_ip6))
9280 return true;
9281
9282 if (bpf_ctx_wide_access_ok(off, size,
9283 struct bpf_sock_addr,
9284 msg_src_ip6))
9285 return true;
9286
9287 if (size != size_default)
9288 return false;
9289 }
9290 break;
9291 case bpf_ctx_range_ptr(struct bpf_sock_addr, sk):
9292 if (type != BPF_READ)
9293 return false;
9294 if (size != sizeof(__u64))
9295 return false;
9296 info->reg_type = PTR_TO_SOCKET;
9297 break;
9298 case bpf_ctx_range(struct bpf_sock_addr, user_family):
9299 case bpf_ctx_range(struct bpf_sock_addr, family):
9300 case bpf_ctx_range(struct bpf_sock_addr, type):
9301 case bpf_ctx_range(struct bpf_sock_addr, protocol):
9302 if (type != BPF_READ)
9303 return false;
9304 if (size != size_default)
9305 return false;
9306 break;
9307 default:
9308 return false;
9309 }
9310
9311 return true;
9312}
9313
9314static bool sock_ops_is_valid_access(int off, int size,
9315 enum bpf_access_type type,
9316 const struct bpf_prog *prog,
9317 struct bpf_insn_access_aux *info)
9318{
9319 const int size_default = sizeof(__u32);
9320
9321 if (off < 0 || off >= sizeof(struct bpf_sock_ops))
9322 return false;
9323
9324 /* The verifier guarantees that size > 0. */
9325 if (off % size != 0)
9326 return false;
9327
9328 if (type == BPF_WRITE) {
9329 switch (off) {
9330 case offsetof(struct bpf_sock_ops, reply):
9331 case offsetof(struct bpf_sock_ops, sk_txhash):
9332 if (size != size_default)
9333 return false;
9334 break;
9335 default:
9336 return false;
9337 }
9338 } else {
9339 switch (off) {
9340 case bpf_ctx_range_till(struct bpf_sock_ops, bytes_received,
9341 bytes_acked):
9342 if (size != sizeof(__u64))
9343 return false;
9344 break;
9345 case bpf_ctx_range_ptr(struct bpf_sock_ops, sk):
9346 if (size != sizeof(__u64))
9347 return false;
9348 info->reg_type = PTR_TO_SOCKET_OR_NULL;
9349 break;
9350 case bpf_ctx_range_ptr(struct bpf_sock_ops, skb_data):
9351 if (size != sizeof(__u64))
9352 return false;
9353 info->reg_type = PTR_TO_PACKET;
9354 break;
9355 case bpf_ctx_range_ptr(struct bpf_sock_ops, skb_data_end):
9356 if (size != sizeof(__u64))
9357 return false;
9358 info->reg_type = PTR_TO_PACKET_END;
9359 break;
9360 case offsetof(struct bpf_sock_ops, skb_tcp_flags):
9361 bpf_ctx_record_field_size(aux: info, size: size_default);
9362 return bpf_ctx_narrow_access_ok(off, size,
9363 size_default);
9364 case bpf_ctx_range(struct bpf_sock_ops, skb_hwtstamp):
9365 if (size != sizeof(__u64))
9366 return false;
9367 break;
9368 default:
9369 if (size != size_default)
9370 return false;
9371 break;
9372 }
9373 }
9374
9375 return true;
9376}
9377
9378static int sk_skb_prologue(struct bpf_insn *insn_buf, bool direct_write,
9379 const struct bpf_prog *prog)
9380{
9381 return bpf_unclone_prologue(insn_buf, direct_write, prog, drop_verdict: SK_DROP);
9382}
9383
9384static bool sk_skb_is_valid_access(int off, int size,
9385 enum bpf_access_type type,
9386 const struct bpf_prog *prog,
9387 struct bpf_insn_access_aux *info)
9388{
9389 switch (off) {
9390 case bpf_ctx_range(struct __sk_buff, tc_classid):
9391 case bpf_ctx_range(struct __sk_buff, data_meta):
9392 case bpf_ctx_range(struct __sk_buff, tstamp):
9393 case bpf_ctx_range(struct __sk_buff, wire_len):
9394 case bpf_ctx_range(struct __sk_buff, hwtstamp):
9395 return false;
9396 }
9397
9398 if (type == BPF_WRITE) {
9399 switch (off) {
9400 case bpf_ctx_range(struct __sk_buff, tc_index):
9401 case bpf_ctx_range(struct __sk_buff, priority):
9402 break;
9403 default:
9404 return false;
9405 }
9406 }
9407
9408 switch (off) {
9409 case bpf_ctx_range(struct __sk_buff, mark):
9410 return false;
9411 case bpf_ctx_range(struct __sk_buff, data):
9412 info->reg_type = PTR_TO_PACKET;
9413 break;
9414 case bpf_ctx_range(struct __sk_buff, data_end):
9415 info->reg_type = PTR_TO_PACKET_END;
9416 break;
9417 }
9418
9419 return bpf_skb_is_valid_access(off, size, type, prog, info);
9420}
9421
9422static bool sk_msg_is_valid_access(int off, int size,
9423 enum bpf_access_type type,
9424 const struct bpf_prog *prog,
9425 struct bpf_insn_access_aux *info)
9426{
9427 if (type == BPF_WRITE)
9428 return false;
9429
9430 if (off % size != 0)
9431 return false;
9432
9433 switch (off) {
9434 case bpf_ctx_range_ptr(struct sk_msg_md, data):
9435 info->reg_type = PTR_TO_PACKET;
9436 if (size != sizeof(__u64))
9437 return false;
9438 break;
9439 case bpf_ctx_range_ptr(struct sk_msg_md, data_end):
9440 info->reg_type = PTR_TO_PACKET_END;
9441 if (size != sizeof(__u64))
9442 return false;
9443 break;
9444 case bpf_ctx_range_ptr(struct sk_msg_md, sk):
9445 if (size != sizeof(__u64))
9446 return false;
9447 info->reg_type = PTR_TO_SOCKET;
9448 break;
9449 case bpf_ctx_range(struct sk_msg_md, family):
9450 case bpf_ctx_range(struct sk_msg_md, remote_ip4):
9451 case bpf_ctx_range(struct sk_msg_md, local_ip4):
9452 case bpf_ctx_range_till(struct sk_msg_md, remote_ip6[0], remote_ip6[3]):
9453 case bpf_ctx_range_till(struct sk_msg_md, local_ip6[0], local_ip6[3]):
9454 case bpf_ctx_range(struct sk_msg_md, remote_port):
9455 case bpf_ctx_range(struct sk_msg_md, local_port):
9456 case bpf_ctx_range(struct sk_msg_md, size):
9457 if (size != sizeof(__u32))
9458 return false;
9459 break;
9460 default:
9461 return false;
9462 }
9463 return true;
9464}
9465
9466static bool flow_dissector_is_valid_access(int off, int size,
9467 enum bpf_access_type type,
9468 const struct bpf_prog *prog,
9469 struct bpf_insn_access_aux *info)
9470{
9471 const int size_default = sizeof(__u32);
9472
9473 if (off < 0 || off >= sizeof(struct __sk_buff))
9474 return false;
9475
9476 if (off % size != 0)
9477 return false;
9478
9479 if (type == BPF_WRITE)
9480 return false;
9481
9482 switch (off) {
9483 case bpf_ctx_range(struct __sk_buff, data):
9484 if (info->is_ldsx || size != size_default)
9485 return false;
9486 info->reg_type = PTR_TO_PACKET;
9487 return true;
9488 case bpf_ctx_range(struct __sk_buff, data_end):
9489 if (info->is_ldsx || size != size_default)
9490 return false;
9491 info->reg_type = PTR_TO_PACKET_END;
9492 return true;
9493 case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
9494 if (size != sizeof(__u64))
9495 return false;
9496 info->reg_type = PTR_TO_FLOW_KEYS;
9497 return true;
9498 default:
9499 return false;
9500 }
9501}
9502
9503static u32 flow_dissector_convert_ctx_access(enum bpf_access_type type,
9504 const struct bpf_insn *si,
9505 struct bpf_insn *insn_buf,
9506 struct bpf_prog *prog,
9507 u32 *target_size)
9508
9509{
9510 struct bpf_insn *insn = insn_buf;
9511
9512 switch (si->off) {
9513 case offsetof(struct __sk_buff, data):
9514 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data),
9515 si->dst_reg, si->src_reg,
9516 offsetof(struct bpf_flow_dissector, data));
9517 break;
9518
9519 case offsetof(struct __sk_buff, data_end):
9520 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data_end),
9521 si->dst_reg, si->src_reg,
9522 offsetof(struct bpf_flow_dissector, data_end));
9523 break;
9524
9525 case offsetof(struct __sk_buff, flow_keys):
9526 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, flow_keys),
9527 si->dst_reg, si->src_reg,
9528 offsetof(struct bpf_flow_dissector, flow_keys));
9529 break;
9530 }
9531
9532 return insn - insn_buf;
9533}
9534
9535static struct bpf_insn *bpf_convert_tstamp_type_read(const struct bpf_insn *si,
9536 struct bpf_insn *insn)
9537{
9538 __u8 value_reg = si->dst_reg;
9539 __u8 skb_reg = si->src_reg;
9540 BUILD_BUG_ON(__SKB_CLOCK_MAX != (int)BPF_SKB_CLOCK_TAI);
9541 BUILD_BUG_ON(SKB_CLOCK_REALTIME != (int)BPF_SKB_CLOCK_REALTIME);
9542 BUILD_BUG_ON(SKB_CLOCK_MONOTONIC != (int)BPF_SKB_CLOCK_MONOTONIC);
9543 BUILD_BUG_ON(SKB_CLOCK_TAI != (int)BPF_SKB_CLOCK_TAI);
9544 *insn++ = BPF_LDX_MEM(BPF_B, value_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
9545 *insn++ = BPF_ALU32_IMM(BPF_AND, value_reg, SKB_TSTAMP_TYPE_MASK);
9546#ifdef __BIG_ENDIAN_BITFIELD
9547 *insn++ = BPF_ALU32_IMM(BPF_RSH, value_reg, SKB_TSTAMP_TYPE_RSHIFT);
9548#else
9549 BUILD_BUG_ON(!(SKB_TSTAMP_TYPE_MASK & 0x1));
9550#endif
9551
9552 return insn;
9553}
9554
9555static struct bpf_insn *bpf_convert_shinfo_access(__u8 dst_reg, __u8 skb_reg,
9556 struct bpf_insn *insn)
9557{
9558 /* si->dst_reg = skb_shinfo(SKB); */
9559#ifdef NET_SKBUFF_DATA_USES_OFFSET
9560 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
9561 BPF_REG_AX, skb_reg,
9562 offsetof(struct sk_buff, end));
9563 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, head),
9564 dst_reg, skb_reg,
9565 offsetof(struct sk_buff, head));
9566 *insn++ = BPF_ALU64_REG(BPF_ADD, dst_reg, BPF_REG_AX);
9567#else
9568 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
9569 dst_reg, skb_reg,
9570 offsetof(struct sk_buff, end));
9571#endif
9572
9573 return insn;
9574}
9575
9576static struct bpf_insn *bpf_convert_tstamp_read(const struct bpf_prog *prog,
9577 const struct bpf_insn *si,
9578 struct bpf_insn *insn)
9579{
9580 __u8 value_reg = si->dst_reg;
9581 __u8 skb_reg = si->src_reg;
9582
9583#ifdef CONFIG_NET_XGRESS
9584 /* If the tstamp_type is read,
9585 * the bpf prog is aware the tstamp could have delivery time.
9586 * Thus, read skb->tstamp as is if tstamp_type_access is true.
9587 */
9588 if (!prog->tstamp_type_access) {
9589 /* AX is needed because src_reg and dst_reg could be the same */
9590 __u8 tmp_reg = BPF_REG_AX;
9591
9592 *insn++ = BPF_LDX_MEM(BPF_B, tmp_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
9593 /* check if ingress mask bits is set */
9594 *insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, TC_AT_INGRESS_MASK, 1);
9595 *insn++ = BPF_JMP_A(4);
9596 *insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, SKB_TSTAMP_TYPE_MASK, 1);
9597 *insn++ = BPF_JMP_A(2);
9598 /* skb->tc_at_ingress && skb->tstamp_type,
9599 * read 0 as the (rcv) timestamp.
9600 */
9601 *insn++ = BPF_MOV64_IMM(value_reg, 0);
9602 *insn++ = BPF_JMP_A(1);
9603 }
9604#endif
9605
9606 *insn++ = BPF_LDX_MEM(BPF_DW, value_reg, skb_reg,
9607 offsetof(struct sk_buff, tstamp));
9608 return insn;
9609}
9610
9611static struct bpf_insn *bpf_convert_tstamp_write(const struct bpf_prog *prog,
9612 const struct bpf_insn *si,
9613 struct bpf_insn *insn)
9614{
9615 __u8 value_reg = si->src_reg;
9616 __u8 skb_reg = si->dst_reg;
9617
9618#ifdef CONFIG_NET_XGRESS
9619 /* If the tstamp_type is read,
9620 * the bpf prog is aware the tstamp could have delivery time.
9621 * Thus, write skb->tstamp as is if tstamp_type_access is true.
9622 * Otherwise, writing at ingress will have to clear the
9623 * skb->tstamp_type bit also.
9624 */
9625 if (!prog->tstamp_type_access) {
9626 __u8 tmp_reg = BPF_REG_AX;
9627
9628 *insn++ = BPF_LDX_MEM(BPF_B, tmp_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
9629 /* Writing __sk_buff->tstamp as ingress, goto <clear> */
9630 *insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, TC_AT_INGRESS_MASK, 1);
9631 /* goto <store> */
9632 *insn++ = BPF_JMP_A(2);
9633 /* <clear>: skb->tstamp_type */
9634 *insn++ = BPF_ALU32_IMM(BPF_AND, tmp_reg, ~SKB_TSTAMP_TYPE_MASK);
9635 *insn++ = BPF_STX_MEM(BPF_B, skb_reg, tmp_reg, SKB_BF_MONO_TC_OFFSET);
9636 }
9637#endif
9638
9639 /* <store>: skb->tstamp = tstamp */
9640 *insn++ = BPF_RAW_INSN(BPF_CLASS(si->code) | BPF_DW | BPF_MEM,
9641 skb_reg, value_reg, offsetof(struct sk_buff, tstamp), si->imm);
9642 return insn;
9643}
9644
9645#define BPF_EMIT_STORE(size, si, off) \
9646 BPF_RAW_INSN(BPF_CLASS((si)->code) | (size) | BPF_MEM, \
9647 (si)->dst_reg, (si)->src_reg, (off), (si)->imm)
9648
9649static u32 bpf_convert_ctx_access(enum bpf_access_type type,
9650 const struct bpf_insn *si,
9651 struct bpf_insn *insn_buf,
9652 struct bpf_prog *prog, u32 *target_size)
9653{
9654 struct bpf_insn *insn = insn_buf;
9655 int off;
9656
9657 switch (si->off) {
9658 case offsetof(struct __sk_buff, len):
9659 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9660 bpf_target_off(struct sk_buff, len, 4,
9661 target_size));
9662 break;
9663
9664 case offsetof(struct __sk_buff, protocol):
9665 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
9666 bpf_target_off(struct sk_buff, protocol, 2,
9667 target_size));
9668 break;
9669
9670 case offsetof(struct __sk_buff, vlan_proto):
9671 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
9672 bpf_target_off(struct sk_buff, vlan_proto, 2,
9673 target_size));
9674 break;
9675
9676 case offsetof(struct __sk_buff, priority):
9677 if (type == BPF_WRITE)
9678 *insn++ = BPF_EMIT_STORE(BPF_W, si,
9679 bpf_target_off(struct sk_buff, priority, 4,
9680 target_size));
9681 else
9682 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9683 bpf_target_off(struct sk_buff, priority, 4,
9684 target_size));
9685 break;
9686
9687 case offsetof(struct __sk_buff, ingress_ifindex):
9688 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9689 bpf_target_off(struct sk_buff, skb_iif, 4,
9690 target_size));
9691 break;
9692
9693 case offsetof(struct __sk_buff, ifindex):
9694 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
9695 si->dst_reg, si->src_reg,
9696 offsetof(struct sk_buff, dev));
9697 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
9698 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9699 bpf_target_off(struct net_device, ifindex, 4,
9700 target_size));
9701 break;
9702
9703 case offsetof(struct __sk_buff, hash):
9704 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9705 bpf_target_off(struct sk_buff, hash, 4,
9706 target_size));
9707 break;
9708
9709 case offsetof(struct __sk_buff, mark):
9710 if (type == BPF_WRITE)
9711 *insn++ = BPF_EMIT_STORE(BPF_W, si,
9712 bpf_target_off(struct sk_buff, mark, 4,
9713 target_size));
9714 else
9715 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9716 bpf_target_off(struct sk_buff, mark, 4,
9717 target_size));
9718 break;
9719
9720 case offsetof(struct __sk_buff, pkt_type):
9721 *target_size = 1;
9722 *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg,
9723 PKT_TYPE_OFFSET);
9724 *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX);
9725#ifdef __BIG_ENDIAN_BITFIELD
9726 *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5);
9727#endif
9728 break;
9729
9730 case offsetof(struct __sk_buff, queue_mapping):
9731 if (type == BPF_WRITE) {
9732 u32 offset = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
9733
9734 if (BPF_CLASS(si->code) == BPF_ST && si->imm >= NO_QUEUE_MAPPING) {
9735 *insn++ = BPF_JMP_A(0); /* noop */
9736 break;
9737 }
9738
9739 if (BPF_CLASS(si->code) == BPF_STX)
9740 *insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
9741 *insn++ = BPF_EMIT_STORE(BPF_H, si, offset);
9742 } else {
9743 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
9744 bpf_target_off(struct sk_buff,
9745 queue_mapping,
9746 2, target_size));
9747 }
9748 break;
9749
9750 case offsetof(struct __sk_buff, vlan_present):
9751 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9752 bpf_target_off(struct sk_buff,
9753 vlan_all, 4, target_size));
9754 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
9755 *insn++ = BPF_ALU32_IMM(BPF_MOV, si->dst_reg, 1);
9756 break;
9757
9758 case offsetof(struct __sk_buff, vlan_tci):
9759 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
9760 bpf_target_off(struct sk_buff, vlan_tci, 2,
9761 target_size));
9762 break;
9763
9764 case offsetof(struct __sk_buff, cb[0]) ...
9765 offsetofend(struct __sk_buff, cb[4]) - 1:
9766 BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, data) < 20);
9767 BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
9768 offsetof(struct qdisc_skb_cb, data)) %
9769 sizeof(__u64));
9770
9771 prog->cb_access = 1;
9772 off = si->off;
9773 off -= offsetof(struct __sk_buff, cb[0]);
9774 off += offsetof(struct sk_buff, cb);
9775 off += offsetof(struct qdisc_skb_cb, data);
9776 if (type == BPF_WRITE)
9777 *insn++ = BPF_EMIT_STORE(BPF_SIZE(si->code), si, off);
9778 else
9779 *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
9780 si->src_reg, off);
9781 break;
9782
9783 case offsetof(struct __sk_buff, tc_classid):
9784 BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, tc_classid) != 2);
9785
9786 off = si->off;
9787 off -= offsetof(struct __sk_buff, tc_classid);
9788 off += offsetof(struct sk_buff, cb);
9789 off += offsetof(struct qdisc_skb_cb, tc_classid);
9790 *target_size = 2;
9791 if (type == BPF_WRITE)
9792 *insn++ = BPF_EMIT_STORE(BPF_H, si, off);
9793 else
9794 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg,
9795 si->src_reg, off);
9796 break;
9797
9798 case offsetof(struct __sk_buff, data):
9799 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
9800 si->dst_reg, si->src_reg,
9801 offsetof(struct sk_buff, data));
9802 break;
9803
9804 case offsetof(struct __sk_buff, data_meta):
9805 off = si->off;
9806 off -= offsetof(struct __sk_buff, data_meta);
9807 off += offsetof(struct sk_buff, cb);
9808 off += offsetof(struct bpf_skb_data_end, data_meta);
9809 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
9810 si->src_reg, off);
9811 break;
9812
9813 case offsetof(struct __sk_buff, data_end):
9814 off = si->off;
9815 off -= offsetof(struct __sk_buff, data_end);
9816 off += offsetof(struct sk_buff, cb);
9817 off += offsetof(struct bpf_skb_data_end, data_end);
9818 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
9819 si->src_reg, off);
9820 break;
9821
9822 case offsetof(struct __sk_buff, tc_index):
9823#ifdef CONFIG_NET_SCHED
9824 if (type == BPF_WRITE)
9825 *insn++ = BPF_EMIT_STORE(BPF_H, si,
9826 bpf_target_off(struct sk_buff, tc_index, 2,
9827 target_size));
9828 else
9829 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
9830 bpf_target_off(struct sk_buff, tc_index, 2,
9831 target_size));
9832#else
9833 *target_size = 2;
9834 if (type == BPF_WRITE)
9835 *insn++ = BPF_MOV64_REG(si->dst_reg, si->dst_reg);
9836 else
9837 *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
9838#endif
9839 break;
9840
9841 case offsetof(struct __sk_buff, napi_id):
9842#if defined(CONFIG_NET_RX_BUSY_POLL)
9843 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
9844 bpf_target_off(struct sk_buff, napi_id, 4,
9845 target_size));
9846 *insn++ = BPF_JMP_IMM(BPF_JGE, si->dst_reg, MIN_NAPI_ID, 1);
9847 *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
9848#else
9849 *target_size = 4;
9850 *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
9851#endif
9852 break;
9853 case offsetof(struct __sk_buff, family):
9854 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
9855
9856 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9857 si->dst_reg, si->src_reg,
9858 offsetof(struct sk_buff, sk));
9859 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9860 bpf_target_off(struct sock_common,
9861 skc_family,
9862 2, target_size));
9863 break;
9864 case offsetof(struct __sk_buff, remote_ip4):
9865 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
9866
9867 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9868 si->dst_reg, si->src_reg,
9869 offsetof(struct sk_buff, sk));
9870 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9871 bpf_target_off(struct sock_common,
9872 skc_daddr,
9873 4, target_size));
9874 break;
9875 case offsetof(struct __sk_buff, local_ip4):
9876 BUILD_BUG_ON(sizeof_field(struct sock_common,
9877 skc_rcv_saddr) != 4);
9878
9879 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9880 si->dst_reg, si->src_reg,
9881 offsetof(struct sk_buff, sk));
9882 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9883 bpf_target_off(struct sock_common,
9884 skc_rcv_saddr,
9885 4, target_size));
9886 break;
9887 case offsetof(struct __sk_buff, remote_ip6[0]) ...
9888 offsetof(struct __sk_buff, remote_ip6[3]):
9889#if IS_ENABLED(CONFIG_IPV6)
9890 BUILD_BUG_ON(sizeof_field(struct sock_common,
9891 skc_v6_daddr.s6_addr32[0]) != 4);
9892
9893 off = si->off;
9894 off -= offsetof(struct __sk_buff, remote_ip6[0]);
9895
9896 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9897 si->dst_reg, si->src_reg,
9898 offsetof(struct sk_buff, sk));
9899 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9900 offsetof(struct sock_common,
9901 skc_v6_daddr.s6_addr32[0]) +
9902 off);
9903#else
9904 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
9905#endif
9906 break;
9907 case offsetof(struct __sk_buff, local_ip6[0]) ...
9908 offsetof(struct __sk_buff, local_ip6[3]):
9909#if IS_ENABLED(CONFIG_IPV6)
9910 BUILD_BUG_ON(sizeof_field(struct sock_common,
9911 skc_v6_rcv_saddr.s6_addr32[0]) != 4);
9912
9913 off = si->off;
9914 off -= offsetof(struct __sk_buff, local_ip6[0]);
9915
9916 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9917 si->dst_reg, si->src_reg,
9918 offsetof(struct sk_buff, sk));
9919 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
9920 offsetof(struct sock_common,
9921 skc_v6_rcv_saddr.s6_addr32[0]) +
9922 off);
9923#else
9924 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
9925#endif
9926 break;
9927
9928 case offsetof(struct __sk_buff, remote_port):
9929 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
9930
9931 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9932 si->dst_reg, si->src_reg,
9933 offsetof(struct sk_buff, sk));
9934 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9935 bpf_target_off(struct sock_common,
9936 skc_dport,
9937 2, target_size));
9938#ifndef __BIG_ENDIAN_BITFIELD
9939 *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
9940#endif
9941 break;
9942
9943 case offsetof(struct __sk_buff, local_port):
9944 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
9945
9946 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9947 si->dst_reg, si->src_reg,
9948 offsetof(struct sk_buff, sk));
9949 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
9950 bpf_target_off(struct sock_common,
9951 skc_num, 2, target_size));
9952 break;
9953
9954 case offsetof(struct __sk_buff, tstamp):
9955 BUILD_BUG_ON(sizeof_field(struct sk_buff, tstamp) != 8);
9956
9957 if (type == BPF_WRITE)
9958 insn = bpf_convert_tstamp_write(prog, si, insn);
9959 else
9960 insn = bpf_convert_tstamp_read(prog, si, insn);
9961 break;
9962
9963 case offsetof(struct __sk_buff, tstamp_type):
9964 insn = bpf_convert_tstamp_type_read(si, insn);
9965 break;
9966
9967 case offsetof(struct __sk_buff, gso_segs):
9968 insn = bpf_convert_shinfo_access(dst_reg: si->dst_reg, skb_reg: si->src_reg, insn);
9969 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_segs),
9970 si->dst_reg, si->dst_reg,
9971 bpf_target_off(struct skb_shared_info,
9972 gso_segs, 2,
9973 target_size));
9974 break;
9975 case offsetof(struct __sk_buff, gso_size):
9976 insn = bpf_convert_shinfo_access(dst_reg: si->dst_reg, skb_reg: si->src_reg, insn);
9977 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_size),
9978 si->dst_reg, si->dst_reg,
9979 bpf_target_off(struct skb_shared_info,
9980 gso_size, 2,
9981 target_size));
9982 break;
9983 case offsetof(struct __sk_buff, wire_len):
9984 BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, pkt_len) != 4);
9985
9986 off = si->off;
9987 off -= offsetof(struct __sk_buff, wire_len);
9988 off += offsetof(struct sk_buff, cb);
9989 off += offsetof(struct qdisc_skb_cb, pkt_len);
9990 *target_size = 4;
9991 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off);
9992 break;
9993
9994 case offsetof(struct __sk_buff, sk):
9995 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
9996 si->dst_reg, si->src_reg,
9997 offsetof(struct sk_buff, sk));
9998 break;
9999 case offsetof(struct __sk_buff, hwtstamp):
10000 BUILD_BUG_ON(sizeof_field(struct skb_shared_hwtstamps, hwtstamp) != 8);
10001 BUILD_BUG_ON(offsetof(struct skb_shared_hwtstamps, hwtstamp) != 0);
10002
10003 insn = bpf_convert_shinfo_access(dst_reg: si->dst_reg, skb_reg: si->src_reg, insn);
10004 *insn++ = BPF_LDX_MEM(BPF_DW,
10005 si->dst_reg, si->dst_reg,
10006 bpf_target_off(struct skb_shared_info,
10007 hwtstamps, 8,
10008 target_size));
10009 break;
10010 }
10011
10012 return insn - insn_buf;
10013}
10014
10015u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
10016 const struct bpf_insn *si,
10017 struct bpf_insn *insn_buf,
10018 struct bpf_prog *prog, u32 *target_size)
10019{
10020 struct bpf_insn *insn = insn_buf;
10021 int off;
10022
10023 switch (si->off) {
10024 case offsetof(struct bpf_sock, bound_dev_if):
10025 BUILD_BUG_ON(sizeof_field(struct sock, sk_bound_dev_if) != 4);
10026
10027 if (type == BPF_WRITE)
10028 *insn++ = BPF_EMIT_STORE(BPF_W, si,
10029 offsetof(struct sock, sk_bound_dev_if));
10030 else
10031 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10032 offsetof(struct sock, sk_bound_dev_if));
10033 break;
10034
10035 case offsetof(struct bpf_sock, mark):
10036 BUILD_BUG_ON(sizeof_field(struct sock, sk_mark) != 4);
10037
10038 if (type == BPF_WRITE)
10039 *insn++ = BPF_EMIT_STORE(BPF_W, si,
10040 offsetof(struct sock, sk_mark));
10041 else
10042 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10043 offsetof(struct sock, sk_mark));
10044 break;
10045
10046 case offsetof(struct bpf_sock, priority):
10047 BUILD_BUG_ON(sizeof_field(struct sock, sk_priority) != 4);
10048
10049 if (type == BPF_WRITE)
10050 *insn++ = BPF_EMIT_STORE(BPF_W, si,
10051 offsetof(struct sock, sk_priority));
10052 else
10053 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10054 offsetof(struct sock, sk_priority));
10055 break;
10056
10057 case offsetof(struct bpf_sock, family):
10058 *insn++ = BPF_LDX_MEM(
10059 BPF_FIELD_SIZEOF(struct sock_common, skc_family),
10060 si->dst_reg, si->src_reg,
10061 bpf_target_off(struct sock_common,
10062 skc_family,
10063 sizeof_field(struct sock_common,
10064 skc_family),
10065 target_size));
10066 break;
10067
10068 case offsetof(struct bpf_sock, type):
10069 *insn++ = BPF_LDX_MEM(
10070 BPF_FIELD_SIZEOF(struct sock, sk_type),
10071 si->dst_reg, si->src_reg,
10072 bpf_target_off(struct sock, sk_type,
10073 sizeof_field(struct sock, sk_type),
10074 target_size));
10075 break;
10076
10077 case offsetof(struct bpf_sock, protocol):
10078 *insn++ = BPF_LDX_MEM(
10079 BPF_FIELD_SIZEOF(struct sock, sk_protocol),
10080 si->dst_reg, si->src_reg,
10081 bpf_target_off(struct sock, sk_protocol,
10082 sizeof_field(struct sock, sk_protocol),
10083 target_size));
10084 break;
10085
10086 case offsetof(struct bpf_sock, src_ip4):
10087 *insn++ = BPF_LDX_MEM(
10088 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10089 bpf_target_off(struct sock_common, skc_rcv_saddr,
10090 sizeof_field(struct sock_common,
10091 skc_rcv_saddr),
10092 target_size));
10093 break;
10094
10095 case offsetof(struct bpf_sock, dst_ip4):
10096 *insn++ = BPF_LDX_MEM(
10097 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10098 bpf_target_off(struct sock_common, skc_daddr,
10099 sizeof_field(struct sock_common,
10100 skc_daddr),
10101 target_size));
10102 break;
10103
10104 case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
10105#if IS_ENABLED(CONFIG_IPV6)
10106 off = si->off;
10107 off -= offsetof(struct bpf_sock, src_ip6[0]);
10108 *insn++ = BPF_LDX_MEM(
10109 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10110 bpf_target_off(
10111 struct sock_common,
10112 skc_v6_rcv_saddr.s6_addr32[0],
10113 sizeof_field(struct sock_common,
10114 skc_v6_rcv_saddr.s6_addr32[0]),
10115 target_size) + off);
10116#else
10117 (void)off;
10118 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10119#endif
10120 break;
10121
10122 case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
10123#if IS_ENABLED(CONFIG_IPV6)
10124 off = si->off;
10125 off -= offsetof(struct bpf_sock, dst_ip6[0]);
10126 *insn++ = BPF_LDX_MEM(
10127 BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10128 bpf_target_off(struct sock_common,
10129 skc_v6_daddr.s6_addr32[0],
10130 sizeof_field(struct sock_common,
10131 skc_v6_daddr.s6_addr32[0]),
10132 target_size) + off);
10133#else
10134 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10135 *target_size = 4;
10136#endif
10137 break;
10138
10139 case offsetof(struct bpf_sock, src_port):
10140 *insn++ = BPF_LDX_MEM(
10141 BPF_FIELD_SIZEOF(struct sock_common, skc_num),
10142 si->dst_reg, si->src_reg,
10143 bpf_target_off(struct sock_common, skc_num,
10144 sizeof_field(struct sock_common,
10145 skc_num),
10146 target_size));
10147 break;
10148
10149 case offsetof(struct bpf_sock, dst_port):
10150 *insn++ = BPF_LDX_MEM(
10151 BPF_FIELD_SIZEOF(struct sock_common, skc_dport),
10152 si->dst_reg, si->src_reg,
10153 bpf_target_off(struct sock_common, skc_dport,
10154 sizeof_field(struct sock_common,
10155 skc_dport),
10156 target_size));
10157 break;
10158
10159 case offsetof(struct bpf_sock, state):
10160 *insn++ = BPF_LDX_MEM(
10161 BPF_FIELD_SIZEOF(struct sock_common, skc_state),
10162 si->dst_reg, si->src_reg,
10163 bpf_target_off(struct sock_common, skc_state,
10164 sizeof_field(struct sock_common,
10165 skc_state),
10166 target_size));
10167 break;
10168 case offsetof(struct bpf_sock, rx_queue_mapping):
10169#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
10170 *insn++ = BPF_LDX_MEM(
10171 BPF_FIELD_SIZEOF(struct sock, sk_rx_queue_mapping),
10172 si->dst_reg, si->src_reg,
10173 bpf_target_off(struct sock, sk_rx_queue_mapping,
10174 sizeof_field(struct sock,
10175 sk_rx_queue_mapping),
10176 target_size));
10177 *insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING,
10178 1);
10179 *insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
10180#else
10181 *insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
10182 *target_size = 2;
10183#endif
10184 break;
10185 }
10186
10187 return insn - insn_buf;
10188}
10189
10190static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type,
10191 const struct bpf_insn *si,
10192 struct bpf_insn *insn_buf,
10193 struct bpf_prog *prog, u32 *target_size)
10194{
10195 struct bpf_insn *insn = insn_buf;
10196
10197 switch (si->off) {
10198 case offsetof(struct __sk_buff, ifindex):
10199 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
10200 si->dst_reg, si->src_reg,
10201 offsetof(struct sk_buff, dev));
10202 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10203 bpf_target_off(struct net_device, ifindex, 4,
10204 target_size));
10205 break;
10206 default:
10207 return bpf_convert_ctx_access(type, si, insn_buf, prog,
10208 target_size);
10209 }
10210
10211 return insn - insn_buf;
10212}
10213
10214static u32 xdp_convert_ctx_access(enum bpf_access_type type,
10215 const struct bpf_insn *si,
10216 struct bpf_insn *insn_buf,
10217 struct bpf_prog *prog, u32 *target_size)
10218{
10219 struct bpf_insn *insn = insn_buf;
10220
10221 switch (si->off) {
10222 case offsetof(struct xdp_md, data):
10223 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data),
10224 si->dst_reg, si->src_reg,
10225 offsetof(struct xdp_buff, data));
10226 break;
10227 case offsetof(struct xdp_md, data_meta):
10228 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_meta),
10229 si->dst_reg, si->src_reg,
10230 offsetof(struct xdp_buff, data_meta));
10231 break;
10232 case offsetof(struct xdp_md, data_end):
10233 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_end),
10234 si->dst_reg, si->src_reg,
10235 offsetof(struct xdp_buff, data_end));
10236 break;
10237 case offsetof(struct xdp_md, ingress_ifindex):
10238 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
10239 si->dst_reg, si->src_reg,
10240 offsetof(struct xdp_buff, rxq));
10241 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_rxq_info, dev),
10242 si->dst_reg, si->dst_reg,
10243 offsetof(struct xdp_rxq_info, dev));
10244 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10245 offsetof(struct net_device, ifindex));
10246 break;
10247 case offsetof(struct xdp_md, rx_queue_index):
10248 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
10249 si->dst_reg, si->src_reg,
10250 offsetof(struct xdp_buff, rxq));
10251 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10252 offsetof(struct xdp_rxq_info,
10253 queue_index));
10254 break;
10255 case offsetof(struct xdp_md, egress_ifindex):
10256 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, txq),
10257 si->dst_reg, si->src_reg,
10258 offsetof(struct xdp_buff, txq));
10259 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_txq_info, dev),
10260 si->dst_reg, si->dst_reg,
10261 offsetof(struct xdp_txq_info, dev));
10262 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10263 offsetof(struct net_device, ifindex));
10264 break;
10265 }
10266
10267 return insn - insn_buf;
10268}
10269
10270/* SOCK_ADDR_LOAD_NESTED_FIELD() loads Nested Field S.F.NF where S is type of
10271 * context Structure, F is Field in context structure that contains a pointer
10272 * to Nested Structure of type NS that has the field NF.
10273 *
10274 * SIZE encodes the load size (BPF_B, BPF_H, etc). It's up to caller to make
10275 * sure that SIZE is not greater than actual size of S.F.NF.
10276 *
10277 * If offset OFF is provided, the load happens from that offset relative to
10278 * offset of NF.
10279 */
10280#define SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF) \
10281 do { \
10282 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), si->dst_reg, \
10283 si->src_reg, offsetof(S, F)); \
10284 *insn++ = BPF_LDX_MEM( \
10285 SIZE, si->dst_reg, si->dst_reg, \
10286 bpf_target_off(NS, NF, sizeof_field(NS, NF), \
10287 target_size) \
10288 + OFF); \
10289 } while (0)
10290
10291#define SOCK_ADDR_LOAD_NESTED_FIELD(S, NS, F, NF) \
10292 SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, \
10293 BPF_FIELD_SIZEOF(NS, NF), 0)
10294
10295/* SOCK_ADDR_STORE_NESTED_FIELD_OFF() has semantic similar to
10296 * SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF() but for store operation.
10297 *
10298 * In addition it uses Temporary Field TF (member of struct S) as the 3rd
10299 * "register" since two registers available in convert_ctx_access are not
10300 * enough: we can't override neither SRC, since it contains value to store, nor
10301 * DST since it contains pointer to context that may be used by later
10302 * instructions. But we need a temporary place to save pointer to nested
10303 * structure whose field we want to store to.
10304 */
10305#define SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, OFF, TF) \
10306 do { \
10307 int tmp_reg = BPF_REG_9; \
10308 if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \
10309 --tmp_reg; \
10310 if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \
10311 --tmp_reg; \
10312 *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, tmp_reg, \
10313 offsetof(S, TF)); \
10314 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), tmp_reg, \
10315 si->dst_reg, offsetof(S, F)); \
10316 *insn++ = BPF_RAW_INSN(SIZE | BPF_MEM | BPF_CLASS(si->code), \
10317 tmp_reg, si->src_reg, \
10318 bpf_target_off(NS, NF, sizeof_field(NS, NF), \
10319 target_size) \
10320 + OFF, \
10321 si->imm); \
10322 *insn++ = BPF_LDX_MEM(BPF_DW, tmp_reg, si->dst_reg, \
10323 offsetof(S, TF)); \
10324 } while (0)
10325
10326#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF, \
10327 TF) \
10328 do { \
10329 if (type == BPF_WRITE) { \
10330 SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, \
10331 OFF, TF); \
10332 } else { \
10333 SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF( \
10334 S, NS, F, NF, SIZE, OFF); \
10335 } \
10336 } while (0)
10337
10338static u32 sock_addr_convert_ctx_access(enum bpf_access_type type,
10339 const struct bpf_insn *si,
10340 struct bpf_insn *insn_buf,
10341 struct bpf_prog *prog, u32 *target_size)
10342{
10343 int off, port_size = sizeof_field(struct sockaddr_in6, sin6_port);
10344 struct bpf_insn *insn = insn_buf;
10345
10346 switch (si->off) {
10347 case offsetof(struct bpf_sock_addr, user_family):
10348 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10349 struct sockaddr, uaddr, sa_family);
10350 break;
10351
10352 case offsetof(struct bpf_sock_addr, user_ip4):
10353 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10354 struct bpf_sock_addr_kern, struct sockaddr_in, uaddr,
10355 sin_addr, BPF_SIZE(si->code), 0, tmp_reg);
10356 break;
10357
10358 case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
10359 off = si->off;
10360 off -= offsetof(struct bpf_sock_addr, user_ip6[0]);
10361 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10362 struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
10363 sin6_addr.s6_addr32[0], BPF_SIZE(si->code), off,
10364 tmp_reg);
10365 break;
10366
10367 case offsetof(struct bpf_sock_addr, user_port):
10368 /* To get port we need to know sa_family first and then treat
10369 * sockaddr as either sockaddr_in or sockaddr_in6.
10370 * Though we can simplify since port field has same offset and
10371 * size in both structures.
10372 * Here we check this invariant and use just one of the
10373 * structures if it's true.
10374 */
10375 BUILD_BUG_ON(offsetof(struct sockaddr_in, sin_port) !=
10376 offsetof(struct sockaddr_in6, sin6_port));
10377 BUILD_BUG_ON(sizeof_field(struct sockaddr_in, sin_port) !=
10378 sizeof_field(struct sockaddr_in6, sin6_port));
10379 /* Account for sin6_port being smaller than user_port. */
10380 port_size = min(port_size, BPF_LDST_BYTES(si));
10381 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10382 struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
10383 sin6_port, bytes_to_bpf_size(port_size), 0, tmp_reg);
10384 break;
10385
10386 case offsetof(struct bpf_sock_addr, family):
10387 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10388 struct sock, sk, sk_family);
10389 break;
10390
10391 case offsetof(struct bpf_sock_addr, type):
10392 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10393 struct sock, sk, sk_type);
10394 break;
10395
10396 case offsetof(struct bpf_sock_addr, protocol):
10397 SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10398 struct sock, sk, sk_protocol);
10399 break;
10400
10401 case offsetof(struct bpf_sock_addr, msg_src_ip4):
10402 /* Treat t_ctx as struct in_addr for msg_src_ip4. */
10403 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10404 struct bpf_sock_addr_kern, struct in_addr, t_ctx,
10405 s_addr, BPF_SIZE(si->code), 0, tmp_reg);
10406 break;
10407
10408 case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
10409 msg_src_ip6[3]):
10410 off = si->off;
10411 off -= offsetof(struct bpf_sock_addr, msg_src_ip6[0]);
10412 /* Treat t_ctx as struct in6_addr for msg_src_ip6. */
10413 SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10414 struct bpf_sock_addr_kern, struct in6_addr, t_ctx,
10415 s6_addr32[0], BPF_SIZE(si->code), off, tmp_reg);
10416 break;
10417 case offsetof(struct bpf_sock_addr, sk):
10418 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_addr_kern, sk),
10419 si->dst_reg, si->src_reg,
10420 offsetof(struct bpf_sock_addr_kern, sk));
10421 break;
10422 }
10423
10424 return insn - insn_buf;
10425}
10426
10427static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
10428 const struct bpf_insn *si,
10429 struct bpf_insn *insn_buf,
10430 struct bpf_prog *prog,
10431 u32 *target_size)
10432{
10433 struct bpf_insn *insn = insn_buf;
10434 int off;
10435
10436/* Helper macro for adding read access to tcp_sock or sock fields. */
10437#define SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ) \
10438 do { \
10439 int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 2; \
10440 BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) > \
10441 sizeof_field(struct bpf_sock_ops, BPF_FIELD)); \
10442 if (si->dst_reg == reg || si->src_reg == reg) \
10443 reg--; \
10444 if (si->dst_reg == reg || si->src_reg == reg) \
10445 reg--; \
10446 if (si->dst_reg == si->src_reg) { \
10447 *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, \
10448 offsetof(struct bpf_sock_ops_kern, \
10449 temp)); \
10450 fullsock_reg = reg; \
10451 jmp += 2; \
10452 } \
10453 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
10454 struct bpf_sock_ops_kern, \
10455 is_locked_tcp_sock), \
10456 fullsock_reg, si->src_reg, \
10457 offsetof(struct bpf_sock_ops_kern, \
10458 is_locked_tcp_sock)); \
10459 *insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp); \
10460 if (si->dst_reg == si->src_reg) \
10461 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
10462 offsetof(struct bpf_sock_ops_kern, \
10463 temp)); \
10464 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
10465 struct bpf_sock_ops_kern, sk),\
10466 si->dst_reg, si->src_reg, \
10467 offsetof(struct bpf_sock_ops_kern, sk));\
10468 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(OBJ, \
10469 OBJ_FIELD), \
10470 si->dst_reg, si->dst_reg, \
10471 offsetof(OBJ, OBJ_FIELD)); \
10472 if (si->dst_reg == si->src_reg) { \
10473 *insn++ = BPF_JMP_A(1); \
10474 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
10475 offsetof(struct bpf_sock_ops_kern, \
10476 temp)); \
10477 } \
10478 } while (0)
10479
10480#define SOCK_OPS_GET_SK() \
10481 do { \
10482 int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 1; \
10483 if (si->dst_reg == reg || si->src_reg == reg) \
10484 reg--; \
10485 if (si->dst_reg == reg || si->src_reg == reg) \
10486 reg--; \
10487 if (si->dst_reg == si->src_reg) { \
10488 *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, \
10489 offsetof(struct bpf_sock_ops_kern, \
10490 temp)); \
10491 fullsock_reg = reg; \
10492 jmp += 2; \
10493 } \
10494 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
10495 struct bpf_sock_ops_kern, \
10496 is_fullsock), \
10497 fullsock_reg, si->src_reg, \
10498 offsetof(struct bpf_sock_ops_kern, \
10499 is_fullsock)); \
10500 *insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp); \
10501 if (si->dst_reg == si->src_reg) \
10502 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
10503 offsetof(struct bpf_sock_ops_kern, \
10504 temp)); \
10505 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
10506 struct bpf_sock_ops_kern, sk),\
10507 si->dst_reg, si->src_reg, \
10508 offsetof(struct bpf_sock_ops_kern, sk));\
10509 if (si->dst_reg == si->src_reg) { \
10510 *insn++ = BPF_JMP_A(1); \
10511 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \
10512 offsetof(struct bpf_sock_ops_kern, \
10513 temp)); \
10514 } \
10515 } while (0)
10516
10517#define SOCK_OPS_GET_TCP_SOCK_FIELD(FIELD) \
10518 SOCK_OPS_GET_FIELD(FIELD, FIELD, struct tcp_sock)
10519
10520/* Helper macro for adding write access to tcp_sock or sock fields.
10521 * The macro is called with two registers, dst_reg which contains a pointer
10522 * to ctx (context) and src_reg which contains the value that should be
10523 * stored. However, we need an additional register since we cannot overwrite
10524 * dst_reg because it may be used later in the program.
10525 * Instead we "borrow" one of the other register. We first save its value
10526 * into a new (temp) field in bpf_sock_ops_kern, use it, and then restore
10527 * it at the end of the macro.
10528 */
10529#define SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ) \
10530 do { \
10531 int reg = BPF_REG_9; \
10532 BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) > \
10533 sizeof_field(struct bpf_sock_ops, BPF_FIELD)); \
10534 if (si->dst_reg == reg || si->src_reg == reg) \
10535 reg--; \
10536 if (si->dst_reg == reg || si->src_reg == reg) \
10537 reg--; \
10538 *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, reg, \
10539 offsetof(struct bpf_sock_ops_kern, \
10540 temp)); \
10541 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
10542 struct bpf_sock_ops_kern, \
10543 is_locked_tcp_sock), \
10544 reg, si->dst_reg, \
10545 offsetof(struct bpf_sock_ops_kern, \
10546 is_locked_tcp_sock)); \
10547 *insn++ = BPF_JMP_IMM(BPF_JEQ, reg, 0, 2); \
10548 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( \
10549 struct bpf_sock_ops_kern, sk),\
10550 reg, si->dst_reg, \
10551 offsetof(struct bpf_sock_ops_kern, sk));\
10552 *insn++ = BPF_RAW_INSN(BPF_FIELD_SIZEOF(OBJ, OBJ_FIELD) | \
10553 BPF_MEM | BPF_CLASS(si->code), \
10554 reg, si->src_reg, \
10555 offsetof(OBJ, OBJ_FIELD), \
10556 si->imm); \
10557 *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->dst_reg, \
10558 offsetof(struct bpf_sock_ops_kern, \
10559 temp)); \
10560 } while (0)
10561
10562#define SOCK_OPS_GET_OR_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ, TYPE) \
10563 do { \
10564 if (TYPE == BPF_WRITE) \
10565 SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ); \
10566 else \
10567 SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ); \
10568 } while (0)
10569
10570 switch (si->off) {
10571 case offsetof(struct bpf_sock_ops, op):
10572 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
10573 op),
10574 si->dst_reg, si->src_reg,
10575 offsetof(struct bpf_sock_ops_kern, op));
10576 break;
10577
10578 case offsetof(struct bpf_sock_ops, replylong[0]) ...
10579 offsetof(struct bpf_sock_ops, replylong[3]):
10580 BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, reply) !=
10581 sizeof_field(struct bpf_sock_ops_kern, reply));
10582 BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, replylong) !=
10583 sizeof_field(struct bpf_sock_ops_kern, replylong));
10584 off = si->off;
10585 off -= offsetof(struct bpf_sock_ops, replylong[0]);
10586 off += offsetof(struct bpf_sock_ops_kern, replylong[0]);
10587 if (type == BPF_WRITE)
10588 *insn++ = BPF_EMIT_STORE(BPF_W, si, off);
10589 else
10590 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10591 off);
10592 break;
10593
10594 case offsetof(struct bpf_sock_ops, family):
10595 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
10596
10597 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10598 struct bpf_sock_ops_kern, sk),
10599 si->dst_reg, si->src_reg,
10600 offsetof(struct bpf_sock_ops_kern, sk));
10601 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10602 offsetof(struct sock_common, skc_family));
10603 break;
10604
10605 case offsetof(struct bpf_sock_ops, remote_ip4):
10606 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
10607
10608 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10609 struct bpf_sock_ops_kern, sk),
10610 si->dst_reg, si->src_reg,
10611 offsetof(struct bpf_sock_ops_kern, sk));
10612 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10613 offsetof(struct sock_common, skc_daddr));
10614 break;
10615
10616 case offsetof(struct bpf_sock_ops, local_ip4):
10617 BUILD_BUG_ON(sizeof_field(struct sock_common,
10618 skc_rcv_saddr) != 4);
10619
10620 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10621 struct bpf_sock_ops_kern, sk),
10622 si->dst_reg, si->src_reg,
10623 offsetof(struct bpf_sock_ops_kern, sk));
10624 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10625 offsetof(struct sock_common,
10626 skc_rcv_saddr));
10627 break;
10628
10629 case offsetof(struct bpf_sock_ops, remote_ip6[0]) ...
10630 offsetof(struct bpf_sock_ops, remote_ip6[3]):
10631#if IS_ENABLED(CONFIG_IPV6)
10632 BUILD_BUG_ON(sizeof_field(struct sock_common,
10633 skc_v6_daddr.s6_addr32[0]) != 4);
10634
10635 off = si->off;
10636 off -= offsetof(struct bpf_sock_ops, remote_ip6[0]);
10637 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10638 struct bpf_sock_ops_kern, sk),
10639 si->dst_reg, si->src_reg,
10640 offsetof(struct bpf_sock_ops_kern, sk));
10641 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10642 offsetof(struct sock_common,
10643 skc_v6_daddr.s6_addr32[0]) +
10644 off);
10645#else
10646 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10647#endif
10648 break;
10649
10650 case offsetof(struct bpf_sock_ops, local_ip6[0]) ...
10651 offsetof(struct bpf_sock_ops, local_ip6[3]):
10652#if IS_ENABLED(CONFIG_IPV6)
10653 BUILD_BUG_ON(sizeof_field(struct sock_common,
10654 skc_v6_rcv_saddr.s6_addr32[0]) != 4);
10655
10656 off = si->off;
10657 off -= offsetof(struct bpf_sock_ops, local_ip6[0]);
10658 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10659 struct bpf_sock_ops_kern, sk),
10660 si->dst_reg, si->src_reg,
10661 offsetof(struct bpf_sock_ops_kern, sk));
10662 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10663 offsetof(struct sock_common,
10664 skc_v6_rcv_saddr.s6_addr32[0]) +
10665 off);
10666#else
10667 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10668#endif
10669 break;
10670
10671 case offsetof(struct bpf_sock_ops, remote_port):
10672 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
10673
10674 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10675 struct bpf_sock_ops_kern, sk),
10676 si->dst_reg, si->src_reg,
10677 offsetof(struct bpf_sock_ops_kern, sk));
10678 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10679 offsetof(struct sock_common, skc_dport));
10680#ifndef __BIG_ENDIAN_BITFIELD
10681 *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
10682#endif
10683 break;
10684
10685 case offsetof(struct bpf_sock_ops, local_port):
10686 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
10687
10688 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10689 struct bpf_sock_ops_kern, sk),
10690 si->dst_reg, si->src_reg,
10691 offsetof(struct bpf_sock_ops_kern, sk));
10692 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10693 offsetof(struct sock_common, skc_num));
10694 break;
10695
10696 case offsetof(struct bpf_sock_ops, is_fullsock):
10697 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10698 struct bpf_sock_ops_kern,
10699 is_fullsock),
10700 si->dst_reg, si->src_reg,
10701 offsetof(struct bpf_sock_ops_kern,
10702 is_fullsock));
10703 break;
10704
10705 case offsetof(struct bpf_sock_ops, state):
10706 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_state) != 1);
10707
10708 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10709 struct bpf_sock_ops_kern, sk),
10710 si->dst_reg, si->src_reg,
10711 offsetof(struct bpf_sock_ops_kern, sk));
10712 *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->dst_reg,
10713 offsetof(struct sock_common, skc_state));
10714 break;
10715
10716 case offsetof(struct bpf_sock_ops, rtt_min):
10717 BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
10718 sizeof(struct minmax));
10719 BUILD_BUG_ON(sizeof(struct minmax) <
10720 sizeof(struct minmax_sample));
10721
10722 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10723 struct bpf_sock_ops_kern, sk),
10724 si->dst_reg, si->src_reg,
10725 offsetof(struct bpf_sock_ops_kern, sk));
10726 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10727 offsetof(struct tcp_sock, rtt_min) +
10728 sizeof_field(struct minmax_sample, t));
10729 break;
10730
10731 case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags):
10732 SOCK_OPS_GET_FIELD(bpf_sock_ops_cb_flags, bpf_sock_ops_cb_flags,
10733 struct tcp_sock);
10734 break;
10735
10736 case offsetof(struct bpf_sock_ops, sk_txhash):
10737 SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,
10738 struct sock, type);
10739 break;
10740 case offsetof(struct bpf_sock_ops, snd_cwnd):
10741 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_cwnd);
10742 break;
10743 case offsetof(struct bpf_sock_ops, srtt_us):
10744 SOCK_OPS_GET_TCP_SOCK_FIELD(srtt_us);
10745 break;
10746 case offsetof(struct bpf_sock_ops, snd_ssthresh):
10747 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_ssthresh);
10748 break;
10749 case offsetof(struct bpf_sock_ops, rcv_nxt):
10750 SOCK_OPS_GET_TCP_SOCK_FIELD(rcv_nxt);
10751 break;
10752 case offsetof(struct bpf_sock_ops, snd_nxt):
10753 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_nxt);
10754 break;
10755 case offsetof(struct bpf_sock_ops, snd_una):
10756 SOCK_OPS_GET_TCP_SOCK_FIELD(snd_una);
10757 break;
10758 case offsetof(struct bpf_sock_ops, mss_cache):
10759 SOCK_OPS_GET_TCP_SOCK_FIELD(mss_cache);
10760 break;
10761 case offsetof(struct bpf_sock_ops, ecn_flags):
10762 SOCK_OPS_GET_TCP_SOCK_FIELD(ecn_flags);
10763 break;
10764 case offsetof(struct bpf_sock_ops, rate_delivered):
10765 SOCK_OPS_GET_TCP_SOCK_FIELD(rate_delivered);
10766 break;
10767 case offsetof(struct bpf_sock_ops, rate_interval_us):
10768 SOCK_OPS_GET_TCP_SOCK_FIELD(rate_interval_us);
10769 break;
10770 case offsetof(struct bpf_sock_ops, packets_out):
10771 SOCK_OPS_GET_TCP_SOCK_FIELD(packets_out);
10772 break;
10773 case offsetof(struct bpf_sock_ops, retrans_out):
10774 SOCK_OPS_GET_TCP_SOCK_FIELD(retrans_out);
10775 break;
10776 case offsetof(struct bpf_sock_ops, total_retrans):
10777 SOCK_OPS_GET_TCP_SOCK_FIELD(total_retrans);
10778 break;
10779 case offsetof(struct bpf_sock_ops, segs_in):
10780 SOCK_OPS_GET_TCP_SOCK_FIELD(segs_in);
10781 break;
10782 case offsetof(struct bpf_sock_ops, data_segs_in):
10783 SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_in);
10784 break;
10785 case offsetof(struct bpf_sock_ops, segs_out):
10786 SOCK_OPS_GET_TCP_SOCK_FIELD(segs_out);
10787 break;
10788 case offsetof(struct bpf_sock_ops, data_segs_out):
10789 SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_out);
10790 break;
10791 case offsetof(struct bpf_sock_ops, lost_out):
10792 SOCK_OPS_GET_TCP_SOCK_FIELD(lost_out);
10793 break;
10794 case offsetof(struct bpf_sock_ops, sacked_out):
10795 SOCK_OPS_GET_TCP_SOCK_FIELD(sacked_out);
10796 break;
10797 case offsetof(struct bpf_sock_ops, bytes_received):
10798 SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_received);
10799 break;
10800 case offsetof(struct bpf_sock_ops, bytes_acked):
10801 SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_acked);
10802 break;
10803 case offsetof(struct bpf_sock_ops, sk):
10804 SOCK_OPS_GET_SK();
10805 break;
10806 case offsetof(struct bpf_sock_ops, skb_data_end):
10807 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
10808 skb_data_end),
10809 si->dst_reg, si->src_reg,
10810 offsetof(struct bpf_sock_ops_kern,
10811 skb_data_end));
10812 break;
10813 case offsetof(struct bpf_sock_ops, skb_data):
10814 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
10815 skb),
10816 si->dst_reg, si->src_reg,
10817 offsetof(struct bpf_sock_ops_kern,
10818 skb));
10819 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10820 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
10821 si->dst_reg, si->dst_reg,
10822 offsetof(struct sk_buff, data));
10823 break;
10824 case offsetof(struct bpf_sock_ops, skb_len):
10825 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
10826 skb),
10827 si->dst_reg, si->src_reg,
10828 offsetof(struct bpf_sock_ops_kern,
10829 skb));
10830 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10831 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
10832 si->dst_reg, si->dst_reg,
10833 offsetof(struct sk_buff, len));
10834 break;
10835 case offsetof(struct bpf_sock_ops, skb_tcp_flags):
10836 off = offsetof(struct sk_buff, cb);
10837 off += offsetof(struct tcp_skb_cb, tcp_flags);
10838 *target_size = sizeof_field(struct tcp_skb_cb, tcp_flags);
10839 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
10840 skb),
10841 si->dst_reg, si->src_reg,
10842 offsetof(struct bpf_sock_ops_kern,
10843 skb));
10844 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10845 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_skb_cb,
10846 tcp_flags),
10847 si->dst_reg, si->dst_reg, off);
10848 break;
10849 case offsetof(struct bpf_sock_ops, skb_hwtstamp): {
10850 struct bpf_insn *jmp_on_null_skb;
10851
10852 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
10853 skb),
10854 si->dst_reg, si->src_reg,
10855 offsetof(struct bpf_sock_ops_kern,
10856 skb));
10857 /* Reserve one insn to test skb == NULL */
10858 jmp_on_null_skb = insn++;
10859 insn = bpf_convert_shinfo_access(dst_reg: si->dst_reg, skb_reg: si->dst_reg, insn);
10860 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
10861 bpf_target_off(struct skb_shared_info,
10862 hwtstamps, 8,
10863 target_size));
10864 *jmp_on_null_skb = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0,
10865 insn - jmp_on_null_skb - 1);
10866 break;
10867 }
10868 }
10869 return insn - insn_buf;
10870}
10871
10872/* data_end = skb->data + skb_headlen() */
10873static struct bpf_insn *bpf_convert_data_end_access(const struct bpf_insn *si,
10874 struct bpf_insn *insn)
10875{
10876 int reg;
10877 int temp_reg_off = offsetof(struct sk_buff, cb) +
10878 offsetof(struct sk_skb_cb, temp_reg);
10879
10880 if (si->src_reg == si->dst_reg) {
10881 /* We need an extra register, choose and save a register. */
10882 reg = BPF_REG_9;
10883 if (si->src_reg == reg || si->dst_reg == reg)
10884 reg--;
10885 if (si->src_reg == reg || si->dst_reg == reg)
10886 reg--;
10887 *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, temp_reg_off);
10888 } else {
10889 reg = si->dst_reg;
10890 }
10891
10892 /* reg = skb->data */
10893 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
10894 reg, si->src_reg,
10895 offsetof(struct sk_buff, data));
10896 /* AX = skb->len */
10897 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
10898 BPF_REG_AX, si->src_reg,
10899 offsetof(struct sk_buff, len));
10900 /* reg = skb->data + skb->len */
10901 *insn++ = BPF_ALU64_REG(BPF_ADD, reg, BPF_REG_AX);
10902 /* AX = skb->data_len */
10903 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data_len),
10904 BPF_REG_AX, si->src_reg,
10905 offsetof(struct sk_buff, data_len));
10906
10907 /* reg = skb->data + skb->len - skb->data_len */
10908 *insn++ = BPF_ALU64_REG(BPF_SUB, reg, BPF_REG_AX);
10909
10910 if (si->src_reg == si->dst_reg) {
10911 /* Restore the saved register */
10912 *insn++ = BPF_MOV64_REG(BPF_REG_AX, si->src_reg);
10913 *insn++ = BPF_MOV64_REG(si->dst_reg, reg);
10914 *insn++ = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_AX, temp_reg_off);
10915 }
10916
10917 return insn;
10918}
10919
10920static u32 sk_skb_convert_ctx_access(enum bpf_access_type type,
10921 const struct bpf_insn *si,
10922 struct bpf_insn *insn_buf,
10923 struct bpf_prog *prog, u32 *target_size)
10924{
10925 struct bpf_insn *insn = insn_buf;
10926 int off;
10927
10928 switch (si->off) {
10929 case offsetof(struct __sk_buff, data_end):
10930 insn = bpf_convert_data_end_access(si, insn);
10931 break;
10932 case offsetof(struct __sk_buff, cb[0]) ...
10933 offsetofend(struct __sk_buff, cb[4]) - 1:
10934 BUILD_BUG_ON(sizeof_field(struct sk_skb_cb, data) < 20);
10935 BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
10936 offsetof(struct sk_skb_cb, data)) %
10937 sizeof(__u64));
10938
10939 prog->cb_access = 1;
10940 off = si->off;
10941 off -= offsetof(struct __sk_buff, cb[0]);
10942 off += offsetof(struct sk_buff, cb);
10943 off += offsetof(struct sk_skb_cb, data);
10944 if (type == BPF_WRITE)
10945 *insn++ = BPF_EMIT_STORE(BPF_SIZE(si->code), si, off);
10946 else
10947 *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
10948 si->src_reg, off);
10949 break;
10950
10951
10952 default:
10953 return bpf_convert_ctx_access(type, si, insn_buf, prog,
10954 target_size);
10955 }
10956
10957 return insn - insn_buf;
10958}
10959
10960static u32 sk_msg_convert_ctx_access(enum bpf_access_type type,
10961 const struct bpf_insn *si,
10962 struct bpf_insn *insn_buf,
10963 struct bpf_prog *prog, u32 *target_size)
10964{
10965 struct bpf_insn *insn = insn_buf;
10966#if IS_ENABLED(CONFIG_IPV6)
10967 int off;
10968#endif
10969
10970 /* convert ctx uses the fact sg element is first in struct */
10971 BUILD_BUG_ON(offsetof(struct sk_msg, sg) != 0);
10972
10973 switch (si->off) {
10974 case offsetof(struct sk_msg_md, data):
10975 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data),
10976 si->dst_reg, si->src_reg,
10977 offsetof(struct sk_msg, data));
10978 break;
10979 case offsetof(struct sk_msg_md, data_end):
10980 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data_end),
10981 si->dst_reg, si->src_reg,
10982 offsetof(struct sk_msg, data_end));
10983 break;
10984 case offsetof(struct sk_msg_md, family):
10985 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
10986
10987 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10988 struct sk_msg, sk),
10989 si->dst_reg, si->src_reg,
10990 offsetof(struct sk_msg, sk));
10991 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10992 offsetof(struct sock_common, skc_family));
10993 break;
10994
10995 case offsetof(struct sk_msg_md, remote_ip4):
10996 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
10997
10998 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
10999 struct sk_msg, sk),
11000 si->dst_reg, si->src_reg,
11001 offsetof(struct sk_msg, sk));
11002 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11003 offsetof(struct sock_common, skc_daddr));
11004 break;
11005
11006 case offsetof(struct sk_msg_md, local_ip4):
11007 BUILD_BUG_ON(sizeof_field(struct sock_common,
11008 skc_rcv_saddr) != 4);
11009
11010 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11011 struct sk_msg, sk),
11012 si->dst_reg, si->src_reg,
11013 offsetof(struct sk_msg, sk));
11014 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11015 offsetof(struct sock_common,
11016 skc_rcv_saddr));
11017 break;
11018
11019 case offsetof(struct sk_msg_md, remote_ip6[0]) ...
11020 offsetof(struct sk_msg_md, remote_ip6[3]):
11021#if IS_ENABLED(CONFIG_IPV6)
11022 BUILD_BUG_ON(sizeof_field(struct sock_common,
11023 skc_v6_daddr.s6_addr32[0]) != 4);
11024
11025 off = si->off;
11026 off -= offsetof(struct sk_msg_md, remote_ip6[0]);
11027 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11028 struct sk_msg, sk),
11029 si->dst_reg, si->src_reg,
11030 offsetof(struct sk_msg, sk));
11031 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11032 offsetof(struct sock_common,
11033 skc_v6_daddr.s6_addr32[0]) +
11034 off);
11035#else
11036 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11037#endif
11038 break;
11039
11040 case offsetof(struct sk_msg_md, local_ip6[0]) ...
11041 offsetof(struct sk_msg_md, local_ip6[3]):
11042#if IS_ENABLED(CONFIG_IPV6)
11043 BUILD_BUG_ON(sizeof_field(struct sock_common,
11044 skc_v6_rcv_saddr.s6_addr32[0]) != 4);
11045
11046 off = si->off;
11047 off -= offsetof(struct sk_msg_md, local_ip6[0]);
11048 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11049 struct sk_msg, sk),
11050 si->dst_reg, si->src_reg,
11051 offsetof(struct sk_msg, sk));
11052 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11053 offsetof(struct sock_common,
11054 skc_v6_rcv_saddr.s6_addr32[0]) +
11055 off);
11056#else
11057 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11058#endif
11059 break;
11060
11061 case offsetof(struct sk_msg_md, remote_port):
11062 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
11063
11064 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11065 struct sk_msg, sk),
11066 si->dst_reg, si->src_reg,
11067 offsetof(struct sk_msg, sk));
11068 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11069 offsetof(struct sock_common, skc_dport));
11070#ifndef __BIG_ENDIAN_BITFIELD
11071 *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
11072#endif
11073 break;
11074
11075 case offsetof(struct sk_msg_md, local_port):
11076 BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
11077
11078 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11079 struct sk_msg, sk),
11080 si->dst_reg, si->src_reg,
11081 offsetof(struct sk_msg, sk));
11082 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11083 offsetof(struct sock_common, skc_num));
11084 break;
11085
11086 case offsetof(struct sk_msg_md, size):
11087 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_sg, size),
11088 si->dst_reg, si->src_reg,
11089 offsetof(struct sk_msg_sg, size));
11090 break;
11091
11092 case offsetof(struct sk_msg_md, sk):
11093 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, sk),
11094 si->dst_reg, si->src_reg,
11095 offsetof(struct sk_msg, sk));
11096 break;
11097 }
11098
11099 return insn - insn_buf;
11100}
11101
11102const struct bpf_verifier_ops sk_filter_verifier_ops = {
11103 .get_func_proto = sk_filter_func_proto,
11104 .is_valid_access = sk_filter_is_valid_access,
11105 .convert_ctx_access = bpf_convert_ctx_access,
11106 .gen_ld_abs = bpf_gen_ld_abs,
11107};
11108
11109const struct bpf_prog_ops sk_filter_prog_ops = {
11110 .test_run = bpf_prog_test_run_skb,
11111};
11112
11113const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
11114 .get_func_proto = tc_cls_act_func_proto,
11115 .is_valid_access = tc_cls_act_is_valid_access,
11116 .convert_ctx_access = tc_cls_act_convert_ctx_access,
11117 .gen_prologue = tc_cls_act_prologue,
11118 .gen_ld_abs = bpf_gen_ld_abs,
11119 .btf_struct_access = tc_cls_act_btf_struct_access,
11120};
11121
11122const struct bpf_prog_ops tc_cls_act_prog_ops = {
11123 .test_run = bpf_prog_test_run_skb,
11124};
11125
11126const struct bpf_verifier_ops xdp_verifier_ops = {
11127 .get_func_proto = xdp_func_proto,
11128 .is_valid_access = xdp_is_valid_access,
11129 .convert_ctx_access = xdp_convert_ctx_access,
11130 .gen_prologue = bpf_noop_prologue,
11131 .btf_struct_access = xdp_btf_struct_access,
11132};
11133
11134const struct bpf_prog_ops xdp_prog_ops = {
11135 .test_run = bpf_prog_test_run_xdp,
11136};
11137
11138const struct bpf_verifier_ops cg_skb_verifier_ops = {
11139 .get_func_proto = cg_skb_func_proto,
11140 .is_valid_access = cg_skb_is_valid_access,
11141 .convert_ctx_access = bpf_convert_ctx_access,
11142};
11143
11144const struct bpf_prog_ops cg_skb_prog_ops = {
11145 .test_run = bpf_prog_test_run_skb,
11146};
11147
11148const struct bpf_verifier_ops lwt_in_verifier_ops = {
11149 .get_func_proto = lwt_in_func_proto,
11150 .is_valid_access = lwt_is_valid_access,
11151 .convert_ctx_access = bpf_convert_ctx_access,
11152};
11153
11154const struct bpf_prog_ops lwt_in_prog_ops = {
11155 .test_run = bpf_prog_test_run_skb,
11156};
11157
11158const struct bpf_verifier_ops lwt_out_verifier_ops = {
11159 .get_func_proto = lwt_out_func_proto,
11160 .is_valid_access = lwt_is_valid_access,
11161 .convert_ctx_access = bpf_convert_ctx_access,
11162};
11163
11164const struct bpf_prog_ops lwt_out_prog_ops = {
11165 .test_run = bpf_prog_test_run_skb,
11166};
11167
11168const struct bpf_verifier_ops lwt_xmit_verifier_ops = {
11169 .get_func_proto = lwt_xmit_func_proto,
11170 .is_valid_access = lwt_is_valid_access,
11171 .convert_ctx_access = bpf_convert_ctx_access,
11172 .gen_prologue = tc_cls_act_prologue,
11173};
11174
11175const struct bpf_prog_ops lwt_xmit_prog_ops = {
11176 .test_run = bpf_prog_test_run_skb,
11177};
11178
11179const struct bpf_verifier_ops lwt_seg6local_verifier_ops = {
11180 .get_func_proto = lwt_seg6local_func_proto,
11181 .is_valid_access = lwt_is_valid_access,
11182 .convert_ctx_access = bpf_convert_ctx_access,
11183};
11184
11185const struct bpf_prog_ops lwt_seg6local_prog_ops = {
11186};
11187
11188const struct bpf_verifier_ops cg_sock_verifier_ops = {
11189 .get_func_proto = sock_filter_func_proto,
11190 .is_valid_access = sock_filter_is_valid_access,
11191 .convert_ctx_access = bpf_sock_convert_ctx_access,
11192};
11193
11194const struct bpf_prog_ops cg_sock_prog_ops = {
11195};
11196
11197const struct bpf_verifier_ops cg_sock_addr_verifier_ops = {
11198 .get_func_proto = sock_addr_func_proto,
11199 .is_valid_access = sock_addr_is_valid_access,
11200 .convert_ctx_access = sock_addr_convert_ctx_access,
11201};
11202
11203const struct bpf_prog_ops cg_sock_addr_prog_ops = {
11204};
11205
11206const struct bpf_verifier_ops sock_ops_verifier_ops = {
11207 .get_func_proto = sock_ops_func_proto,
11208 .is_valid_access = sock_ops_is_valid_access,
11209 .convert_ctx_access = sock_ops_convert_ctx_access,
11210};
11211
11212const struct bpf_prog_ops sock_ops_prog_ops = {
11213};
11214
11215const struct bpf_verifier_ops sk_skb_verifier_ops = {
11216 .get_func_proto = sk_skb_func_proto,
11217 .is_valid_access = sk_skb_is_valid_access,
11218 .convert_ctx_access = sk_skb_convert_ctx_access,
11219 .gen_prologue = sk_skb_prologue,
11220};
11221
11222const struct bpf_prog_ops sk_skb_prog_ops = {
11223};
11224
11225const struct bpf_verifier_ops sk_msg_verifier_ops = {
11226 .get_func_proto = sk_msg_func_proto,
11227 .is_valid_access = sk_msg_is_valid_access,
11228 .convert_ctx_access = sk_msg_convert_ctx_access,
11229 .gen_prologue = bpf_noop_prologue,
11230};
11231
11232const struct bpf_prog_ops sk_msg_prog_ops = {
11233};
11234
11235const struct bpf_verifier_ops flow_dissector_verifier_ops = {
11236 .get_func_proto = flow_dissector_func_proto,
11237 .is_valid_access = flow_dissector_is_valid_access,
11238 .convert_ctx_access = flow_dissector_convert_ctx_access,
11239};
11240
11241const struct bpf_prog_ops flow_dissector_prog_ops = {
11242 .test_run = bpf_prog_test_run_flow_dissector,
11243};
11244
11245int sk_detach_filter(struct sock *sk)
11246{
11247 int ret = -ENOENT;
11248 struct sk_filter *filter;
11249
11250 if (sock_flag(sk, flag: SOCK_FILTER_LOCKED))
11251 return -EPERM;
11252
11253 filter = rcu_dereference_protected(sk->sk_filter,
11254 lockdep_sock_is_held(sk));
11255 if (filter) {
11256 RCU_INIT_POINTER(sk->sk_filter, NULL);
11257 sk_filter_uncharge(sk, fp: filter);
11258 ret = 0;
11259 }
11260
11261 return ret;
11262}
11263EXPORT_SYMBOL_GPL(sk_detach_filter);
11264
11265int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len)
11266{
11267 struct sock_fprog_kern *fprog;
11268 struct sk_filter *filter;
11269 int ret = 0;
11270
11271 sockopt_lock_sock(sk);
11272 filter = rcu_dereference_protected(sk->sk_filter,
11273 lockdep_sock_is_held(sk));
11274 if (!filter)
11275 goto out;
11276
11277 /* We're copying the filter that has been originally attached,
11278 * so no conversion/decode needed anymore. eBPF programs that
11279 * have no original program cannot be dumped through this.
11280 */
11281 ret = -EACCES;
11282 fprog = filter->prog->orig_prog;
11283 if (!fprog)
11284 goto out;
11285
11286 ret = fprog->len;
11287 if (!len)
11288 /* User space only enquires number of filter blocks. */
11289 goto out;
11290
11291 ret = -EINVAL;
11292 if (len < fprog->len)
11293 goto out;
11294
11295 ret = -EFAULT;
11296 if (copy_to_sockptr(dst: optval, src: fprog->filter, bpf_classic_proglen(fprog)))
11297 goto out;
11298
11299 /* Instead of bytes, the API requests to return the number
11300 * of filter blocks.
11301 */
11302 ret = fprog->len;
11303out:
11304 sockopt_release_sock(sk);
11305 return ret;
11306}
11307
11308#ifdef CONFIG_INET
11309static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern,
11310 struct sock_reuseport *reuse,
11311 struct sock *sk, struct sk_buff *skb,
11312 struct sock *migrating_sk,
11313 u32 hash)
11314{
11315 reuse_kern->skb = skb;
11316 reuse_kern->sk = sk;
11317 reuse_kern->selected_sk = NULL;
11318 reuse_kern->migrating_sk = migrating_sk;
11319 reuse_kern->data_end = skb->data + skb_headlen(skb);
11320 reuse_kern->hash = hash;
11321 reuse_kern->reuseport_id = reuse->reuseport_id;
11322 reuse_kern->bind_inany = reuse->bind_inany;
11323}
11324
11325struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
11326 struct bpf_prog *prog, struct sk_buff *skb,
11327 struct sock *migrating_sk,
11328 u32 hash)
11329{
11330 struct sk_reuseport_kern reuse_kern;
11331 enum sk_action action;
11332
11333 bpf_init_reuseport_kern(reuse_kern: &reuse_kern, reuse, sk, skb, migrating_sk, hash);
11334 action = bpf_prog_run(prog, ctx: &reuse_kern);
11335
11336 if (action == SK_PASS)
11337 return reuse_kern.selected_sk;
11338 else
11339 return ERR_PTR(error: -ECONNREFUSED);
11340}
11341
11342BPF_CALL_4(sk_select_reuseport, struct sk_reuseport_kern *, reuse_kern,
11343 struct bpf_map *, map, void *, key, u32, flags)
11344{
11345 bool is_sockarray = map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
11346 struct sock_reuseport *reuse;
11347 struct sock *selected_sk;
11348 int err;
11349
11350 selected_sk = map->ops->map_lookup_elem(map, key);
11351 if (!selected_sk)
11352 return -ENOENT;
11353
11354 reuse = rcu_dereference(selected_sk->sk_reuseport_cb);
11355 if (!reuse) {
11356 /* reuseport_array has only sk with non NULL sk_reuseport_cb.
11357 * The only (!reuse) case here is - the sk has already been
11358 * unhashed (e.g. by close()), so treat it as -ENOENT.
11359 *
11360 * Other maps (e.g. sock_map) do not provide this guarantee and
11361 * the sk may never be in the reuseport group to begin with.
11362 */
11363 err = is_sockarray ? -ENOENT : -EINVAL;
11364 goto error;
11365 }
11366
11367 if (unlikely(reuse->reuseport_id != reuse_kern->reuseport_id)) {
11368 struct sock *sk = reuse_kern->sk;
11369
11370 if (sk->sk_protocol != selected_sk->sk_protocol) {
11371 err = -EPROTOTYPE;
11372 } else if (sk->sk_family != selected_sk->sk_family) {
11373 err = -EAFNOSUPPORT;
11374 } else {
11375 /* Catch all. Likely bound to a different sockaddr. */
11376 err = -EBADFD;
11377 }
11378 goto error;
11379 }
11380
11381 reuse_kern->selected_sk = selected_sk;
11382
11383 return 0;
11384error:
11385 /* Lookup in sock_map can return TCP ESTABLISHED sockets. */
11386 if (sk_is_refcounted(sk: selected_sk))
11387 sock_put(sk: selected_sk);
11388
11389 return err;
11390}
11391
11392static const struct bpf_func_proto sk_select_reuseport_proto = {
11393 .func = sk_select_reuseport,
11394 .gpl_only = false,
11395 .ret_type = RET_INTEGER,
11396 .arg1_type = ARG_PTR_TO_CTX,
11397 .arg2_type = ARG_CONST_MAP_PTR,
11398 .arg3_type = ARG_PTR_TO_MAP_KEY,
11399 .arg4_type = ARG_ANYTHING,
11400};
11401
11402BPF_CALL_4(sk_reuseport_load_bytes,
11403 const struct sk_reuseport_kern *, reuse_kern, u32, offset,
11404 void *, to, u32, len)
11405{
11406 return ____bpf_skb_load_bytes(skb: reuse_kern->skb, offset, to, len);
11407}
11408
11409static const struct bpf_func_proto sk_reuseport_load_bytes_proto = {
11410 .func = sk_reuseport_load_bytes,
11411 .gpl_only = false,
11412 .ret_type = RET_INTEGER,
11413 .arg1_type = ARG_PTR_TO_CTX,
11414 .arg2_type = ARG_ANYTHING,
11415 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
11416 .arg4_type = ARG_CONST_SIZE,
11417};
11418
11419BPF_CALL_5(sk_reuseport_load_bytes_relative,
11420 const struct sk_reuseport_kern *, reuse_kern, u32, offset,
11421 void *, to, u32, len, u32, start_header)
11422{
11423 return ____bpf_skb_load_bytes_relative(skb: reuse_kern->skb, offset, to,
11424 len, start_header);
11425}
11426
11427static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = {
11428 .func = sk_reuseport_load_bytes_relative,
11429 .gpl_only = false,
11430 .ret_type = RET_INTEGER,
11431 .arg1_type = ARG_PTR_TO_CTX,
11432 .arg2_type = ARG_ANYTHING,
11433 .arg3_type = ARG_PTR_TO_UNINIT_MEM,
11434 .arg4_type = ARG_CONST_SIZE,
11435 .arg5_type = ARG_ANYTHING,
11436};
11437
11438static const struct bpf_func_proto *
11439sk_reuseport_func_proto(enum bpf_func_id func_id,
11440 const struct bpf_prog *prog)
11441{
11442 switch (func_id) {
11443 case BPF_FUNC_sk_select_reuseport:
11444 return &sk_select_reuseport_proto;
11445 case BPF_FUNC_skb_load_bytes:
11446 return &sk_reuseport_load_bytes_proto;
11447 case BPF_FUNC_skb_load_bytes_relative:
11448 return &sk_reuseport_load_bytes_relative_proto;
11449 case BPF_FUNC_get_socket_cookie:
11450 return &bpf_get_socket_ptr_cookie_proto;
11451 case BPF_FUNC_ktime_get_coarse_ns:
11452 return &bpf_ktime_get_coarse_ns_proto;
11453 default:
11454 return bpf_base_func_proto(func_id, prog);
11455 }
11456}
11457
11458static bool
11459sk_reuseport_is_valid_access(int off, int size,
11460 enum bpf_access_type type,
11461 const struct bpf_prog *prog,
11462 struct bpf_insn_access_aux *info)
11463{
11464 const u32 size_default = sizeof(__u32);
11465
11466 if (off < 0 || off >= sizeof(struct sk_reuseport_md) ||
11467 off % size || type != BPF_READ)
11468 return false;
11469
11470 switch (off) {
11471 case offsetof(struct sk_reuseport_md, data):
11472 info->reg_type = PTR_TO_PACKET;
11473 return size == sizeof(__u64);
11474
11475 case offsetof(struct sk_reuseport_md, data_end):
11476 info->reg_type = PTR_TO_PACKET_END;
11477 return size == sizeof(__u64);
11478
11479 case offsetof(struct sk_reuseport_md, hash):
11480 return size == size_default;
11481
11482 case offsetof(struct sk_reuseport_md, sk):
11483 info->reg_type = PTR_TO_SOCKET;
11484 return size == sizeof(__u64);
11485
11486 case offsetof(struct sk_reuseport_md, migrating_sk):
11487 info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
11488 return size == sizeof(__u64);
11489
11490 /* Fields that allow narrowing */
11491 case bpf_ctx_range(struct sk_reuseport_md, eth_protocol):
11492 if (size < sizeof_field(struct sk_buff, protocol))
11493 return false;
11494 fallthrough;
11495 case bpf_ctx_range(struct sk_reuseport_md, ip_protocol):
11496 case bpf_ctx_range(struct sk_reuseport_md, bind_inany):
11497 case bpf_ctx_range(struct sk_reuseport_md, len):
11498 bpf_ctx_record_field_size(aux: info, size: size_default);
11499 return bpf_ctx_narrow_access_ok(off, size, size_default);
11500
11501 default:
11502 return false;
11503 }
11504}
11505
11506#define SK_REUSEPORT_LOAD_FIELD(F) ({ \
11507 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_reuseport_kern, F), \
11508 si->dst_reg, si->src_reg, \
11509 bpf_target_off(struct sk_reuseport_kern, F, \
11510 sizeof_field(struct sk_reuseport_kern, F), \
11511 target_size)); \
11512 })
11513
11514#define SK_REUSEPORT_LOAD_SKB_FIELD(SKB_FIELD) \
11515 SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern, \
11516 struct sk_buff, \
11517 skb, \
11518 SKB_FIELD)
11519
11520#define SK_REUSEPORT_LOAD_SK_FIELD(SK_FIELD) \
11521 SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern, \
11522 struct sock, \
11523 sk, \
11524 SK_FIELD)
11525
11526static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type,
11527 const struct bpf_insn *si,
11528 struct bpf_insn *insn_buf,
11529 struct bpf_prog *prog,
11530 u32 *target_size)
11531{
11532 struct bpf_insn *insn = insn_buf;
11533
11534 switch (si->off) {
11535 case offsetof(struct sk_reuseport_md, data):
11536 SK_REUSEPORT_LOAD_SKB_FIELD(data);
11537 break;
11538
11539 case offsetof(struct sk_reuseport_md, len):
11540 SK_REUSEPORT_LOAD_SKB_FIELD(len);
11541 break;
11542
11543 case offsetof(struct sk_reuseport_md, eth_protocol):
11544 SK_REUSEPORT_LOAD_SKB_FIELD(protocol);
11545 break;
11546
11547 case offsetof(struct sk_reuseport_md, ip_protocol):
11548 SK_REUSEPORT_LOAD_SK_FIELD(sk_protocol);
11549 break;
11550
11551 case offsetof(struct sk_reuseport_md, data_end):
11552 SK_REUSEPORT_LOAD_FIELD(data_end);
11553 break;
11554
11555 case offsetof(struct sk_reuseport_md, hash):
11556 SK_REUSEPORT_LOAD_FIELD(hash);
11557 break;
11558
11559 case offsetof(struct sk_reuseport_md, bind_inany):
11560 SK_REUSEPORT_LOAD_FIELD(bind_inany);
11561 break;
11562
11563 case offsetof(struct sk_reuseport_md, sk):
11564 SK_REUSEPORT_LOAD_FIELD(sk);
11565 break;
11566
11567 case offsetof(struct sk_reuseport_md, migrating_sk):
11568 SK_REUSEPORT_LOAD_FIELD(migrating_sk);
11569 break;
11570 }
11571
11572 return insn - insn_buf;
11573}
11574
11575const struct bpf_verifier_ops sk_reuseport_verifier_ops = {
11576 .get_func_proto = sk_reuseport_func_proto,
11577 .is_valid_access = sk_reuseport_is_valid_access,
11578 .convert_ctx_access = sk_reuseport_convert_ctx_access,
11579};
11580
11581const struct bpf_prog_ops sk_reuseport_prog_ops = {
11582};
11583
11584DEFINE_STATIC_KEY_FALSE(bpf_sk_lookup_enabled);
11585EXPORT_SYMBOL(bpf_sk_lookup_enabled);
11586
11587BPF_CALL_3(bpf_sk_lookup_assign, struct bpf_sk_lookup_kern *, ctx,
11588 struct sock *, sk, u64, flags)
11589{
11590 if (unlikely(flags & ~(BPF_SK_LOOKUP_F_REPLACE |
11591 BPF_SK_LOOKUP_F_NO_REUSEPORT)))
11592 return -EINVAL;
11593 if (unlikely(sk && sk_is_refcounted(sk)))
11594 return -ESOCKTNOSUPPORT; /* reject non-RCU freed sockets */
11595 if (unlikely(sk && sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN))
11596 return -ESOCKTNOSUPPORT; /* only accept TCP socket in LISTEN */
11597 if (unlikely(sk && sk_is_udp(sk) && sk->sk_state != TCP_CLOSE))
11598 return -ESOCKTNOSUPPORT; /* only accept UDP socket in CLOSE */
11599
11600 /* Check if socket is suitable for packet L3/L4 protocol */
11601 if (sk && sk->sk_protocol != ctx->protocol)
11602 return -EPROTOTYPE;
11603 if (sk && sk->sk_family != ctx->family &&
11604 (sk->sk_family == AF_INET || ipv6_only_sock(sk)))
11605 return -EAFNOSUPPORT;
11606
11607 if (ctx->selected_sk && !(flags & BPF_SK_LOOKUP_F_REPLACE))
11608 return -EEXIST;
11609
11610 /* Select socket as lookup result */
11611 ctx->selected_sk = sk;
11612 ctx->no_reuseport = flags & BPF_SK_LOOKUP_F_NO_REUSEPORT;
11613 return 0;
11614}
11615
11616static const struct bpf_func_proto bpf_sk_lookup_assign_proto = {
11617 .func = bpf_sk_lookup_assign,
11618 .gpl_only = false,
11619 .ret_type = RET_INTEGER,
11620 .arg1_type = ARG_PTR_TO_CTX,
11621 .arg2_type = ARG_PTR_TO_SOCKET_OR_NULL,
11622 .arg3_type = ARG_ANYTHING,
11623};
11624
11625static const struct bpf_func_proto *
11626sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
11627{
11628 switch (func_id) {
11629 case BPF_FUNC_perf_event_output:
11630 return &bpf_event_output_data_proto;
11631 case BPF_FUNC_sk_assign:
11632 return &bpf_sk_lookup_assign_proto;
11633 case BPF_FUNC_sk_release:
11634 return &bpf_sk_release_proto;
11635 default:
11636 return bpf_sk_base_func_proto(func_id, prog);
11637 }
11638}
11639
11640static bool sk_lookup_is_valid_access(int off, int size,
11641 enum bpf_access_type type,
11642 const struct bpf_prog *prog,
11643 struct bpf_insn_access_aux *info)
11644{
11645 if (off < 0 || off >= sizeof(struct bpf_sk_lookup))
11646 return false;
11647 if (off % size != 0)
11648 return false;
11649 if (type != BPF_READ)
11650 return false;
11651
11652 switch (off) {
11653 case bpf_ctx_range_ptr(struct bpf_sk_lookup, sk):
11654 info->reg_type = PTR_TO_SOCKET_OR_NULL;
11655 return size == sizeof(__u64);
11656
11657 case bpf_ctx_range(struct bpf_sk_lookup, family):
11658 case bpf_ctx_range(struct bpf_sk_lookup, protocol):
11659 case bpf_ctx_range(struct bpf_sk_lookup, remote_ip4):
11660 case bpf_ctx_range(struct bpf_sk_lookup, local_ip4):
11661 case bpf_ctx_range_till(struct bpf_sk_lookup, remote_ip6[0], remote_ip6[3]):
11662 case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]):
11663 case bpf_ctx_range(struct bpf_sk_lookup, local_port):
11664 case bpf_ctx_range(struct bpf_sk_lookup, ingress_ifindex):
11665 bpf_ctx_record_field_size(aux: info, size: sizeof(__u32));
11666 return bpf_ctx_narrow_access_ok(off, size, size_default: sizeof(__u32));
11667
11668 case bpf_ctx_range(struct bpf_sk_lookup, remote_port):
11669 /* Allow 4-byte access to 2-byte field for backward compatibility */
11670 if (size == sizeof(__u32))
11671 return true;
11672 bpf_ctx_record_field_size(aux: info, size: sizeof(__be16));
11673 return bpf_ctx_narrow_access_ok(off, size, size_default: sizeof(__be16));
11674
11675 case offsetofend(struct bpf_sk_lookup, remote_port) ...
11676 offsetof(struct bpf_sk_lookup, local_ip4) - 1:
11677 /* Allow access to zero padding for backward compatibility */
11678 bpf_ctx_record_field_size(aux: info, size: sizeof(__u16));
11679 return bpf_ctx_narrow_access_ok(off, size, size_default: sizeof(__u16));
11680
11681 default:
11682 return false;
11683 }
11684}
11685
11686static u32 sk_lookup_convert_ctx_access(enum bpf_access_type type,
11687 const struct bpf_insn *si,
11688 struct bpf_insn *insn_buf,
11689 struct bpf_prog *prog,
11690 u32 *target_size)
11691{
11692 struct bpf_insn *insn = insn_buf;
11693
11694 switch (si->off) {
11695 case offsetof(struct bpf_sk_lookup, sk):
11696 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
11697 offsetof(struct bpf_sk_lookup_kern, selected_sk));
11698 break;
11699
11700 case offsetof(struct bpf_sk_lookup, family):
11701 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
11702 bpf_target_off(struct bpf_sk_lookup_kern,
11703 family, 2, target_size));
11704 break;
11705
11706 case offsetof(struct bpf_sk_lookup, protocol):
11707 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
11708 bpf_target_off(struct bpf_sk_lookup_kern,
11709 protocol, 2, target_size));
11710 break;
11711
11712 case offsetof(struct bpf_sk_lookup, remote_ip4):
11713 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
11714 bpf_target_off(struct bpf_sk_lookup_kern,
11715 v4.saddr, 4, target_size));
11716 break;
11717
11718 case offsetof(struct bpf_sk_lookup, local_ip4):
11719 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
11720 bpf_target_off(struct bpf_sk_lookup_kern,
11721 v4.daddr, 4, target_size));
11722 break;
11723
11724 case bpf_ctx_range_till(struct bpf_sk_lookup,
11725 remote_ip6[0], remote_ip6[3]): {
11726#if IS_ENABLED(CONFIG_IPV6)
11727 int off = si->off;
11728
11729 off -= offsetof(struct bpf_sk_lookup, remote_ip6[0]);
11730 off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
11731 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
11732 offsetof(struct bpf_sk_lookup_kern, v6.saddr));
11733 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
11734 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
11735#else
11736 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11737#endif
11738 break;
11739 }
11740 case bpf_ctx_range_till(struct bpf_sk_lookup,
11741 local_ip6[0], local_ip6[3]): {
11742#if IS_ENABLED(CONFIG_IPV6)
11743 int off = si->off;
11744
11745 off -= offsetof(struct bpf_sk_lookup, local_ip6[0]);
11746 off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
11747 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
11748 offsetof(struct bpf_sk_lookup_kern, v6.daddr));
11749 *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
11750 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
11751#else
11752 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11753#endif
11754 break;
11755 }
11756 case offsetof(struct bpf_sk_lookup, remote_port):
11757 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
11758 bpf_target_off(struct bpf_sk_lookup_kern,
11759 sport, 2, target_size));
11760 break;
11761
11762 case offsetofend(struct bpf_sk_lookup, remote_port):
11763 *target_size = 2;
11764 *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11765 break;
11766
11767 case offsetof(struct bpf_sk_lookup, local_port):
11768 *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
11769 bpf_target_off(struct bpf_sk_lookup_kern,
11770 dport, 2, target_size));
11771 break;
11772
11773 case offsetof(struct bpf_sk_lookup, ingress_ifindex):
11774 *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
11775 bpf_target_off(struct bpf_sk_lookup_kern,
11776 ingress_ifindex, 4, target_size));
11777 break;
11778 }
11779
11780 return insn - insn_buf;
11781}
11782
11783const struct bpf_prog_ops sk_lookup_prog_ops = {
11784 .test_run = bpf_prog_test_run_sk_lookup,
11785};
11786
11787const struct bpf_verifier_ops sk_lookup_verifier_ops = {
11788 .get_func_proto = sk_lookup_func_proto,
11789 .is_valid_access = sk_lookup_is_valid_access,
11790 .convert_ctx_access = sk_lookup_convert_ctx_access,
11791};
11792
11793#endif /* CONFIG_INET */
11794
11795DEFINE_BPF_DISPATCHER(xdp)
11796
11797void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog)
11798{
11799 bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(xdp), from: prev_prog, to: prog);
11800}
11801
11802BTF_ID_LIST_GLOBAL(btf_sock_ids, MAX_BTF_SOCK_TYPE)
11803#define BTF_SOCK_TYPE(name, type) BTF_ID(struct, type)
11804BTF_SOCK_TYPE_xxx
11805#undef BTF_SOCK_TYPE
11806
11807BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
11808{
11809 /* tcp6_sock type is not generated in dwarf and hence btf,
11810 * trigger an explicit type generation here.
11811 */
11812 BTF_TYPE_EMIT(struct tcp6_sock);
11813 if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
11814 sk->sk_family == AF_INET6)
11815 return (unsigned long)sk;
11816
11817 return (unsigned long)NULL;
11818}
11819
11820const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
11821 .func = bpf_skc_to_tcp6_sock,
11822 .gpl_only = false,
11823 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11824 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
11825 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP6],
11826};
11827
11828BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
11829{
11830 if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
11831 return (unsigned long)sk;
11832
11833 return (unsigned long)NULL;
11834}
11835
11836const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = {
11837 .func = bpf_skc_to_tcp_sock,
11838 .gpl_only = false,
11839 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11840 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
11841 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
11842};
11843
11844BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
11845{
11846 /* BTF types for tcp_timewait_sock and inet_timewait_sock are not
11847 * generated if CONFIG_INET=n. Trigger an explicit generation here.
11848 */
11849 BTF_TYPE_EMIT(struct inet_timewait_sock);
11850 BTF_TYPE_EMIT(struct tcp_timewait_sock);
11851
11852#ifdef CONFIG_INET
11853 if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
11854 return (unsigned long)sk;
11855#endif
11856
11857#if IS_BUILTIN(CONFIG_IPV6)
11858 if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_TIME_WAIT)
11859 return (unsigned long)sk;
11860#endif
11861
11862 return (unsigned long)NULL;
11863}
11864
11865const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = {
11866 .func = bpf_skc_to_tcp_timewait_sock,
11867 .gpl_only = false,
11868 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11869 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
11870 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP_TW],
11871};
11872
11873BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk)
11874{
11875#ifdef CONFIG_INET
11876 if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV)
11877 return (unsigned long)sk;
11878#endif
11879
11880#if IS_BUILTIN(CONFIG_IPV6)
11881 if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_NEW_SYN_RECV)
11882 return (unsigned long)sk;
11883#endif
11884
11885 return (unsigned long)NULL;
11886}
11887
11888const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto = {
11889 .func = bpf_skc_to_tcp_request_sock,
11890 .gpl_only = false,
11891 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11892 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
11893 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP_REQ],
11894};
11895
11896BPF_CALL_1(bpf_skc_to_udp6_sock, struct sock *, sk)
11897{
11898 /* udp6_sock type is not generated in dwarf and hence btf,
11899 * trigger an explicit type generation here.
11900 */
11901 BTF_TYPE_EMIT(struct udp6_sock);
11902 if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_UDP &&
11903 sk->sk_type == SOCK_DGRAM && sk->sk_family == AF_INET6)
11904 return (unsigned long)sk;
11905
11906 return (unsigned long)NULL;
11907}
11908
11909const struct bpf_func_proto bpf_skc_to_udp6_sock_proto = {
11910 .func = bpf_skc_to_udp6_sock,
11911 .gpl_only = false,
11912 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11913 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
11914 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_UDP6],
11915};
11916
11917BPF_CALL_1(bpf_skc_to_unix_sock, struct sock *, sk)
11918{
11919 /* unix_sock type is not generated in dwarf and hence btf,
11920 * trigger an explicit type generation here.
11921 */
11922 BTF_TYPE_EMIT(struct unix_sock);
11923 if (sk && sk_fullsock(sk) && sk->sk_family == AF_UNIX)
11924 return (unsigned long)sk;
11925
11926 return (unsigned long)NULL;
11927}
11928
11929const struct bpf_func_proto bpf_skc_to_unix_sock_proto = {
11930 .func = bpf_skc_to_unix_sock,
11931 .gpl_only = false,
11932 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11933 .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
11934 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_UNIX],
11935};
11936
11937BPF_CALL_1(bpf_skc_to_mptcp_sock, struct sock *, sk)
11938{
11939 BTF_TYPE_EMIT(struct mptcp_sock);
11940 return (unsigned long)bpf_mptcp_sock_from_subflow(sk);
11941}
11942
11943const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto = {
11944 .func = bpf_skc_to_mptcp_sock,
11945 .gpl_only = false,
11946 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11947 .arg1_type = ARG_PTR_TO_SOCK_COMMON,
11948 .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_MPTCP],
11949};
11950
11951BPF_CALL_1(bpf_sock_from_file, struct file *, file)
11952{
11953 return (unsigned long)sock_from_file(file);
11954}
11955
11956BTF_ID_LIST(bpf_sock_from_file_btf_ids)
11957BTF_ID(struct, socket)
11958BTF_ID(struct, file)
11959
11960const struct bpf_func_proto bpf_sock_from_file_proto = {
11961 .func = bpf_sock_from_file,
11962 .gpl_only = false,
11963 .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
11964 .ret_btf_id = &bpf_sock_from_file_btf_ids[0],
11965 .arg1_type = ARG_PTR_TO_BTF_ID,
11966 .arg1_btf_id = &bpf_sock_from_file_btf_ids[1],
11967};
11968
11969static const struct bpf_func_proto *
11970bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
11971{
11972 const struct bpf_func_proto *func;
11973
11974 switch (func_id) {
11975 case BPF_FUNC_skc_to_tcp6_sock:
11976 func = &bpf_skc_to_tcp6_sock_proto;
11977 break;
11978 case BPF_FUNC_skc_to_tcp_sock:
11979 func = &bpf_skc_to_tcp_sock_proto;
11980 break;
11981 case BPF_FUNC_skc_to_tcp_timewait_sock:
11982 func = &bpf_skc_to_tcp_timewait_sock_proto;
11983 break;
11984 case BPF_FUNC_skc_to_tcp_request_sock:
11985 func = &bpf_skc_to_tcp_request_sock_proto;
11986 break;
11987 case BPF_FUNC_skc_to_udp6_sock:
11988 func = &bpf_skc_to_udp6_sock_proto;
11989 break;
11990 case BPF_FUNC_skc_to_unix_sock:
11991 func = &bpf_skc_to_unix_sock_proto;
11992 break;
11993 case BPF_FUNC_skc_to_mptcp_sock:
11994 func = &bpf_skc_to_mptcp_sock_proto;
11995 break;
11996 case BPF_FUNC_ktime_get_coarse_ns:
11997 return &bpf_ktime_get_coarse_ns_proto;
11998 default:
11999 return bpf_base_func_proto(func_id, prog);
12000 }
12001
12002 if (!bpf_token_capable(token: prog->aux->token, CAP_PERFMON))
12003 return NULL;
12004
12005 return func;
12006}
12007
12008/**
12009 * bpf_skb_meta_pointer() - Gets a mutable pointer within the skb metadata area.
12010 * @skb: socket buffer carrying the metadata
12011 * @offset: offset into the metadata area, must be <= skb_metadata_len()
12012 */
12013void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset)
12014{
12015 return skb_metadata_end(skb) - skb_metadata_len(skb) + offset;
12016}
12017
12018__bpf_kfunc_start_defs();
12019__bpf_kfunc int bpf_dynptr_from_skb(struct __sk_buff *s, u64 flags,
12020 struct bpf_dynptr *ptr__uninit)
12021{
12022 struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12023 struct sk_buff *skb = (struct sk_buff *)s;
12024
12025 if (flags) {
12026 bpf_dynptr_set_null(ptr);
12027 return -EINVAL;
12028 }
12029
12030 bpf_dynptr_init(ptr, data: skb, type: BPF_DYNPTR_TYPE_SKB, offset: 0, size: skb->len);
12031
12032 return 0;
12033}
12034
12035/**
12036 * bpf_dynptr_from_skb_meta() - Initialize a dynptr to the skb metadata area.
12037 * @skb_: socket buffer carrying the metadata
12038 * @flags: future use, must be zero
12039 * @ptr__uninit: dynptr to initialize
12040 *
12041 * Set up a dynptr for access to the metadata area earlier allocated from the
12042 * XDP context with bpf_xdp_adjust_meta(). Serves as an alternative to
12043 * &__sk_buff->data_meta.
12044 *
12045 * If passed @skb_ is a clone which shares the data with the original, the
12046 * dynptr will be read-only. This limitation may be lifted in the future.
12047 *
12048 * Return:
12049 * * %0 - dynptr ready to use
12050 * * %-EINVAL - invalid flags, dynptr set to null
12051 */
12052__bpf_kfunc int bpf_dynptr_from_skb_meta(struct __sk_buff *skb_, u64 flags,
12053 struct bpf_dynptr *ptr__uninit)
12054{
12055 struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12056 struct sk_buff *skb = (struct sk_buff *)skb_;
12057
12058 if (flags) {
12059 bpf_dynptr_set_null(ptr);
12060 return -EINVAL;
12061 }
12062
12063 bpf_dynptr_init(ptr, data: skb, type: BPF_DYNPTR_TYPE_SKB_META, offset: 0, size: skb_metadata_len(skb));
12064
12065 if (skb_cloned(skb))
12066 bpf_dynptr_set_rdonly(ptr);
12067
12068 return 0;
12069}
12070
12071__bpf_kfunc int bpf_dynptr_from_xdp(struct xdp_md *x, u64 flags,
12072 struct bpf_dynptr *ptr__uninit)
12073{
12074 struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12075 struct xdp_buff *xdp = (struct xdp_buff *)x;
12076
12077 if (flags) {
12078 bpf_dynptr_set_null(ptr);
12079 return -EINVAL;
12080 }
12081
12082 bpf_dynptr_init(ptr, data: xdp, type: BPF_DYNPTR_TYPE_XDP, offset: 0, size: xdp_get_buff_len(xdp));
12083
12084 return 0;
12085}
12086
12087__bpf_kfunc int bpf_sock_addr_set_sun_path(struct bpf_sock_addr_kern *sa_kern,
12088 const u8 *sun_path, u32 sun_path__sz)
12089{
12090 struct sockaddr_un *un;
12091
12092 if (sa_kern->sk->sk_family != AF_UNIX)
12093 return -EINVAL;
12094
12095 /* We do not allow changing the address to unnamed or larger than the
12096 * maximum allowed address size for a unix sockaddr.
12097 */
12098 if (sun_path__sz == 0 || sun_path__sz > UNIX_PATH_MAX)
12099 return -EINVAL;
12100
12101 un = (struct sockaddr_un *)sa_kern->uaddr;
12102 memcpy(to: un->sun_path, from: sun_path, len: sun_path__sz);
12103 sa_kern->uaddrlen = offsetof(struct sockaddr_un, sun_path) + sun_path__sz;
12104
12105 return 0;
12106}
12107
12108__bpf_kfunc int bpf_sk_assign_tcp_reqsk(struct __sk_buff *s, struct sock *sk,
12109 struct bpf_tcp_req_attrs *attrs, int attrs__sz)
12110{
12111#if IS_ENABLED(CONFIG_SYN_COOKIES)
12112 struct sk_buff *skb = (struct sk_buff *)s;
12113 const struct request_sock_ops *ops;
12114 struct inet_request_sock *ireq;
12115 struct tcp_request_sock *treq;
12116 struct request_sock *req;
12117 struct net *net;
12118 __u16 min_mss;
12119 u32 tsoff = 0;
12120
12121 if (attrs__sz != sizeof(*attrs) ||
12122 attrs->reserved[0] || attrs->reserved[1] || attrs->reserved[2])
12123 return -EINVAL;
12124
12125 if (!skb_at_tc_ingress(skb))
12126 return -EINVAL;
12127
12128 net = dev_net(dev: skb->dev);
12129 if (net != sock_net(sk))
12130 return -ENETUNREACH;
12131
12132 switch (skb->protocol) {
12133 case htons(ETH_P_IP):
12134 ops = &tcp_request_sock_ops;
12135 min_mss = 536;
12136 break;
12137#if IS_BUILTIN(CONFIG_IPV6)
12138 case htons(ETH_P_IPV6):
12139 ops = &tcp6_request_sock_ops;
12140 min_mss = IPV6_MIN_MTU - 60;
12141 break;
12142#endif
12143 default:
12144 return -EINVAL;
12145 }
12146
12147 if (sk->sk_type != SOCK_STREAM || sk->sk_state != TCP_LISTEN ||
12148 sk_is_mptcp(sk))
12149 return -EINVAL;
12150
12151 if (attrs->mss < min_mss)
12152 return -EINVAL;
12153
12154 if (attrs->wscale_ok) {
12155 if (!READ_ONCE(net->ipv4.sysctl_tcp_window_scaling))
12156 return -EINVAL;
12157
12158 if (attrs->snd_wscale > TCP_MAX_WSCALE ||
12159 attrs->rcv_wscale > TCP_MAX_WSCALE)
12160 return -EINVAL;
12161 }
12162
12163 if (attrs->sack_ok && !READ_ONCE(net->ipv4.sysctl_tcp_sack))
12164 return -EINVAL;
12165
12166 if (attrs->tstamp_ok) {
12167 if (!READ_ONCE(net->ipv4.sysctl_tcp_timestamps))
12168 return -EINVAL;
12169
12170 tsoff = attrs->rcv_tsecr - tcp_ns_to_ts(usec_ts: attrs->usec_ts_ok, val: tcp_clock_ns());
12171 }
12172
12173 req = inet_reqsk_alloc(ops, sk_listener: sk, attach_listener: false);
12174 if (!req)
12175 return -ENOMEM;
12176
12177 ireq = inet_rsk(sk: req);
12178 treq = tcp_rsk(req);
12179
12180 req->rsk_listener = sk;
12181 req->syncookie = 1;
12182 req->mss = attrs->mss;
12183 req->ts_recent = attrs->rcv_tsval;
12184
12185 ireq->snd_wscale = attrs->snd_wscale;
12186 ireq->rcv_wscale = attrs->rcv_wscale;
12187 ireq->tstamp_ok = !!attrs->tstamp_ok;
12188 ireq->sack_ok = !!attrs->sack_ok;
12189 ireq->wscale_ok = !!attrs->wscale_ok;
12190 ireq->ecn_ok = !!attrs->ecn_ok;
12191
12192 treq->req_usec_ts = !!attrs->usec_ts_ok;
12193 treq->ts_off = tsoff;
12194
12195 skb_orphan(skb);
12196 skb->sk = req_to_sk(req);
12197 skb->destructor = sock_pfree;
12198
12199 return 0;
12200#else
12201 return -EOPNOTSUPP;
12202#endif
12203}
12204
12205__bpf_kfunc int bpf_sock_ops_enable_tx_tstamp(struct bpf_sock_ops_kern *skops,
12206 u64 flags)
12207{
12208 struct sk_buff *skb;
12209
12210 if (skops->op != BPF_SOCK_OPS_TSTAMP_SENDMSG_CB)
12211 return -EOPNOTSUPP;
12212
12213 if (flags)
12214 return -EINVAL;
12215
12216 skb = skops->skb;
12217 skb_shinfo(skb)->tx_flags |= SKBTX_BPF;
12218 TCP_SKB_CB(skb)->txstamp_ack |= TSTAMP_ACK_BPF;
12219 skb_shinfo(skb)->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
12220
12221 return 0;
12222}
12223
12224/**
12225 * bpf_xdp_pull_data() - Pull in non-linear xdp data.
12226 * @x: &xdp_md associated with the XDP buffer
12227 * @len: length of data to be made directly accessible in the linear part
12228 *
12229 * Pull in data in case the XDP buffer associated with @x is non-linear and
12230 * not all @len are in the linear data area.
12231 *
12232 * Direct packet access allows reading and writing linear XDP data through
12233 * packet pointers (i.e., &xdp_md->data + offsets). The amount of data which
12234 * ends up in the linear part of the xdp_buff depends on the NIC and its
12235 * configuration. When a frag-capable XDP program wants to directly access
12236 * headers that may be in the non-linear area, call this kfunc to make sure
12237 * the data is available in the linear area. Alternatively, use dynptr or
12238 * bpf_xdp_{load,store}_bytes() to access data without pulling.
12239 *
12240 * This kfunc can also be used with bpf_xdp_adjust_head() to decapsulate
12241 * headers in the non-linear data area.
12242 *
12243 * A call to this kfunc may reduce headroom. If there is not enough tailroom
12244 * in the linear data area, metadata and data will be shifted down.
12245 *
12246 * A call to this kfunc is susceptible to change the buffer geometry.
12247 * Therefore, at load time, all checks on pointers previously done by the
12248 * verifier are invalidated and must be performed again, if the kfunc is used
12249 * in combination with direct packet access.
12250 *
12251 * Return:
12252 * * %0 - success
12253 * * %-EINVAL - invalid len
12254 */
12255__bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len)
12256{
12257 struct xdp_buff *xdp = (struct xdp_buff *)x;
12258 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
12259 int i, delta, shift, headroom, tailroom, n_frags_free = 0;
12260 void *data_hard_end = xdp_data_hard_end(xdp);
12261 int data_len = xdp->data_end - xdp->data;
12262 void *start;
12263
12264 if (len <= data_len)
12265 return 0;
12266
12267 if (unlikely(len > xdp_get_buff_len(xdp)))
12268 return -EINVAL;
12269
12270 start = xdp_data_meta_unsupported(xdp) ? xdp->data : xdp->data_meta;
12271
12272 headroom = start - xdp->data_hard_start - sizeof(struct xdp_frame);
12273 tailroom = data_hard_end - xdp->data_end;
12274
12275 delta = len - data_len;
12276 if (unlikely(delta > tailroom + headroom))
12277 return -EINVAL;
12278
12279 shift = delta - tailroom;
12280 if (shift > 0) {
12281 memmove(dest: start - shift, src: start, count: xdp->data_end - start);
12282
12283 xdp->data_meta -= shift;
12284 xdp->data -= shift;
12285 xdp->data_end -= shift;
12286 }
12287
12288 for (i = 0; i < sinfo->nr_frags && delta; i++) {
12289 skb_frag_t *frag = &sinfo->frags[i];
12290 u32 shrink = min_t(u32, delta, skb_frag_size(frag));
12291
12292 memcpy(to: xdp->data_end, from: skb_frag_address(frag), len: shrink);
12293
12294 xdp->data_end += shrink;
12295 sinfo->xdp_frags_size -= shrink;
12296 delta -= shrink;
12297 if (bpf_xdp_shrink_data(xdp, frag, shrink, tail: false))
12298 n_frags_free++;
12299 }
12300
12301 if (unlikely(n_frags_free)) {
12302 memmove(dest: sinfo->frags, src: sinfo->frags + n_frags_free,
12303 count: (sinfo->nr_frags - n_frags_free) * sizeof(skb_frag_t));
12304
12305 sinfo->nr_frags -= n_frags_free;
12306
12307 if (!sinfo->nr_frags) {
12308 xdp_buff_clear_frags_flag(xdp);
12309 xdp_buff_clear_frag_pfmemalloc(xdp);
12310 }
12311 }
12312
12313 return 0;
12314}
12315
12316__bpf_kfunc_end_defs();
12317
12318int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
12319 struct bpf_dynptr *ptr__uninit)
12320{
12321 struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12322 int err;
12323
12324 err = bpf_dynptr_from_skb(s: skb, flags, ptr__uninit);
12325 if (err)
12326 return err;
12327
12328 bpf_dynptr_set_rdonly(ptr);
12329
12330 return 0;
12331}
12332
12333BTF_KFUNCS_START(bpf_kfunc_check_set_skb)
12334BTF_ID_FLAGS(func, bpf_dynptr_from_skb, KF_TRUSTED_ARGS)
12335BTF_KFUNCS_END(bpf_kfunc_check_set_skb)
12336
12337BTF_KFUNCS_START(bpf_kfunc_check_set_skb_meta)
12338BTF_ID_FLAGS(func, bpf_dynptr_from_skb_meta, KF_TRUSTED_ARGS)
12339BTF_KFUNCS_END(bpf_kfunc_check_set_skb_meta)
12340
12341BTF_KFUNCS_START(bpf_kfunc_check_set_xdp)
12342BTF_ID_FLAGS(func, bpf_dynptr_from_xdp)
12343BTF_ID_FLAGS(func, bpf_xdp_pull_data)
12344BTF_KFUNCS_END(bpf_kfunc_check_set_xdp)
12345
12346BTF_KFUNCS_START(bpf_kfunc_check_set_sock_addr)
12347BTF_ID_FLAGS(func, bpf_sock_addr_set_sun_path)
12348BTF_KFUNCS_END(bpf_kfunc_check_set_sock_addr)
12349
12350BTF_KFUNCS_START(bpf_kfunc_check_set_tcp_reqsk)
12351BTF_ID_FLAGS(func, bpf_sk_assign_tcp_reqsk, KF_TRUSTED_ARGS)
12352BTF_KFUNCS_END(bpf_kfunc_check_set_tcp_reqsk)
12353
12354BTF_KFUNCS_START(bpf_kfunc_check_set_sock_ops)
12355BTF_ID_FLAGS(func, bpf_sock_ops_enable_tx_tstamp, KF_TRUSTED_ARGS)
12356BTF_KFUNCS_END(bpf_kfunc_check_set_sock_ops)
12357
12358static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
12359 .owner = THIS_MODULE,
12360 .set = &bpf_kfunc_check_set_skb,
12361};
12362
12363static const struct btf_kfunc_id_set bpf_kfunc_set_skb_meta = {
12364 .owner = THIS_MODULE,
12365 .set = &bpf_kfunc_check_set_skb_meta,
12366};
12367
12368static const struct btf_kfunc_id_set bpf_kfunc_set_xdp = {
12369 .owner = THIS_MODULE,
12370 .set = &bpf_kfunc_check_set_xdp,
12371};
12372
12373static const struct btf_kfunc_id_set bpf_kfunc_set_sock_addr = {
12374 .owner = THIS_MODULE,
12375 .set = &bpf_kfunc_check_set_sock_addr,
12376};
12377
12378static const struct btf_kfunc_id_set bpf_kfunc_set_tcp_reqsk = {
12379 .owner = THIS_MODULE,
12380 .set = &bpf_kfunc_check_set_tcp_reqsk,
12381};
12382
12383static const struct btf_kfunc_id_set bpf_kfunc_set_sock_ops = {
12384 .owner = THIS_MODULE,
12385 .set = &bpf_kfunc_check_set_sock_ops,
12386};
12387
12388static int __init bpf_kfunc_init(void)
12389{
12390 int ret;
12391
12392 ret = register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_CLS, s: &bpf_kfunc_set_skb);
12393 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_ACT, s: &bpf_kfunc_set_skb);
12394 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SK_SKB, s: &bpf_kfunc_set_skb);
12395 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SOCKET_FILTER, s: &bpf_kfunc_set_skb);
12396 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_CGROUP_SKB, s: &bpf_kfunc_set_skb);
12397 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_LWT_OUT, s: &bpf_kfunc_set_skb);
12398 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_LWT_IN, s: &bpf_kfunc_set_skb);
12399 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_LWT_XMIT, s: &bpf_kfunc_set_skb);
12400 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_LWT_SEG6LOCAL, s: &bpf_kfunc_set_skb);
12401 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_NETFILTER, s: &bpf_kfunc_set_skb);
12402 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_TRACING, s: &bpf_kfunc_set_skb);
12403 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_CLS, s: &bpf_kfunc_set_skb_meta);
12404 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_ACT, s: &bpf_kfunc_set_skb_meta);
12405 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_XDP, s: &bpf_kfunc_set_xdp);
12406 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
12407 s: &bpf_kfunc_set_sock_addr);
12408 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_CLS, s: &bpf_kfunc_set_tcp_reqsk);
12409 return ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SOCK_OPS, s: &bpf_kfunc_set_sock_ops);
12410}
12411late_initcall(bpf_kfunc_init);
12412
12413__bpf_kfunc_start_defs();
12414
12415/* bpf_sock_destroy: Destroy the given socket with ECONNABORTED error code.
12416 *
12417 * The function expects a non-NULL pointer to a socket, and invokes the
12418 * protocol specific socket destroy handlers.
12419 *
12420 * The helper can only be called from BPF contexts that have acquired the socket
12421 * locks.
12422 *
12423 * Parameters:
12424 * @sock: Pointer to socket to be destroyed
12425 *
12426 * Return:
12427 * On error, may return EPROTONOSUPPORT, EINVAL.
12428 * EPROTONOSUPPORT if protocol specific destroy handler is not supported.
12429 * 0 otherwise
12430 */
12431__bpf_kfunc int bpf_sock_destroy(struct sock_common *sock)
12432{
12433 struct sock *sk = (struct sock *)sock;
12434
12435 /* The locking semantics that allow for synchronous execution of the
12436 * destroy handlers are only supported for TCP and UDP.
12437 * Supporting protocols will need to acquire sock lock in the BPF context
12438 * prior to invoking this kfunc.
12439 */
12440 if (!sk->sk_prot->diag_destroy || (sk->sk_protocol != IPPROTO_TCP &&
12441 sk->sk_protocol != IPPROTO_UDP))
12442 return -EOPNOTSUPP;
12443
12444 return sk->sk_prot->diag_destroy(sk, ECONNABORTED);
12445}
12446
12447__bpf_kfunc_end_defs();
12448
12449BTF_KFUNCS_START(bpf_sk_iter_kfunc_ids)
12450BTF_ID_FLAGS(func, bpf_sock_destroy, KF_TRUSTED_ARGS)
12451BTF_KFUNCS_END(bpf_sk_iter_kfunc_ids)
12452
12453static int tracing_iter_filter(const struct bpf_prog *prog, u32 kfunc_id)
12454{
12455 if (btf_id_set8_contains(set: &bpf_sk_iter_kfunc_ids, id: kfunc_id) &&
12456 prog->expected_attach_type != BPF_TRACE_ITER)
12457 return -EACCES;
12458 return 0;
12459}
12460
12461static const struct btf_kfunc_id_set bpf_sk_iter_kfunc_set = {
12462 .owner = THIS_MODULE,
12463 .set = &bpf_sk_iter_kfunc_ids,
12464 .filter = tracing_iter_filter,
12465};
12466
12467static int init_subsystem(void)
12468{
12469 return register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_TRACING, s: &bpf_sk_iter_kfunc_set);
12470}
12471late_initcall(init_subsystem);
12472