1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * IPV4 GSO/GRO offload support
4 * Linux INET implementation
5 *
6 * UDPv4 GSO support
7 */
8
9#include <linux/skbuff.h>
10#include <net/gro.h>
11#include <net/gso.h>
12#include <net/udp.h>
13#include <net/protocol.h>
14#include <net/inet_common.h>
15#include <net/udp_tunnel.h>
16
17#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
18
19/*
20 * Dummy GRO tunnel callback, exists mainly to avoid dangling/NULL
21 * values for the udp tunnel static call.
22 */
23static struct sk_buff *dummy_gro_rcv(struct sock *sk,
24 struct list_head *head,
25 struct sk_buff *skb)
26{
27 NAPI_GRO_CB(skb)->flush = 1;
28 return NULL;
29}
30
31typedef struct sk_buff *(*udp_tunnel_gro_rcv_t)(struct sock *sk,
32 struct list_head *head,
33 struct sk_buff *skb);
34
35struct udp_tunnel_type_entry {
36 udp_tunnel_gro_rcv_t gro_receive;
37 refcount_t count;
38};
39
40#define UDP_MAX_TUNNEL_TYPES (IS_ENABLED(CONFIG_GENEVE) + \
41 IS_ENABLED(CONFIG_VXLAN) * 2 + \
42 IS_ENABLED(CONFIG_NET_FOU) * 2 + \
43 IS_ENABLED(CONFIG_XFRM) * 2)
44
45DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
46static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);
47static DEFINE_MUTEX(udp_tunnel_gro_type_lock);
48static struct udp_tunnel_type_entry udp_tunnel_gro_types[UDP_MAX_TUNNEL_TYPES];
49static unsigned int udp_tunnel_gro_type_nr;
50static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
51
52void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
53{
54 bool is_ipv6 = sk->sk_family == AF_INET6;
55 struct udp_sock *tup, *up = udp_sk(sk);
56 struct udp_tunnel_gro *udp_tunnel_gro;
57
58 spin_lock(&udp_tunnel_gro_lock);
59 udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];
60 if (add)
61 hlist_add_head(&up->tunnel_list, &udp_tunnel_gro->list);
62 else if (up->tunnel_list.pprev)
63 hlist_del_init(&up->tunnel_list);
64
65 if (udp_tunnel_gro->list.first &&
66 !udp_tunnel_gro->list.first->next) {
67 tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
68 tunnel_list);
69
70 rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
71 } else {
72 RCU_INIT_POINTER(udp_tunnel_gro->sk, NULL);
73 }
74
75 spin_unlock(&udp_tunnel_gro_lock);
76}
77EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
78
79void udp_tunnel_update_gro_rcv(struct sock *sk, bool add)
80{
81 struct udp_tunnel_type_entry *cur = NULL;
82 struct udp_sock *up = udp_sk(sk);
83 int i, old_gro_type_nr;
84
85 if (!UDP_MAX_TUNNEL_TYPES || !up->gro_receive)
86 return;
87
88 mutex_lock(&udp_tunnel_gro_type_lock);
89
90 /* Check if the static call is permanently disabled. */
91 if (udp_tunnel_gro_type_nr > UDP_MAX_TUNNEL_TYPES)
92 goto out;
93
94 for (i = 0; i < udp_tunnel_gro_type_nr; i++)
95 if (udp_tunnel_gro_types[i].gro_receive == up->gro_receive)
96 cur = &udp_tunnel_gro_types[i];
97
98 old_gro_type_nr = udp_tunnel_gro_type_nr;
99 if (add) {
100 /*
101 * Update the matching entry, if found, or add a new one
102 * if needed
103 */
104 if (cur) {
105 refcount_inc(&cur->count);
106 goto out;
107 }
108
109 if (unlikely(udp_tunnel_gro_type_nr == UDP_MAX_TUNNEL_TYPES)) {
110 pr_err_once("Too many UDP tunnel types, please increase UDP_MAX_TUNNEL_TYPES\n");
111 /* Ensure static call will never be enabled */
112 udp_tunnel_gro_type_nr = UDP_MAX_TUNNEL_TYPES + 1;
113 } else {
114 cur = &udp_tunnel_gro_types[udp_tunnel_gro_type_nr++];
115 refcount_set(&cur->count, 1);
116 cur->gro_receive = up->gro_receive;
117 }
118 } else {
119 /*
120 * The stack cleanups only successfully added tunnel, the
121 * lookup on removal should never fail.
122 */
123 if (WARN_ON_ONCE(!cur))
124 goto out;
125
126 if (!refcount_dec_and_test(&cur->count))
127 goto out;
128
129 /* Avoid gaps, so that the enable tunnel has always id 0 */
130 *cur = udp_tunnel_gro_types[--udp_tunnel_gro_type_nr];
131 }
132
133 if (udp_tunnel_gro_type_nr == 1) {
134 static_call_update(udp_tunnel_gro_rcv,
135 udp_tunnel_gro_types[0].gro_receive);
136 static_branch_enable(&udp_tunnel_static_call);
137 } else if (old_gro_type_nr == 1) {
138 static_branch_disable(&udp_tunnel_static_call);
139 static_call_update(udp_tunnel_gro_rcv, dummy_gro_rcv);
140 }
141
142out:
143 mutex_unlock(&udp_tunnel_gro_type_lock);
144}
145EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_rcv);
146
147static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
148 struct list_head *head,
149 struct sk_buff *skb)
150{
151 if (static_branch_likely(&udp_tunnel_static_call)) {
152 if (unlikely(gro_recursion_inc_test(skb))) {
153 NAPI_GRO_CB(skb)->flush |= 1;
154 return NULL;
155 }
156 return static_call(udp_tunnel_gro_rcv)(sk, head, skb);
157 }
158 return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
159}
160
161#else
162
163static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
164 struct list_head *head,
165 struct sk_buff *skb)
166{
167 return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
168}
169
170#endif
171
172static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
173 netdev_features_t features,
174 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
175 netdev_features_t features),
176 __be16 new_protocol, bool is_ipv6)
177{
178 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
179 bool remcsum, need_csum, offload_csum, gso_partial;
180 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
181 struct udphdr *uh = udp_hdr(skb);
182 u16 mac_offset = skb->mac_header;
183 __be16 protocol = skb->protocol;
184 u16 mac_len = skb->mac_len;
185 int udp_offset, outer_hlen;
186 __wsum partial;
187 bool need_ipsec;
188
189 if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
190 goto out;
191
192 /* Adjust partial header checksum to negate old length.
193 * We cannot rely on the value contained in uh->len as it is
194 * possible that the actual value exceeds the boundaries of the
195 * 16 bit length field due to the header being added outside of an
196 * IP or IPv6 frame that was already limited to 64K - 1.
197 */
198 if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)
199 partial = (__force __wsum)uh->len;
200 else
201 partial = (__force __wsum)htonl(skb->len);
202 partial = csum_sub(csum: csum_unfold(n: uh->check), addend: partial);
203
204 /* setup inner skb. */
205 skb->encapsulation = 0;
206 SKB_GSO_CB(skb)->encap_level = 0;
207 __skb_pull(skb, len: tnl_hlen);
208 skb_reset_mac_header(skb);
209 skb_set_network_header(skb, offset: skb_inner_network_offset(skb));
210 skb_set_transport_header(skb, offset: skb_inner_transport_offset(skb));
211 skb->mac_len = skb_inner_network_offset(skb);
212 skb->protocol = new_protocol;
213
214 need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
215 skb->encap_hdr_csum = need_csum;
216
217 remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);
218 skb->remcsum_offload = remcsum;
219
220 need_ipsec = (skb_dst(skb) && dst_xfrm(dst: skb_dst(skb))) || skb_sec_path(skb);
221 /* Try to offload checksum if possible */
222 offload_csum = !!(need_csum &&
223 !need_ipsec &&
224 (skb->dev->features &
225 (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) :
226 (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM))));
227
228 features &= skb->dev->hw_enc_features;
229 if (need_csum)
230 features &= ~NETIF_F_SCTP_CRC;
231
232 /* The only checksum offload we care about from here on out is the
233 * outer one so strip the existing checksum feature flags and
234 * instead set the flag based on our outer checksum offload value.
235 */
236 if (remcsum) {
237 features &= ~NETIF_F_CSUM_MASK;
238 if (!need_csum || offload_csum)
239 features |= NETIF_F_HW_CSUM;
240 }
241
242 /* segment inner packet. */
243 segs = gso_inner_segment(skb, features);
244 if (IS_ERR_OR_NULL(ptr: segs)) {
245 skb_gso_error_unwind(skb, protocol, pulled_hlen: tnl_hlen, mac_offset,
246 mac_len);
247 goto out;
248 }
249
250 gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
251
252 outer_hlen = skb_tnl_header_len(inner_skb: skb);
253 udp_offset = outer_hlen - tnl_hlen;
254 skb = segs;
255 do {
256 unsigned int len;
257
258 if (remcsum)
259 skb->ip_summed = CHECKSUM_NONE;
260
261 /* Set up inner headers if we are offloading inner checksum */
262 if (skb->ip_summed == CHECKSUM_PARTIAL) {
263 skb_reset_inner_headers(skb);
264 skb->encapsulation = 1;
265 }
266
267 skb->mac_len = mac_len;
268 skb->protocol = protocol;
269
270 __skb_push(skb, len: outer_hlen);
271 skb_reset_mac_header(skb);
272 skb_set_network_header(skb, offset: mac_len);
273 skb_set_transport_header(skb, offset: udp_offset);
274 len = skb->len - udp_offset;
275 uh = udp_hdr(skb);
276
277 /* If we are only performing partial GSO the inner header
278 * will be using a length value equal to only one MSS sized
279 * segment instead of the entire frame.
280 */
281 if (gso_partial && skb_is_gso(skb)) {
282 uh->len = htons(skb_shinfo(skb)->gso_size +
283 SKB_GSO_CB(skb)->data_offset +
284 skb->head - (unsigned char *)uh);
285 } else {
286 uh->len = htons(len);
287 }
288
289 if (!need_csum)
290 continue;
291
292 uh->check = ~csum_fold(sum: csum_add(csum: partial,
293 addend: (__force __wsum)htonl(len)));
294
295 if (skb->encapsulation || !offload_csum) {
296 uh->check = gso_make_checksum(skb, res: ~uh->check);
297 if (uh->check == 0)
298 uh->check = CSUM_MANGLED_0;
299 } else {
300 skb->ip_summed = CHECKSUM_PARTIAL;
301 skb->csum_start = skb_transport_header(skb) - skb->head;
302 skb->csum_offset = offsetof(struct udphdr, check);
303 }
304 } while ((skb = skb->next));
305out:
306 return segs;
307}
308
309struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
310 netdev_features_t features,
311 bool is_ipv6)
312{
313 const struct net_offload __rcu **offloads;
314 __be16 protocol = skb->protocol;
315 const struct net_offload *ops;
316 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
317 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
318 netdev_features_t features);
319
320 rcu_read_lock();
321
322 switch (skb->inner_protocol_type) {
323 case ENCAP_TYPE_ETHER:
324 protocol = skb->inner_protocol;
325 gso_inner_segment = skb_mac_gso_segment;
326 break;
327 case ENCAP_TYPE_IPPROTO:
328 offloads = is_ipv6 ? inet6_offloads : inet_offloads;
329 ops = rcu_dereference(offloads[skb->inner_ipproto]);
330 if (!ops || !ops->callbacks.gso_segment)
331 goto out_unlock;
332 gso_inner_segment = ops->callbacks.gso_segment;
333 break;
334 default:
335 goto out_unlock;
336 }
337
338 segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,
339 new_protocol: protocol, is_ipv6);
340
341out_unlock:
342 rcu_read_unlock();
343
344 return segs;
345}
346EXPORT_SYMBOL(skb_udp_tunnel_segment);
347
348static void __udpv4_gso_segment_csum(struct sk_buff *seg,
349 __be32 *oldip, __be32 *newip,
350 __be16 *oldport, __be16 *newport)
351{
352 struct udphdr *uh;
353 struct iphdr *iph;
354
355 if (*oldip == *newip && *oldport == *newport)
356 return;
357
358 uh = udp_hdr(skb: seg);
359 iph = ip_hdr(skb: seg);
360
361 if (uh->check) {
362 inet_proto_csum_replace4(sum: &uh->check, skb: seg, from: *oldip, to: *newip,
363 pseudohdr: true);
364 inet_proto_csum_replace2(sum: &uh->check, skb: seg, from: *oldport, to: *newport,
365 pseudohdr: false);
366 if (!uh->check)
367 uh->check = CSUM_MANGLED_0;
368 }
369 *oldport = *newport;
370
371 csum_replace4(sum: &iph->check, from: *oldip, to: *newip);
372 *oldip = *newip;
373}
374
375static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs)
376{
377 struct sk_buff *seg;
378 struct udphdr *uh, *uh2;
379 struct iphdr *iph, *iph2;
380
381 seg = segs;
382 uh = udp_hdr(skb: seg);
383 iph = ip_hdr(skb: seg);
384
385 if ((udp_hdr(skb: seg)->dest == udp_hdr(skb: seg->next)->dest) &&
386 (udp_hdr(skb: seg)->source == udp_hdr(skb: seg->next)->source) &&
387 (ip_hdr(skb: seg)->daddr == ip_hdr(skb: seg->next)->daddr) &&
388 (ip_hdr(skb: seg)->saddr == ip_hdr(skb: seg->next)->saddr))
389 return segs;
390
391 while ((seg = seg->next)) {
392 uh2 = udp_hdr(skb: seg);
393 iph2 = ip_hdr(skb: seg);
394
395 __udpv4_gso_segment_csum(seg,
396 oldip: &iph2->saddr, newip: &iph->saddr,
397 oldport: &uh2->source, newport: &uh->source);
398 __udpv4_gso_segment_csum(seg,
399 oldip: &iph2->daddr, newip: &iph->daddr,
400 oldport: &uh2->dest, newport: &uh->dest);
401 }
402
403 return segs;
404}
405
406static void __udpv6_gso_segment_csum(struct sk_buff *seg,
407 struct in6_addr *oldip,
408 const struct in6_addr *newip,
409 __be16 *oldport, __be16 newport)
410{
411 struct udphdr *uh = udp_hdr(skb: seg);
412
413 if (ipv6_addr_equal(a1: oldip, a2: newip) && *oldport == newport)
414 return;
415
416 if (uh->check) {
417 inet_proto_csum_replace16(sum: &uh->check, skb: seg, from: oldip->s6_addr32,
418 to: newip->s6_addr32, pseudohdr: true);
419
420 inet_proto_csum_replace2(sum: &uh->check, skb: seg, from: *oldport, to: newport,
421 pseudohdr: false);
422 if (!uh->check)
423 uh->check = CSUM_MANGLED_0;
424 }
425
426 *oldip = *newip;
427 *oldport = newport;
428}
429
430static struct sk_buff *__udpv6_gso_segment_list_csum(struct sk_buff *segs)
431{
432 const struct ipv6hdr *iph;
433 const struct udphdr *uh;
434 struct ipv6hdr *iph2;
435 struct sk_buff *seg;
436 struct udphdr *uh2;
437
438 seg = segs;
439 uh = udp_hdr(skb: seg);
440 iph = ipv6_hdr(skb: seg);
441 uh2 = udp_hdr(skb: seg->next);
442 iph2 = ipv6_hdr(skb: seg->next);
443
444 if (!(*(const u32 *)&uh->source ^ *(const u32 *)&uh2->source) &&
445 ipv6_addr_equal(a1: &iph->saddr, a2: &iph2->saddr) &&
446 ipv6_addr_equal(a1: &iph->daddr, a2: &iph2->daddr))
447 return segs;
448
449 while ((seg = seg->next)) {
450 uh2 = udp_hdr(skb: seg);
451 iph2 = ipv6_hdr(skb: seg);
452
453 __udpv6_gso_segment_csum(seg, oldip: &iph2->saddr, newip: &iph->saddr,
454 oldport: &uh2->source, newport: uh->source);
455 __udpv6_gso_segment_csum(seg, oldip: &iph2->daddr, newip: &iph->daddr,
456 oldport: &uh2->dest, newport: uh->dest);
457 }
458
459 return segs;
460}
461
462static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,
463 netdev_features_t features,
464 bool is_ipv6)
465{
466 unsigned int mss = skb_shinfo(skb)->gso_size;
467
468 skb = skb_segment_list(skb, features, offset: skb_mac_header_len(skb));
469 if (IS_ERR(ptr: skb))
470 return skb;
471
472 udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);
473
474 if (is_ipv6)
475 return __udpv6_gso_segment_list_csum(segs: skb);
476 else
477 return __udpv4_gso_segment_list_csum(segs: skb);
478}
479
480struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
481 netdev_features_t features, bool is_ipv6)
482{
483 struct sock *sk = gso_skb->sk;
484 unsigned int sum_truesize = 0;
485 struct sk_buff *segs, *seg;
486 struct udphdr *uh;
487 unsigned int mss;
488 bool copy_dtor;
489 __sum16 check;
490 __be16 newlen;
491 int ret = 0;
492
493 mss = skb_shinfo(gso_skb)->gso_size;
494 if (gso_skb->len <= sizeof(*uh) + mss)
495 return ERR_PTR(error: -EINVAL);
496
497 if (unlikely(skb_checksum_start(gso_skb) !=
498 skb_transport_header(gso_skb) &&
499 !(skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST)))
500 return ERR_PTR(error: -EINVAL);
501
502 /* We don't know if egress device can segment and checksum the packet
503 * when IPv6 extension headers are present. Fall back to software GSO.
504 */
505 if (gso_skb->ip_summed != CHECKSUM_PARTIAL)
506 features &= ~(NETIF_F_GSO_UDP_L4 | NETIF_F_CSUM_MASK);
507
508 if (skb_gso_ok(skb: gso_skb, features: features | NETIF_F_GSO_ROBUST)) {
509 /* Packet is from an untrusted source, reset gso_segs. */
510 skb_shinfo(gso_skb)->gso_segs = DIV_ROUND_UP(gso_skb->len - sizeof(*uh),
511 mss);
512 return NULL;
513 }
514
515 if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST) {
516 /* Detect modified geometry and pass those to skb_segment. */
517 if (skb_pagelen(skb: gso_skb) - sizeof(*uh) == skb_shinfo(gso_skb)->gso_size)
518 return __udp_gso_segment_list(skb: gso_skb, features, is_ipv6);
519
520 ret = __skb_linearize(skb: gso_skb);
521 if (ret)
522 return ERR_PTR(error: ret);
523
524 /* Setup csum, as fraglist skips this in udp4_gro_receive. */
525 gso_skb->csum_start = skb_transport_header(skb: gso_skb) - gso_skb->head;
526 gso_skb->csum_offset = offsetof(struct udphdr, check);
527 gso_skb->ip_summed = CHECKSUM_PARTIAL;
528
529 uh = udp_hdr(skb: gso_skb);
530 if (is_ipv6)
531 uh->check = ~udp_v6_check(len: gso_skb->len,
532 saddr: &ipv6_hdr(skb: gso_skb)->saddr,
533 daddr: &ipv6_hdr(skb: gso_skb)->daddr, base: 0);
534 else
535 uh->check = ~udp_v4_check(len: gso_skb->len,
536 saddr: ip_hdr(skb: gso_skb)->saddr,
537 daddr: ip_hdr(skb: gso_skb)->daddr, base: 0);
538 }
539
540 skb_pull(skb: gso_skb, len: sizeof(*uh));
541
542 /* clear destructor to avoid skb_segment assigning it to tail */
543 copy_dtor = gso_skb->destructor == sock_wfree;
544 if (copy_dtor) {
545 gso_skb->destructor = NULL;
546 gso_skb->sk = NULL;
547 }
548
549 segs = skb_segment(skb: gso_skb, features);
550 if (IS_ERR_OR_NULL(ptr: segs)) {
551 if (copy_dtor) {
552 gso_skb->destructor = sock_wfree;
553 gso_skb->sk = sk;
554 }
555 return segs;
556 }
557
558 /* GSO partial and frag_list segmentation only requires splitting
559 * the frame into an MSS multiple and possibly a remainder, both
560 * cases return a GSO skb. So update the mss now.
561 */
562 if (skb_is_gso(skb: segs))
563 mss *= skb_shinfo(segs)->gso_segs;
564
565 seg = segs;
566 uh = udp_hdr(skb: seg);
567
568 /* preserve TX timestamp flags and TS key for first segment */
569 skb_shinfo(seg)->tskey = skb_shinfo(gso_skb)->tskey;
570 skb_shinfo(seg)->tx_flags |=
571 (skb_shinfo(gso_skb)->tx_flags & SKBTX_ANY_TSTAMP);
572
573 /* compute checksum adjustment based on old length versus new */
574 newlen = htons(sizeof(*uh) + mss);
575 check = csum16_add(csum: csum16_sub(csum: uh->check, addend: uh->len), addend: newlen);
576
577 for (;;) {
578 if (copy_dtor) {
579 seg->destructor = sock_wfree;
580 seg->sk = sk;
581 sum_truesize += seg->truesize;
582 }
583
584 if (!seg->next)
585 break;
586
587 uh->len = newlen;
588 uh->check = check;
589
590 if (seg->ip_summed == CHECKSUM_PARTIAL)
591 gso_reset_checksum(skb: seg, res: ~check);
592 else
593 uh->check = gso_make_checksum(skb: seg, res: ~check) ? :
594 CSUM_MANGLED_0;
595
596 seg = seg->next;
597 uh = udp_hdr(skb: seg);
598 }
599
600 /* last packet can be partial gso_size, account for that in checksum */
601 newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
602 seg->data_len);
603 check = csum16_add(csum: csum16_sub(csum: uh->check, addend: uh->len), addend: newlen);
604
605 uh->len = newlen;
606 uh->check = check;
607
608 if (seg->ip_summed == CHECKSUM_PARTIAL)
609 gso_reset_checksum(skb: seg, res: ~check);
610 else
611 uh->check = gso_make_checksum(skb: seg, res: ~check) ? : CSUM_MANGLED_0;
612
613 /* On the TX path, CHECKSUM_NONE and CHECKSUM_UNNECESSARY have the same
614 * meaning. However, check for bad offloads in the GSO stack expects the
615 * latter, if the checksum was calculated in software. To vouch for the
616 * segment skbs we actually need to set it on the gso_skb.
617 */
618 if (gso_skb->ip_summed == CHECKSUM_NONE)
619 gso_skb->ip_summed = CHECKSUM_UNNECESSARY;
620
621 /* update refcount for the packet */
622 if (copy_dtor) {
623 int delta = sum_truesize - gso_skb->truesize;
624
625 /* In some pathological cases, delta can be negative.
626 * We need to either use refcount_add() or refcount_sub_and_test()
627 */
628 if (likely(delta >= 0))
629 refcount_add(i: delta, r: &sk->sk_wmem_alloc);
630 else
631 WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc));
632 }
633 return segs;
634}
635EXPORT_SYMBOL_GPL(__udp_gso_segment);
636
637static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
638 netdev_features_t features)
639{
640 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
641 unsigned int mss;
642 __wsum csum;
643 struct udphdr *uh;
644 struct iphdr *iph;
645
646 if (skb->encapsulation &&
647 (skb_shinfo(skb)->gso_type &
648 (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) {
649 segs = skb_udp_tunnel_segment(skb, features, false);
650 goto out;
651 }
652
653 if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
654 goto out;
655
656 if (!pskb_may_pull(skb, len: sizeof(struct udphdr)))
657 goto out;
658
659 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
660 return __udp_gso_segment(skb, features, false);
661
662 mss = skb_shinfo(skb)->gso_size;
663 if (unlikely(skb->len <= mss))
664 goto out;
665
666 /* Do software UFO. Complete and fill in the UDP checksum as
667 * HW cannot do checksum of UDP packets sent as multiple
668 * IP fragments.
669 */
670
671 uh = udp_hdr(skb);
672 iph = ip_hdr(skb);
673
674 uh->check = 0;
675 csum = skb_checksum(skb, offset: 0, len: skb->len, csum: 0);
676 uh->check = udp_v4_check(len: skb->len, saddr: iph->saddr, daddr: iph->daddr, base: csum);
677 if (uh->check == 0)
678 uh->check = CSUM_MANGLED_0;
679
680 skb->ip_summed = CHECKSUM_UNNECESSARY;
681
682 /* If there is no outer header we can fake a checksum offload
683 * due to the fact that we have already done the checksum in
684 * software prior to segmenting the frame.
685 */
686 if (!skb->encap_hdr_csum)
687 features |= NETIF_F_HW_CSUM;
688
689 /* Fragment the skb. IP headers of the fragments are updated in
690 * inet_gso_segment()
691 */
692 segs = skb_segment(skb, features);
693out:
694 return segs;
695}
696
697
698#define UDP_GRO_CNT_MAX 64
699static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
700 struct sk_buff *skb)
701{
702 struct udphdr *uh = udp_gro_udphdr(skb);
703 struct sk_buff *pp = NULL;
704 struct udphdr *uh2;
705 struct sk_buff *p;
706 unsigned int ulen;
707 int ret = 0;
708 int flush;
709
710 /* requires non zero csum, for symmetry with GSO */
711 if (!uh->check) {
712 NAPI_GRO_CB(skb)->flush = 1;
713 return NULL;
714 }
715
716 /* Do not deal with padded or malicious packets, sorry ! */
717 ulen = ntohs(uh->len);
718 if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {
719 NAPI_GRO_CB(skb)->flush = 1;
720 return NULL;
721 }
722 /* pull encapsulating udp header */
723 skb_gro_pull(skb, len: sizeof(struct udphdr));
724
725 list_for_each_entry(p, head, list) {
726 if (!NAPI_GRO_CB(p)->same_flow)
727 continue;
728
729 uh2 = udp_hdr(skb: p);
730
731 /* Match ports only, as csum is always non zero */
732 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) {
733 NAPI_GRO_CB(p)->same_flow = 0;
734 continue;
735 }
736
737 if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {
738 NAPI_GRO_CB(skb)->flush = 1;
739 return p;
740 }
741
742 flush = gro_receive_network_flush(th: uh, th2: uh2, p);
743
744 /* Terminate the flow on len mismatch or if it grow "too much".
745 * Under small packet flood GRO count could elsewhere grow a lot
746 * leading to excessive truesize values.
747 * On len mismatch merge the first packet shorter than gso_size,
748 * otherwise complete the GRO packet.
749 */
750 if (ulen > ntohs(uh2->len) || flush) {
751 pp = p;
752 } else {
753 if (NAPI_GRO_CB(skb)->is_flist) {
754 if (!pskb_may_pull(skb, len: skb_gro_offset(skb))) {
755 NAPI_GRO_CB(skb)->flush = 1;
756 return NULL;
757 }
758 if ((skb->ip_summed != p->ip_summed) ||
759 (skb->csum_level != p->csum_level)) {
760 NAPI_GRO_CB(skb)->flush = 1;
761 return NULL;
762 }
763 skb_set_network_header(skb, offset: skb_gro_receive_network_offset(skb));
764 ret = skb_gro_receive_list(p, skb);
765 } else {
766 skb_gro_postpull_rcsum(skb, start: uh,
767 len: sizeof(struct udphdr));
768
769 ret = skb_gro_receive(p, skb);
770 }
771 }
772
773 if (ret || ulen != ntohs(uh2->len) ||
774 NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
775 pp = p;
776
777 return pp;
778 }
779
780 /* mismatch, but we never need to flush */
781 return NULL;
782}
783
784struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
785 struct udphdr *uh, struct sock *sk)
786{
787 struct sk_buff *pp = NULL;
788 struct sk_buff *p;
789 struct udphdr *uh2;
790 unsigned int off = skb_gro_offset(skb);
791 int flush = 1;
792
793 /* We can do L4 aggregation only if the packet can't land in a tunnel
794 * otherwise we could corrupt the inner stream. Detecting such packets
795 * cannot be foolproof and the aggregation might still happen in some
796 * cases. Such packets should be caught in udp_unexpected_gso later.
797 */
798 NAPI_GRO_CB(skb)->is_flist = 0;
799 if (!sk || !udp_sk(sk)->gro_receive) {
800 /* If the packet was locally encapsulated in a UDP tunnel that
801 * wasn't detected above, do not GRO.
802 */
803 if (skb->encapsulation)
804 goto out;
805
806 if (skb->dev->features & NETIF_F_GRO_FRAGLIST)
807 NAPI_GRO_CB(skb)->is_flist = sk ? !udp_test_bit(GRO_ENABLED, sk) : 1;
808
809 if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||
810 (sk && udp_test_bit(GRO_ENABLED, sk)) || NAPI_GRO_CB(skb)->is_flist)
811 return call_gro_receive(cb: udp_gro_receive_segment, head, skb);
812
813 /* no GRO, be sure flush the current packet */
814 goto out;
815 }
816
817 if (NAPI_GRO_CB(skb)->encap_mark ||
818 (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&
819 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
820 !NAPI_GRO_CB(skb)->csum_valid))
821 goto out;
822
823 /* mark that this skb passed once through the tunnel gro layer */
824 NAPI_GRO_CB(skb)->encap_mark = 1;
825
826 flush = 0;
827
828 list_for_each_entry(p, head, list) {
829 if (!NAPI_GRO_CB(p)->same_flow)
830 continue;
831
832 uh2 = (struct udphdr *)(p->data + off);
833
834 /* Match ports and either checksums are either both zero
835 * or nonzero.
836 */
837 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
838 (!uh->check ^ !uh2->check)) {
839 NAPI_GRO_CB(p)->same_flow = 0;
840 continue;
841 }
842 }
843
844 skb_gro_pull(skb, len: sizeof(struct udphdr)); /* pull encapsulating udp header */
845 skb_gro_postpull_rcsum(skb, start: uh, len: sizeof(struct udphdr));
846 pp = udp_tunnel_gro_rcv(sk, head, skb);
847
848out:
849 skb_gro_flush_final(skb, pp, flush);
850 return pp;
851}
852EXPORT_SYMBOL(udp_gro_receive);
853
854static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
855 __be16 dport)
856{
857 const struct iphdr *iph = skb_gro_network_header(skb);
858 struct net *net = dev_net_rcu(dev: skb->dev);
859 struct sock *sk;
860 int iif, sdif;
861
862 sk = udp_tunnel_sk(net, is_ipv6: false);
863 if (sk && dport == htons(sk->sk_num))
864 return sk;
865
866 inet_get_iif_sdif(skb, iif: &iif, sdif: &sdif);
867
868 return __udp4_lib_lookup(net, saddr: iph->saddr, sport,
869 daddr: iph->daddr, dport, dif: iif,
870 sdif, tbl: net->ipv4.udp_table, NULL);
871}
872
873INDIRECT_CALLABLE_SCOPE
874struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
875{
876 struct udphdr *uh = udp_gro_udphdr(skb);
877 struct sock *sk = NULL;
878 struct sk_buff *pp;
879
880 if (unlikely(!uh))
881 goto flush;
882
883 /* Don't bother verifying checksum if we're going to flush anyway. */
884 if (NAPI_GRO_CB(skb)->flush)
885 goto skip;
886
887 if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
888 inet_gro_compute_pseudo))
889 goto flush;
890 else if (uh->check)
891 skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
892 inet_gro_compute_pseudo);
893skip:
894 if (static_branch_unlikely(&udp_encap_needed_key))
895 sk = udp4_gro_lookup_skb(skb, sport: uh->source, dport: uh->dest);
896
897 pp = udp_gro_receive(head, skb, uh, sk);
898 return pp;
899
900flush:
901 NAPI_GRO_CB(skb)->flush = 1;
902 return NULL;
903}
904
905static int udp_gro_complete_segment(struct sk_buff *skb)
906{
907 struct udphdr *uh = udp_hdr(skb);
908
909 skb->csum_start = (unsigned char *)uh - skb->head;
910 skb->csum_offset = offsetof(struct udphdr, check);
911 skb->ip_summed = CHECKSUM_PARTIAL;
912
913 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
914 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
915
916 if (skb->encapsulation)
917 skb->inner_transport_header = skb->transport_header;
918
919 return 0;
920}
921
922int udp_gro_complete(struct sk_buff *skb, int nhoff,
923 udp_lookup_t lookup)
924{
925 __be16 newlen = htons(skb->len - nhoff);
926 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
927 struct sock *sk;
928 int err;
929
930 uh->len = newlen;
931
932 sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
933 udp4_lib_lookup_skb, skb, uh->source, uh->dest);
934 if (sk && udp_sk(sk)->gro_complete) {
935 skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
936 : SKB_GSO_UDP_TUNNEL;
937
938 /* clear the encap mark, so that inner frag_list gro_complete
939 * can take place
940 */
941 NAPI_GRO_CB(skb)->encap_mark = 0;
942
943 /* Set encapsulation before calling into inner gro_complete()
944 * functions to make them set up the inner offsets.
945 */
946 skb->encapsulation = 1;
947 err = udp_sk(sk)->gro_complete(sk, skb,
948 nhoff + sizeof(struct udphdr));
949 } else {
950 err = udp_gro_complete_segment(skb);
951 }
952
953 if (skb->remcsum_offload)
954 skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;
955
956 return err;
957}
958EXPORT_SYMBOL(udp_gro_complete);
959
960INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
961{
962 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
963 const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
964 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
965
966 /* do fraglist only if there is no outer UDP encap (or we already processed it) */
967 if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
968 uh->len = htons(skb->len - nhoff);
969
970 skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
971 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
972
973 __skb_incr_checksum_unnecessary(skb);
974
975 return 0;
976 }
977
978 if (uh->check)
979 uh->check = ~udp_v4_check(len: skb->len - nhoff, saddr: iph->saddr,
980 daddr: iph->daddr, base: 0);
981
982 return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
983}
984
985int __init udpv4_offload_init(void)
986{
987 net_hotdata.udpv4_offload = (struct net_offload) {
988 .callbacks = {
989 .gso_segment = udp4_ufo_fragment,
990 .gro_receive = udp4_gro_receive,
991 .gro_complete = udp4_gro_complete,
992 },
993 };
994
995 return inet_add_offload(prot: &net_hotdata.udpv4_offload, IPPROTO_UDP);
996}
997