1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * IPV6 GSO/GRO offload support
4 * Linux INET6 implementation
5 *
6 * TCPv6 GSO/GRO support
7 */
8#include <linux/indirect_call_wrapper.h>
9#include <linux/skbuff.h>
10#include <net/inet6_hashtables.h>
11#include <net/gro.h>
12#include <net/protocol.h>
13#include <net/tcp.h>
14#include <net/ip6_checksum.h>
15#include "ip6_offload.h"
16
17static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
18 struct tcphdr *th)
19{
20#if IS_ENABLED(CONFIG_IPV6)
21 const struct ipv6hdr *hdr;
22 struct sk_buff *p;
23 struct sock *sk;
24 struct net *net;
25 int iif, sdif;
26
27 if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST)))
28 return;
29
30 p = tcp_gro_lookup(head, th);
31 if (p) {
32 NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
33 return;
34 }
35
36 inet6_get_iif_sdif(skb, iif: &iif, sdif: &sdif);
37 hdr = skb_gro_network_header(skb);
38 net = dev_net_rcu(dev: skb->dev);
39 sk = __inet6_lookup_established(net, saddr: &hdr->saddr, sport: th->source,
40 daddr: &hdr->daddr, ntohs(th->dest),
41 dif: iif, sdif);
42 NAPI_GRO_CB(skb)->is_flist = !sk;
43 if (sk)
44 sock_gen_put(sk);
45#endif /* IS_ENABLED(CONFIG_IPV6) */
46}
47
48INDIRECT_CALLABLE_SCOPE
49struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)
50{
51 struct tcphdr *th;
52
53 /* Don't bother verifying checksum if we're going to flush anyway. */
54 if (!NAPI_GRO_CB(skb)->flush &&
55 skb_gro_checksum_validate(skb, IPPROTO_TCP,
56 ip6_gro_compute_pseudo))
57 goto flush;
58
59 th = tcp_gro_pull_header(skb);
60 if (!th)
61 goto flush;
62
63 tcp6_check_fraglist_gro(head, skb, th);
64
65 return tcp_gro_receive(head, skb, th);
66
67flush:
68 NAPI_GRO_CB(skb)->flush = 1;
69 return NULL;
70}
71
72INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
73{
74 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
75 const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset);
76 struct tcphdr *th = tcp_hdr(skb);
77
78 if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
79 skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
80 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
81
82 __skb_incr_checksum_unnecessary(skb);
83
84 return 0;
85 }
86
87 th->check = ~tcp_v6_check(len: skb->len - thoff, saddr: &iph->saddr,
88 daddr: &iph->daddr, base: 0);
89 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
90
91 tcp_gro_complete(skb);
92 return 0;
93}
94
95static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
96 struct in6_addr *oldip,
97 const struct in6_addr *newip,
98 __be16 *oldport, __be16 newport)
99{
100 struct tcphdr *th = tcp_hdr(skb: seg);
101
102 if (!ipv6_addr_equal(a1: oldip, a2: newip)) {
103 inet_proto_csum_replace16(sum: &th->check, skb: seg,
104 from: oldip->s6_addr32,
105 to: newip->s6_addr32,
106 pseudohdr: true);
107 *oldip = *newip;
108 }
109
110 if (*oldport == newport)
111 return;
112
113 inet_proto_csum_replace2(sum: &th->check, skb: seg, from: *oldport, to: newport, pseudohdr: false);
114 *oldport = newport;
115}
116
117static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
118{
119 const struct tcphdr *th;
120 const struct ipv6hdr *iph;
121 struct sk_buff *seg;
122 struct tcphdr *th2;
123 struct ipv6hdr *iph2;
124
125 seg = segs;
126 th = tcp_hdr(skb: seg);
127 iph = ipv6_hdr(skb: seg);
128 th2 = tcp_hdr(skb: seg->next);
129 iph2 = ipv6_hdr(skb: seg->next);
130
131 if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
132 ipv6_addr_equal(a1: &iph->saddr, a2: &iph2->saddr) &&
133 ipv6_addr_equal(a1: &iph->daddr, a2: &iph2->daddr))
134 return segs;
135
136 while ((seg = seg->next)) {
137 th2 = tcp_hdr(skb: seg);
138 iph2 = ipv6_hdr(skb: seg);
139
140 __tcpv6_gso_segment_csum(seg, oldip: &iph2->saddr, newip: &iph->saddr,
141 oldport: &th2->source, newport: th->source);
142 __tcpv6_gso_segment_csum(seg, oldip: &iph2->daddr, newip: &iph->daddr,
143 oldport: &th2->dest, newport: th->dest);
144 }
145
146 return segs;
147}
148
149static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
150 netdev_features_t features)
151{
152 skb = skb_segment_list(skb, features, offset: skb_mac_header_len(skb));
153 if (IS_ERR(ptr: skb))
154 return skb;
155
156 return __tcpv6_gso_segment_list_csum(segs: skb);
157}
158
159static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
160 netdev_features_t features)
161{
162 struct tcphdr *th;
163
164 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
165 return ERR_PTR(error: -EINVAL);
166
167 if (!pskb_may_pull(skb, len: sizeof(*th)))
168 return ERR_PTR(error: -EINVAL);
169
170 if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) {
171 struct tcphdr *th = tcp_hdr(skb);
172
173 if (skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size)
174 return __tcp6_gso_segment_list(skb, features);
175
176 skb->ip_summed = CHECKSUM_NONE;
177 }
178
179 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
180 const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
181 struct tcphdr *th = tcp_hdr(skb);
182
183 /* Set up pseudo header, usually expect stack to have done
184 * this.
185 */
186
187 th->check = 0;
188 skb->ip_summed = CHECKSUM_PARTIAL;
189 __tcp_v6_send_check(skb, saddr: &ipv6h->saddr, daddr: &ipv6h->daddr);
190 }
191
192 return tcp_gso_segment(skb, features);
193}
194
195int __init tcpv6_offload_init(void)
196{
197 net_hotdata.tcpv6_offload = (struct net_offload) {
198 .callbacks = {
199 .gso_segment = tcp6_gso_segment,
200 .gro_receive = tcp6_gro_receive,
201 .gro_complete = tcp6_gro_complete,
202 },
203 };
204 return inet6_add_offload(prot: &net_hotdata.tcpv6_offload, IPPROTO_TCP);
205}
206