1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the TCP module.
8 *
9 * Version: @(#)tcp.h 1.0.5 05/23/93
10 *
11 * Authors: Ross Biro
12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13 */
14#ifndef _TCP_H
15#define _TCP_H
16
17#define FASTRETRANS_DEBUG 1
18
19#include <linux/list.h>
20#include <linux/tcp.h>
21#include <linux/bug.h>
22#include <linux/slab.h>
23#include <linux/cache.h>
24#include <linux/percpu.h>
25#include <linux/skbuff.h>
26#include <linux/kref.h>
27#include <linux/ktime.h>
28#include <linux/indirect_call_wrapper.h>
29#include <linux/bits.h>
30
31#include <net/inet_connection_sock.h>
32#include <net/inet_timewait_sock.h>
33#include <net/inet_hashtables.h>
34#include <net/checksum.h>
35#include <net/request_sock.h>
36#include <net/sock_reuseport.h>
37#include <net/sock.h>
38#include <net/snmp.h>
39#include <net/ip.h>
40#include <net/tcp_states.h>
41#include <net/tcp_ao.h>
42#include <net/inet_ecn.h>
43#include <net/dst.h>
44#include <net/mptcp.h>
45#include <net/xfrm.h>
46
47#include <linux/seq_file.h>
48#include <linux/memcontrol.h>
49#include <linux/bpf-cgroup.h>
50#include <linux/siphash.h>
51
52extern struct inet_hashinfo tcp_hashinfo;
53
54DECLARE_PER_CPU(unsigned int, tcp_orphan_count);
55int tcp_orphan_count_sum(void);
56
57static inline void tcp_orphan_count_inc(void)
58{
59 this_cpu_inc(tcp_orphan_count);
60}
61
62static inline void tcp_orphan_count_dec(void)
63{
64 this_cpu_dec(tcp_orphan_count);
65}
66
67DECLARE_PER_CPU(u32, tcp_tw_isn);
68
69void tcp_time_wait(struct sock *sk, int state, int timeo);
70
71#define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER)
72#define MAX_TCP_OPTION_SPACE 40
73#define TCP_MIN_SND_MSS 48
74#define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
75
76/*
77 * Never offer a window over 32767 without using window scaling. Some
78 * poor stacks do signed 16bit maths!
79 */
80#define MAX_TCP_WINDOW 32767U
81
82/* Minimal accepted MSS. It is (60+60+8) - (20+20). */
83#define TCP_MIN_MSS 88U
84
85/* The initial MTU to use for probing */
86#define TCP_BASE_MSS 1024
87
88/* probing interval, default to 10 minutes as per RFC4821 */
89#define TCP_PROBE_INTERVAL 600
90
91/* Specify interval when tcp mtu probing will stop */
92#define TCP_PROBE_THRESHOLD 8
93
94/* After receiving this amount of duplicate ACKs fast retransmit starts. */
95#define TCP_FASTRETRANS_THRESH 3
96
97/* Maximal number of ACKs sent quickly to accelerate slow-start. */
98#define TCP_MAX_QUICKACKS 16U
99
100/* Maximal number of window scale according to RFC1323 */
101#define TCP_MAX_WSCALE 14U
102
103/* Default sending frequency of accurate ECN option per RTT */
104#define TCP_ACCECN_OPTION_BEACON 3
105
106/* urg_data states */
107#define TCP_URG_VALID 0x0100
108#define TCP_URG_NOTYET 0x0200
109#define TCP_URG_READ 0x0400
110
111#define TCP_RETR1 3 /*
112 * This is how many retries it does before it
113 * tries to figure out if the gateway is
114 * down. Minimal RFC value is 3; it corresponds
115 * to ~3sec-8min depending on RTO.
116 */
117
118#define TCP_RETR2 15 /*
119 * This should take at least
120 * 90 minutes to time out.
121 * RFC1122 says that the limit is 100 sec.
122 * 15 is ~13-30min depending on RTO.
123 */
124
125#define TCP_SYN_RETRIES 6 /* This is how many retries are done
126 * when active opening a connection.
127 * RFC1122 says the minimum retry MUST
128 * be at least 180secs. Nevertheless
129 * this value is corresponding to
130 * 63secs of retransmission with the
131 * current initial RTO.
132 */
133
134#define TCP_SYNACK_RETRIES 5 /* This is how may retries are done
135 * when passive opening a connection.
136 * This is corresponding to 31secs of
137 * retransmission with the current
138 * initial RTO.
139 */
140
141#define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
142 * state, about 60 seconds */
143#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
144 /* BSD style FIN_WAIT2 deadlock breaker.
145 * It used to be 3min, new value is 60sec,
146 * to combine FIN-WAIT-2 timeout with
147 * TIME-WAIT timer.
148 */
149#define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */
150
151#define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */
152static_assert((1 << ATO_BITS) > TCP_DELACK_MAX);
153
154#if HZ >= 100
155#define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */
156#define TCP_ATO_MIN ((unsigned)(HZ/25))
157#else
158#define TCP_DELACK_MIN 4U
159#define TCP_ATO_MIN 4U
160#endif
161#define TCP_RTO_MAX_SEC 120
162#define TCP_RTO_MAX ((unsigned)(TCP_RTO_MAX_SEC * HZ))
163#define TCP_RTO_MIN ((unsigned)(HZ / 5))
164#define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */
165
166#define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */
167
168#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */
169#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now
170 * used as a fallback RTO for the
171 * initial data transmission if no
172 * valid RTT sample has been acquired,
173 * most likely due to retrans in 3WHS.
174 */
175
176#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
177 * for local resources.
178 */
179#define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */
180#define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */
181#define TCP_KEEPALIVE_INTVL (75*HZ)
182
183#define MAX_TCP_KEEPIDLE 32767
184#define MAX_TCP_KEEPINTVL 32767
185#define MAX_TCP_KEEPCNT 127
186#define MAX_TCP_SYNCNT 127
187
188/* Ensure that TCP PAWS checks are relaxed after ~2147 seconds
189 * to avoid overflows. This assumes a clock smaller than 1 Mhz.
190 * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz.
191 */
192#define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC)
193
194#define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated
195 * after this time. It should be equal
196 * (or greater than) TCP_TIMEWAIT_LEN
197 * to provide reliability equal to one
198 * provided by timewait state.
199 */
200#define TCP_PAWS_WINDOW 1 /* Replay window for per-host
201 * timestamps. It must be less than
202 * minimal timewait lifetime.
203 */
204/*
205 * TCP option
206 */
207
208#define TCPOPT_NOP 1 /* Padding */
209#define TCPOPT_EOL 0 /* End of options */
210#define TCPOPT_MSS 2 /* Segment size negotiating */
211#define TCPOPT_WINDOW 3 /* Window scaling */
212#define TCPOPT_SACK_PERM 4 /* SACK Permitted */
213#define TCPOPT_SACK 5 /* SACK Block */
214#define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */
215#define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */
216#define TCPOPT_AO 29 /* Authentication Option (RFC5925) */
217#define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */
218#define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */
219#define TCPOPT_ACCECN0 172 /* 0xAC: Accurate ECN Order 0 */
220#define TCPOPT_ACCECN1 174 /* 0xAE: Accurate ECN Order 1 */
221#define TCPOPT_EXP 254 /* Experimental */
222/* Magic number to be after the option value for sharing TCP
223 * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
224 */
225#define TCPOPT_FASTOPEN_MAGIC 0xF989
226#define TCPOPT_SMC_MAGIC 0xE2D4C3D9
227
228/*
229 * TCP option lengths
230 */
231
232#define TCPOLEN_MSS 4
233#define TCPOLEN_WINDOW 3
234#define TCPOLEN_SACK_PERM 2
235#define TCPOLEN_TIMESTAMP 10
236#define TCPOLEN_MD5SIG 18
237#define TCPOLEN_FASTOPEN_BASE 2
238#define TCPOLEN_ACCECN_BASE 2
239#define TCPOLEN_EXP_FASTOPEN_BASE 4
240#define TCPOLEN_EXP_SMC_BASE 6
241
242/* But this is what stacks really send out. */
243#define TCPOLEN_TSTAMP_ALIGNED 12
244#define TCPOLEN_WSCALE_ALIGNED 4
245#define TCPOLEN_SACKPERM_ALIGNED 4
246#define TCPOLEN_SACK_BASE 2
247#define TCPOLEN_SACK_BASE_ALIGNED 4
248#define TCPOLEN_SACK_PERBLOCK 8
249#define TCPOLEN_MD5SIG_ALIGNED 20
250#define TCPOLEN_MSS_ALIGNED 4
251#define TCPOLEN_EXP_SMC_BASE_ALIGNED 8
252#define TCPOLEN_ACCECN_PERFIELD 3
253
254/* Maximum number of byte counters in AccECN option + size */
255#define TCP_ACCECN_NUMFIELDS 3
256#define TCP_ACCECN_MAXSIZE (TCPOLEN_ACCECN_BASE + \
257 TCPOLEN_ACCECN_PERFIELD * \
258 TCP_ACCECN_NUMFIELDS)
259#define TCP_ACCECN_SAFETY_SHIFT 1 /* SAFETY_FACTOR in accecn draft */
260
261/* Flags in tp->nonagle */
262#define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */
263#define TCP_NAGLE_CORK 2 /* Socket is corked */
264#define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */
265
266/* TCP thin-stream limits */
267#define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */
268
269/* TCP initial congestion window as per rfc6928 */
270#define TCP_INIT_CWND 10
271
272/* Bit Flags for sysctl_tcp_fastopen */
273#define TFO_CLIENT_ENABLE 1
274#define TFO_SERVER_ENABLE 2
275#define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */
276
277/* Accept SYN data w/o any cookie option */
278#define TFO_SERVER_COOKIE_NOT_REQD 0x200
279
280/* Force enable TFO on all listeners, i.e., not requiring the
281 * TCP_FASTOPEN socket option.
282 */
283#define TFO_SERVER_WO_SOCKOPT1 0x400
284
285
286/* sysctl variables for tcp */
287extern int sysctl_tcp_max_orphans;
288extern long sysctl_tcp_mem[3];
289
290#define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */
291#define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */
292#define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */
293
294DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
295
296extern struct percpu_counter tcp_sockets_allocated;
297extern unsigned long tcp_memory_pressure;
298
299/* optimized version of sk_under_memory_pressure() for TCP sockets */
300static inline bool tcp_under_memory_pressure(const struct sock *sk)
301{
302 if (mem_cgroup_sk_enabled(sk) &&
303 mem_cgroup_sk_under_memory_pressure(sk))
304 return true;
305
306 return READ_ONCE(tcp_memory_pressure);
307}
308/*
309 * The next routines deal with comparing 32 bit unsigned ints
310 * and worry about wraparound (automatic with unsigned arithmetic).
311 */
312
313static inline bool before(__u32 seq1, __u32 seq2)
314{
315 return (__s32)(seq1-seq2) < 0;
316}
317#define after(seq2, seq1) before(seq1, seq2)
318
319/* is s2<=s1<=s3 ? */
320static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
321{
322 return seq3 - seq2 >= seq1 - seq2;
323}
324
325static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
326{
327 sk_wmem_queued_add(sk, val: -skb->truesize);
328 if (!skb_zcopy_pure(skb))
329 sk_mem_uncharge(sk, size: skb->truesize);
330 else
331 sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb)));
332 __kfree_skb(skb);
333}
334
335void sk_forced_mem_schedule(struct sock *sk, int size);
336
337bool tcp_check_oom(const struct sock *sk, int shift);
338
339
340extern struct proto tcp_prot;
341
342#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
343#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
344#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
345#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
346
347void tcp_tsq_work_init(void);
348
349int tcp_v4_err(struct sk_buff *skb, u32);
350
351void tcp_shutdown(struct sock *sk, int how);
352
353int tcp_v4_early_demux(struct sk_buff *skb);
354int tcp_v4_rcv(struct sk_buff *skb);
355
356void tcp_remove_empty_skb(struct sock *sk);
357int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
358int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
359int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
360 size_t size, struct ubuf_info *uarg);
361void tcp_splice_eof(struct socket *sock);
362int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
363int tcp_wmem_schedule(struct sock *sk, int copy);
364void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
365 int size_goal);
366void tcp_release_cb(struct sock *sk);
367void tcp_wfree(struct sk_buff *skb);
368void tcp_write_timer_handler(struct sock *sk);
369void tcp_delack_timer_handler(struct sock *sk);
370int tcp_ioctl(struct sock *sk, int cmd, int *karg);
371enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
372void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
373void tcp_rcvbuf_grow(struct sock *sk);
374void tcp_rcv_space_adjust(struct sock *sk);
375int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
376void tcp_twsk_destructor(struct sock *sk);
377void tcp_twsk_purge(struct list_head *net_exit_list);
378ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
379 struct pipe_inode_info *pipe, size_t len,
380 unsigned int flags);
381struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
382 bool force_schedule);
383
384static inline void tcp_dec_quickack_mode(struct sock *sk)
385{
386 struct inet_connection_sock *icsk = inet_csk(sk);
387
388 if (icsk->icsk_ack.quick) {
389 /* How many ACKs S/ACKing new data have we sent? */
390 const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0;
391
392 if (pkts >= icsk->icsk_ack.quick) {
393 icsk->icsk_ack.quick = 0;
394 /* Leaving quickack mode we deflate ATO. */
395 icsk->icsk_ack.ato = TCP_ATO_MIN;
396 } else
397 icsk->icsk_ack.quick -= pkts;
398 }
399}
400
401#define TCP_ECN_MODE_RFC3168 BIT(0)
402#define TCP_ECN_QUEUE_CWR BIT(1)
403#define TCP_ECN_DEMAND_CWR BIT(2)
404#define TCP_ECN_SEEN BIT(3)
405#define TCP_ECN_MODE_ACCECN BIT(4)
406
407#define TCP_ECN_DISABLED 0
408#define TCP_ECN_MODE_PENDING (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
409#define TCP_ECN_MODE_ANY (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN)
410
411static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp)
412{
413 return tp->ecn_flags & TCP_ECN_MODE_ANY;
414}
415
416static inline bool tcp_ecn_mode_rfc3168(const struct tcp_sock *tp)
417{
418 return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_RFC3168;
419}
420
421static inline bool tcp_ecn_mode_accecn(const struct tcp_sock *tp)
422{
423 return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_ACCECN;
424}
425
426static inline bool tcp_ecn_disabled(const struct tcp_sock *tp)
427{
428 return !tcp_ecn_mode_any(tp);
429}
430
431static inline bool tcp_ecn_mode_pending(const struct tcp_sock *tp)
432{
433 return (tp->ecn_flags & TCP_ECN_MODE_PENDING) == TCP_ECN_MODE_PENDING;
434}
435
436static inline void tcp_ecn_mode_set(struct tcp_sock *tp, u8 mode)
437{
438 tp->ecn_flags &= ~TCP_ECN_MODE_ANY;
439 tp->ecn_flags |= mode;
440}
441
442enum tcp_tw_status {
443 TCP_TW_SUCCESS = 0,
444 TCP_TW_RST = 1,
445 TCP_TW_ACK = 2,
446 TCP_TW_SYN = 3,
447 TCP_TW_ACK_OOW = 4
448};
449
450
451enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
452 struct sk_buff *skb,
453 const struct tcphdr *th,
454 u32 *tw_isn,
455 enum skb_drop_reason *drop_reason);
456struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
457 struct request_sock *req, bool fastopen,
458 bool *lost_race, enum skb_drop_reason *drop_reason);
459enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child,
460 struct sk_buff *skb);
461void tcp_enter_loss(struct sock *sk);
462void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
463void tcp_clear_retrans(struct tcp_sock *tp);
464void tcp_update_metrics(struct sock *sk);
465void tcp_init_metrics(struct sock *sk);
466void tcp_metrics_init(void);
467bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
468void __tcp_close(struct sock *sk, long timeout);
469void tcp_close(struct sock *sk, long timeout);
470void tcp_init_sock(struct sock *sk);
471void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
472__poll_t tcp_poll(struct file *file, struct socket *sock,
473 struct poll_table_struct *wait);
474int do_tcp_getsockopt(struct sock *sk, int level,
475 int optname, sockptr_t optval, sockptr_t optlen);
476int tcp_getsockopt(struct sock *sk, int level, int optname,
477 char __user *optval, int __user *optlen);
478bool tcp_bpf_bypass_getsockopt(int level, int optname);
479int do_tcp_setsockopt(struct sock *sk, int level, int optname,
480 sockptr_t optval, unsigned int optlen);
481int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
482 unsigned int optlen);
483void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
484void tcp_set_keepalive(struct sock *sk, int val);
485void tcp_syn_ack_timeout(const struct request_sock *req);
486int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
487 int flags, int *addr_len);
488int tcp_set_rcvlowat(struct sock *sk, int val);
489int tcp_set_window_clamp(struct sock *sk, int val);
490void tcp_update_recv_tstamps(struct sk_buff *skb,
491 struct scm_timestamping_internal *tss);
492void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
493 struct scm_timestamping_internal *tss);
494void tcp_data_ready(struct sock *sk);
495#ifdef CONFIG_MMU
496int tcp_mmap(struct file *file, struct socket *sock,
497 struct vm_area_struct *vma);
498#endif
499void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
500 struct tcp_options_received *opt_rx,
501 int estab, struct tcp_fastopen_cookie *foc);
502
503/*
504 * BPF SKB-less helpers
505 */
506u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
507 struct tcphdr *th, u32 *cookie);
508u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
509 struct tcphdr *th, u32 *cookie);
510u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss);
511u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
512 const struct tcp_request_sock_ops *af_ops,
513 struct sock *sk, struct tcphdr *th);
514/*
515 * TCP v4 functions exported for the inet6 API
516 */
517
518void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
519void tcp_v4_mtu_reduced(struct sock *sk);
520void tcp_req_err(struct sock *sk, u32 seq, bool abort);
521void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
522int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
523struct sock *tcp_create_openreq_child(const struct sock *sk,
524 struct request_sock *req,
525 struct sk_buff *skb);
526void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
527struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
528 struct request_sock *req,
529 struct dst_entry *dst,
530 struct request_sock *req_unhash,
531 bool *own_req);
532int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
533int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
534int tcp_connect(struct sock *sk);
535enum tcp_synack_type {
536 TCP_SYNACK_NORMAL,
537 TCP_SYNACK_FASTOPEN,
538 TCP_SYNACK_COOKIE,
539};
540struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
541 struct request_sock *req,
542 struct tcp_fastopen_cookie *foc,
543 enum tcp_synack_type synack_type,
544 struct sk_buff *syn_skb);
545int tcp_disconnect(struct sock *sk, int flags);
546
547void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
548int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
549void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
550
551/* From syncookies.c */
552struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
553 struct request_sock *req,
554 struct dst_entry *dst);
555int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th);
556struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
557struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
558 struct sock *sk, struct sk_buff *skb,
559 struct tcp_options_received *tcp_opt,
560 int mss, u32 tsoff);
561
562#if IS_ENABLED(CONFIG_BPF)
563struct bpf_tcp_req_attrs {
564 u32 rcv_tsval;
565 u32 rcv_tsecr;
566 u16 mss;
567 u8 rcv_wscale;
568 u8 snd_wscale;
569 u8 ecn_ok;
570 u8 wscale_ok;
571 u8 sack_ok;
572 u8 tstamp_ok;
573 u8 usec_ts_ok;
574 u8 reserved[3];
575};
576#endif
577
578#ifdef CONFIG_SYN_COOKIES
579
580/* Syncookies use a monotonic timer which increments every 60 seconds.
581 * This counter is used both as a hash input and partially encoded into
582 * the cookie value. A cookie is only validated further if the delta
583 * between the current counter value and the encoded one is less than this,
584 * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
585 * the counter advances immediately after a cookie is generated).
586 */
587#define MAX_SYNCOOKIE_AGE 2
588#define TCP_SYNCOOKIE_PERIOD (60 * HZ)
589#define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
590
591/* syncookies: remember time of last synqueue overflow
592 * But do not dirty this field too often (once per second is enough)
593 * It is racy as we do not hold a lock, but race is very minor.
594 */
595static inline void tcp_synq_overflow(const struct sock *sk)
596{
597 unsigned int last_overflow;
598 unsigned int now = jiffies;
599
600 if (sk->sk_reuseport) {
601 struct sock_reuseport *reuse;
602
603 reuse = rcu_dereference(sk->sk_reuseport_cb);
604 if (likely(reuse)) {
605 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
606 if (!time_between32(now, last_overflow,
607 last_overflow + HZ))
608 WRITE_ONCE(reuse->synq_overflow_ts, now);
609 return;
610 }
611 }
612
613 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
614 if (!time_between32(now, last_overflow, last_overflow + HZ))
615 WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now);
616}
617
618/* syncookies: no recent synqueue overflow on this listening socket? */
619static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
620{
621 unsigned int last_overflow;
622 unsigned int now = jiffies;
623
624 if (sk->sk_reuseport) {
625 struct sock_reuseport *reuse;
626
627 reuse = rcu_dereference(sk->sk_reuseport_cb);
628 if (likely(reuse)) {
629 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
630 return !time_between32(now, last_overflow - HZ,
631 last_overflow +
632 TCP_SYNCOOKIE_VALID);
633 }
634 }
635
636 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
637
638 /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID,
639 * then we're under synflood. However, we have to use
640 * 'last_overflow - HZ' as lower bound. That's because a concurrent
641 * tcp_synq_overflow() could update .ts_recent_stamp after we read
642 * jiffies but before we store .ts_recent_stamp into last_overflow,
643 * which could lead to rejecting a valid syncookie.
644 */
645 return !time_between32(now, last_overflow - HZ,
646 last_overflow + TCP_SYNCOOKIE_VALID);
647}
648
649static inline u32 tcp_cookie_time(void)
650{
651 u64 val = get_jiffies_64();
652
653 do_div(val, TCP_SYNCOOKIE_PERIOD);
654 return val;
655}
656
657/* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */
658static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val)
659{
660 if (usec_ts)
661 return div_u64(dividend: val, NSEC_PER_USEC);
662
663 return div_u64(dividend: val, NSEC_PER_MSEC);
664}
665
666u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
667 u16 *mssp);
668__u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
669u64 cookie_init_timestamp(struct request_sock *req, u64 now);
670bool cookie_timestamp_decode(const struct net *net,
671 struct tcp_options_received *opt);
672
673static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst)
674{
675 return READ_ONCE(net->ipv4.sysctl_tcp_ecn) ||
676 dst_feature(dst, RTAX_FEATURE_ECN);
677}
678
679#if IS_ENABLED(CONFIG_BPF)
680static inline bool cookie_bpf_ok(struct sk_buff *skb)
681{
682 return skb->sk;
683}
684
685struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb);
686#else
687static inline bool cookie_bpf_ok(struct sk_buff *skb)
688{
689 return false;
690}
691
692static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk,
693 struct sk_buff *skb)
694{
695 return NULL;
696}
697#endif
698
699/* From net/ipv6/syncookies.c */
700int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th);
701struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
702
703u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
704 const struct tcphdr *th, u16 *mssp);
705__u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
706#endif
707/* tcp_output.c */
708
709void tcp_skb_entail(struct sock *sk, struct sk_buff *skb);
710void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb);
711void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
712 int nonagle);
713int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
714int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
715void tcp_retransmit_timer(struct sock *sk);
716void tcp_xmit_retransmit_queue(struct sock *);
717void tcp_simple_retransmit(struct sock *);
718void tcp_enter_recovery(struct sock *sk, bool ece_ack);
719int tcp_trim_head(struct sock *, struct sk_buff *, u32);
720enum tcp_queue {
721 TCP_FRAG_IN_WRITE_QUEUE,
722 TCP_FRAG_IN_RTX_QUEUE,
723};
724int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
725 struct sk_buff *skb, u32 len,
726 unsigned int mss_now, gfp_t gfp);
727
728void tcp_send_probe0(struct sock *);
729int tcp_write_wakeup(struct sock *, int mib);
730void tcp_send_fin(struct sock *sk);
731void tcp_send_active_reset(struct sock *sk, gfp_t priority,
732 enum sk_rst_reason reason);
733int tcp_send_synack(struct sock *);
734void tcp_push_one(struct sock *, unsigned int mss_now);
735void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags);
736void tcp_send_ack(struct sock *sk);
737void tcp_send_delayed_ack(struct sock *sk);
738void tcp_send_loss_probe(struct sock *sk);
739bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
740void tcp_skb_collapse_tstamp(struct sk_buff *skb,
741 const struct sk_buff *next_skb);
742
743/* tcp_input.c */
744void tcp_rearm_rto(struct sock *sk);
745void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
746void tcp_done_with_error(struct sock *sk, int err);
747void tcp_reset(struct sock *sk, struct sk_buff *skb);
748void tcp_fin(struct sock *sk);
749void tcp_check_space(struct sock *sk);
750void tcp_sack_compress_send_ack(struct sock *sk);
751
752static inline void tcp_cleanup_skb(struct sk_buff *skb)
753{
754 skb_dst_drop(skb);
755 secpath_reset(skb);
756}
757
758static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb)
759{
760 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
761 DEBUG_NET_WARN_ON_ONCE(secpath_exists(skb));
762 __skb_queue_tail(list: &sk->sk_receive_queue, newsk: skb);
763}
764
765/* tcp_timer.c */
766void tcp_init_xmit_timers(struct sock *);
767static inline void tcp_clear_xmit_timers(struct sock *sk)
768{
769 if (hrtimer_try_to_cancel(timer: &tcp_sk(sk)->pacing_timer) == 1)
770 __sock_put(sk);
771
772 if (hrtimer_try_to_cancel(timer: &tcp_sk(sk)->compressed_ack_timer) == 1)
773 __sock_put(sk);
774
775 inet_csk_clear_xmit_timers(sk);
776}
777
778unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
779unsigned int tcp_current_mss(struct sock *sk);
780u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
781
782/* Bound MSS / TSO packet size with the half of the window */
783static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
784{
785 int cutoff;
786
787 /* When peer uses tiny windows, there is no use in packetizing
788 * to sub-MSS pieces for the sake of SWS or making sure there
789 * are enough packets in the pipe for fast recovery.
790 *
791 * On the other hand, for extremely large MSS devices, handling
792 * smaller than MSS windows in this way does make sense.
793 */
794 if (tp->max_window > TCP_MSS_DEFAULT)
795 cutoff = (tp->max_window >> 1);
796 else
797 cutoff = tp->max_window;
798
799 if (cutoff && pktsize > cutoff)
800 return max_t(int, cutoff, 68U - tp->tcp_header_len);
801 else
802 return pktsize;
803}
804
805/* tcp.c */
806void tcp_get_info(struct sock *, struct tcp_info *);
807
808/* Read 'sendfile()'-style from a TCP socket */
809int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
810 sk_read_actor_t recv_actor);
811int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
812 sk_read_actor_t recv_actor, bool noack,
813 u32 *copied_seq);
814int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
815struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
816void tcp_read_done(struct sock *sk, size_t len);
817
818void tcp_initialize_rcv_mss(struct sock *sk);
819
820int tcp_mtu_to_mss(struct sock *sk, int pmtu);
821int tcp_mss_to_mtu(struct sock *sk, int mss);
822void tcp_mtup_init(struct sock *sk);
823
824static inline unsigned int tcp_rto_max(const struct sock *sk)
825{
826 return READ_ONCE(inet_csk(sk)->icsk_rto_max);
827}
828
829static inline void tcp_bound_rto(struct sock *sk)
830{
831 inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk));
832}
833
834static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
835{
836 return usecs_to_jiffies(u: (tp->srtt_us >> 3) + tp->rttvar_us);
837}
838
839u32 tcp_delack_max(const struct sock *sk);
840
841/* Compute the actual rto_min value */
842static inline u32 tcp_rto_min(const struct sock *sk)
843{
844 const struct dst_entry *dst = __sk_dst_get(sk);
845 u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min);
846
847 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
848 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
849 return rto_min;
850}
851
852static inline u32 tcp_rto_min_us(const struct sock *sk)
853{
854 return jiffies_to_usecs(j: tcp_rto_min(sk));
855}
856
857static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
858{
859 return dst_metric_locked(dst, RTAX_CC_ALGO);
860}
861
862/* Minimum RTT in usec. ~0 means not available. */
863static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
864{
865 return minmax_get(m: &tp->rtt_min);
866}
867
868/* Compute the actual receive window we are currently advertising.
869 * Rcv_nxt can be after the window if our peer push more data
870 * than the offered window.
871 */
872static inline u32 tcp_receive_window(const struct tcp_sock *tp)
873{
874 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
875
876 if (win < 0)
877 win = 0;
878 return (u32) win;
879}
880
881/* Choose a new window, without checks for shrinking, and without
882 * scaling applied to the result. The caller does these things
883 * if necessary. This is a "raw" window selection.
884 */
885u32 __tcp_select_window(struct sock *sk);
886
887void tcp_send_window_probe(struct sock *sk);
888
889/* TCP uses 32bit jiffies to save some space.
890 * Note that this is different from tcp_time_stamp, which
891 * historically has been the same until linux-4.13.
892 */
893#define tcp_jiffies32 ((u32)jiffies)
894
895/*
896 * Deliver a 32bit value for TCP timestamp option (RFC 7323)
897 * It is no longer tied to jiffies, but to 1 ms clock.
898 * Note: double check if you want to use tcp_jiffies32 instead of this.
899 */
900#define TCP_TS_HZ 1000
901
902static inline u64 tcp_clock_ns(void)
903{
904 return ktime_get_ns();
905}
906
907static inline u64 tcp_clock_us(void)
908{
909 return div_u64(dividend: tcp_clock_ns(), NSEC_PER_USEC);
910}
911
912static inline u64 tcp_clock_ms(void)
913{
914 return div_u64(dividend: tcp_clock_ns(), NSEC_PER_MSEC);
915}
916
917/* TCP Timestamp included in TS option (RFC 1323) can either use ms
918 * or usec resolution. Each socket carries a flag to select one or other
919 * resolution, as the route attribute could change anytime.
920 * Each flow must stick to initial resolution.
921 */
922static inline u32 tcp_clock_ts(bool usec_ts)
923{
924 return usec_ts ? tcp_clock_us() : tcp_clock_ms();
925}
926
927static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp)
928{
929 return div_u64(dividend: tp->tcp_mstamp, USEC_PER_MSEC);
930}
931
932static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp)
933{
934 if (tp->tcp_usec_ts)
935 return tp->tcp_mstamp;
936 return tcp_time_stamp_ms(tp);
937}
938
939void tcp_mstamp_refresh(struct tcp_sock *tp);
940
941static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
942{
943 return max_t(s64, t1 - t0, 0);
944}
945
946/* provide the departure time in us unit */
947static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
948{
949 return div_u64(dividend: skb->skb_mstamp_ns, NSEC_PER_USEC);
950}
951
952/* Provide skb TSval in usec or ms unit */
953static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb)
954{
955 if (usec_ts)
956 return tcp_skb_timestamp_us(skb);
957
958 return div_u64(dividend: skb->skb_mstamp_ns, NSEC_PER_MSEC);
959}
960
961static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw)
962{
963 return tcp_clock_ts(usec_ts: tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset;
964}
965
966static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq)
967{
968 return tcp_clock_ts(usec_ts: treq->req_usec_ts) + treq->ts_off;
969}
970
971#define tcp_flag_byte(th) (((u_int8_t *)th)[13])
972
973#define TCPHDR_FIN BIT(0)
974#define TCPHDR_SYN BIT(1)
975#define TCPHDR_RST BIT(2)
976#define TCPHDR_PSH BIT(3)
977#define TCPHDR_ACK BIT(4)
978#define TCPHDR_URG BIT(5)
979#define TCPHDR_ECE BIT(6)
980#define TCPHDR_CWR BIT(7)
981#define TCPHDR_AE BIT(8)
982#define TCPHDR_FLAGS_MASK (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
983 TCPHDR_PSH | TCPHDR_ACK | TCPHDR_URG | \
984 TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
985#define tcp_flags_ntohs(th) (ntohs(*(__be16 *)&tcp_flag_word(th)) & \
986 TCPHDR_FLAGS_MASK)
987
988#define TCPHDR_ACE (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)
989#define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
990#define TCPHDR_SYNACK_ACCECN (TCPHDR_SYN | TCPHDR_ACK | TCPHDR_CWR)
991
992#define TCP_ACCECN_CEP_ACE_MASK 0x7
993#define TCP_ACCECN_ACE_MAX_DELTA 6
994
995/* To avoid/detect middlebox interference, not all counters start at 0.
996 * See draft-ietf-tcpm-accurate-ecn for the latest values.
997 */
998#define TCP_ACCECN_CEP_INIT_OFFSET 5
999#define TCP_ACCECN_E1B_INIT_OFFSET 1
1000#define TCP_ACCECN_E0B_INIT_OFFSET 1
1001#define TCP_ACCECN_CEB_INIT_OFFSET 0
1002
1003/* State flags for sacked in struct tcp_skb_cb */
1004enum tcp_skb_cb_sacked_flags {
1005 TCPCB_SACKED_ACKED = (1 << 0), /* SKB ACK'd by a SACK block */
1006 TCPCB_SACKED_RETRANS = (1 << 1), /* SKB retransmitted */
1007 TCPCB_LOST = (1 << 2), /* SKB is lost */
1008 TCPCB_TAGBITS = (TCPCB_SACKED_ACKED | TCPCB_SACKED_RETRANS |
1009 TCPCB_LOST), /* All tag bits */
1010 TCPCB_REPAIRED = (1 << 4), /* SKB repaired (no skb_mstamp_ns) */
1011 TCPCB_EVER_RETRANS = (1 << 7), /* Ever retransmitted frame */
1012 TCPCB_RETRANS = (TCPCB_SACKED_RETRANS | TCPCB_EVER_RETRANS |
1013 TCPCB_REPAIRED),
1014};
1015
1016/* This is what the send packet queuing engine uses to pass
1017 * TCP per-packet control information to the transmission code.
1018 * We also store the host-order sequence numbers in here too.
1019 * This is 44 bytes if IPV6 is enabled.
1020 * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
1021 */
1022struct tcp_skb_cb {
1023 __u32 seq; /* Starting sequence number */
1024 __u32 end_seq; /* SEQ + FIN + SYN + datalen */
1025 union {
1026 /* Note :
1027 * tcp_gso_segs/size are used in write queue only,
1028 * cf tcp_skb_pcount()/tcp_skb_mss()
1029 */
1030 struct {
1031 u16 tcp_gso_segs;
1032 u16 tcp_gso_size;
1033 };
1034 };
1035 __u16 tcp_flags; /* TCP header flags (tcp[12-13])*/
1036
1037 __u8 sacked; /* State flags for SACK. */
1038 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1039#define TSTAMP_ACK_SK 0x1
1040#define TSTAMP_ACK_BPF 0x2
1041 __u8 txstamp_ack:2, /* Record TX timestamp for ack? */
1042 eor:1, /* Is skb MSG_EOR marked? */
1043 has_rxtstamp:1, /* SKB has a RX timestamp */
1044 unused:4;
1045 __u32 ack_seq; /* Sequence number ACK'd */
1046 union {
1047 struct {
1048#define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
1049 /* There is space for up to 24 bytes */
1050 __u32 is_app_limited:1, /* cwnd not fully used? */
1051 delivered_ce:20,
1052 unused:11;
1053 /* pkts S/ACKed so far upon tx of skb, incl retrans: */
1054 __u32 delivered;
1055 /* start of send pipeline phase */
1056 u64 first_tx_mstamp;
1057 /* when we reached the "delivered" count */
1058 u64 delivered_mstamp;
1059 } tx; /* only used for outgoing skbs */
1060 union {
1061 struct inet_skb_parm h4;
1062#if IS_ENABLED(CONFIG_IPV6)
1063 struct inet6_skb_parm h6;
1064#endif
1065 } header; /* For incoming skbs */
1066 };
1067};
1068
1069#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
1070
1071extern const struct inet_connection_sock_af_ops ipv4_specific;
1072
1073#if IS_ENABLED(CONFIG_IPV6)
1074/* This is the variant of inet6_iif() that must be used by TCP,
1075 * as TCP moves IP6CB into a different location in skb->cb[]
1076 */
1077static inline int tcp_v6_iif(const struct sk_buff *skb)
1078{
1079 return TCP_SKB_CB(skb)->header.h6.iif;
1080}
1081
1082static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
1083{
1084 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
1085
1086 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
1087}
1088
1089/* TCP_SKB_CB reference means this can not be used from early demux */
1090static inline int tcp_v6_sdif(const struct sk_buff *skb)
1091{
1092#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1093 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
1094 return TCP_SKB_CB(skb)->header.h6.iif;
1095#endif
1096 return 0;
1097}
1098
1099extern const struct inet_connection_sock_af_ops ipv6_specific;
1100
1101INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
1102INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
1103void tcp_v6_early_demux(struct sk_buff *skb);
1104
1105#endif
1106
1107/* TCP_SKB_CB reference means this can not be used from early demux */
1108static inline int tcp_v4_sdif(struct sk_buff *skb)
1109{
1110#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
1111 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
1112 return TCP_SKB_CB(skb)->header.h4.iif;
1113#endif
1114 return 0;
1115}
1116
1117/* Due to TSO, an SKB can be composed of multiple actual
1118 * packets. To keep these tracked properly, we use this.
1119 */
1120static inline int tcp_skb_pcount(const struct sk_buff *skb)
1121{
1122 return TCP_SKB_CB(skb)->tcp_gso_segs;
1123}
1124
1125static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
1126{
1127 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
1128}
1129
1130static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
1131{
1132 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
1133}
1134
1135/* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */
1136static inline int tcp_skb_mss(const struct sk_buff *skb)
1137{
1138 return TCP_SKB_CB(skb)->tcp_gso_size;
1139}
1140
1141static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
1142{
1143 return likely(!TCP_SKB_CB(skb)->eor);
1144}
1145
1146static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
1147 const struct sk_buff *from)
1148{
1149 /* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */
1150 return likely(tcp_skb_can_collapse_to(to) &&
1151 mptcp_skb_can_collapse(to, from) &&
1152 skb_pure_zcopy_same(to, from) &&
1153 skb_frags_readable(to) == skb_frags_readable(from));
1154}
1155
1156static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to,
1157 const struct sk_buff *from)
1158{
1159 return likely(mptcp_skb_can_collapse(to, from) &&
1160 !skb_cmp_decrypted(to, from));
1161}
1162
1163/* Events passed to congestion control interface */
1164enum tcp_ca_event {
1165 CA_EVENT_TX_START, /* first transmit when no packets in flight */
1166 CA_EVENT_CWND_RESTART, /* congestion window restart */
1167 CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */
1168 CA_EVENT_LOSS, /* loss timeout */
1169 CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */
1170 CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */
1171};
1172
1173/* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1174enum tcp_ca_ack_event_flags {
1175 CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */
1176 CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */
1177 CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */
1178};
1179
1180/*
1181 * Interface for adding new TCP congestion control handlers
1182 */
1183#define TCP_CA_NAME_MAX 16
1184#define TCP_CA_MAX 128
1185#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
1186
1187#define TCP_CA_UNSPEC 0
1188
1189/* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
1190#define TCP_CONG_NON_RESTRICTED BIT(0)
1191/* Requires ECN/ECT set on all packets */
1192#define TCP_CONG_NEEDS_ECN BIT(1)
1193#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
1194
1195union tcp_cc_info;
1196
1197struct ack_sample {
1198 u32 pkts_acked;
1199 s32 rtt_us;
1200 u32 in_flight;
1201};
1202
1203/* A rate sample measures the number of (original/retransmitted) data
1204 * packets delivered "delivered" over an interval of time "interval_us".
1205 * The tcp_rate.c code fills in the rate sample, and congestion
1206 * control modules that define a cong_control function to run at the end
1207 * of ACK processing can optionally chose to consult this sample when
1208 * setting cwnd and pacing rate.
1209 * A sample is invalid if "delivered" or "interval_us" is negative.
1210 */
1211struct rate_sample {
1212 u64 prior_mstamp; /* starting timestamp for interval */
1213 u32 prior_delivered; /* tp->delivered at "prior_mstamp" */
1214 u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1215 s32 delivered; /* number of packets delivered over interval */
1216 s32 delivered_ce; /* number of packets delivered w/ CE marks*/
1217 long interval_us; /* time for tp->delivered to incr "delivered" */
1218 u32 snd_interval_us; /* snd interval for delivered packets */
1219 u32 rcv_interval_us; /* rcv interval for delivered packets */
1220 long rtt_us; /* RTT of last (S)ACKed packet (or -1) */
1221 int losses; /* number of packets marked lost upon ACK */
1222 u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
1223 u32 prior_in_flight; /* in flight before this ACK */
1224 u32 last_end_seq; /* end_seq of most recently ACKed packet */
1225 bool is_app_limited; /* is sample from packet with bubble in pipe? */
1226 bool is_retrans; /* is sample from retransmission? */
1227 bool is_ack_delayed; /* is this (likely) a delayed ACK? */
1228};
1229
1230struct tcp_congestion_ops {
1231/* fast path fields are put first to fill one cache line */
1232
1233 /* return slow start threshold (required) */
1234 u32 (*ssthresh)(struct sock *sk);
1235
1236 /* do new cwnd calculation (required) */
1237 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1238
1239 /* call before changing ca_state (optional) */
1240 void (*set_state)(struct sock *sk, u8 new_state);
1241
1242 /* call when cwnd event occurs (optional) */
1243 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1244
1245 /* call when ack arrives (optional) */
1246 void (*in_ack_event)(struct sock *sk, u32 flags);
1247
1248 /* hook for packet ack accounting (optional) */
1249 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1250
1251 /* override sysctl_tcp_min_tso_segs */
1252 u32 (*min_tso_segs)(struct sock *sk);
1253
1254 /* call when packets are delivered to update cwnd and pacing rate,
1255 * after all the ca_state processing. (optional)
1256 */
1257 void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs);
1258
1259
1260 /* new value of cwnd after loss (required) */
1261 u32 (*undo_cwnd)(struct sock *sk);
1262 /* returns the multiplier used in tcp_sndbuf_expand (optional) */
1263 u32 (*sndbuf_expand)(struct sock *sk);
1264
1265/* control/slow paths put last */
1266 /* get info for inet_diag (optional) */
1267 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1268 union tcp_cc_info *info);
1269
1270 char name[TCP_CA_NAME_MAX];
1271 struct module *owner;
1272 struct list_head list;
1273 u32 key;
1274 u32 flags;
1275
1276 /* initialize private data (optional) */
1277 void (*init)(struct sock *sk);
1278 /* cleanup private data (optional) */
1279 void (*release)(struct sock *sk);
1280} ____cacheline_aligned_in_smp;
1281
1282int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1283void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1284int tcp_update_congestion_control(struct tcp_congestion_ops *type,
1285 struct tcp_congestion_ops *old_type);
1286int tcp_validate_congestion_control(struct tcp_congestion_ops *ca);
1287
1288void tcp_assign_congestion_control(struct sock *sk);
1289void tcp_init_congestion_control(struct sock *sk);
1290void tcp_cleanup_congestion_control(struct sock *sk);
1291int tcp_set_default_congestion_control(struct net *net, const char *name);
1292void tcp_get_default_congestion_control(struct net *net, char *name);
1293void tcp_get_available_congestion_control(char *buf, size_t len);
1294void tcp_get_allowed_congestion_control(char *buf, size_t len);
1295int tcp_set_allowed_congestion_control(char *allowed);
1296int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1297 bool cap_net_admin);
1298u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1299void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1300
1301u32 tcp_reno_ssthresh(struct sock *sk);
1302u32 tcp_reno_undo_cwnd(struct sock *sk);
1303void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1304extern struct tcp_congestion_ops tcp_reno;
1305
1306struct tcp_congestion_ops *tcp_ca_find(const char *name);
1307struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1308u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca);
1309#ifdef CONFIG_INET
1310char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1311#else
1312static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1313{
1314 return NULL;
1315}
1316#endif
1317
1318static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1319{
1320 const struct inet_connection_sock *icsk = inet_csk(sk);
1321
1322 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1323}
1324
1325static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1326{
1327 const struct inet_connection_sock *icsk = inet_csk(sk);
1328
1329 if (icsk->icsk_ca_ops->cwnd_event)
1330 icsk->icsk_ca_ops->cwnd_event(sk, event);
1331}
1332
1333/* From tcp_cong.c */
1334void tcp_set_ca_state(struct sock *sk, const u8 ca_state);
1335
1336/* From tcp_rate.c */
1337void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1338void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1339 struct rate_sample *rs);
1340void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1341 bool is_sack_reneg, struct rate_sample *rs);
1342void tcp_rate_check_app_limited(struct sock *sk);
1343
1344static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2)
1345{
1346 return t1 > t2 || (t1 == t2 && after(seq1, seq2));
1347}
1348
1349/* These functions determine how the current flow behaves in respect of SACK
1350 * handling. SACK is negotiated with the peer, and therefore it can vary
1351 * between different flows.
1352 *
1353 * tcp_is_sack - SACK enabled
1354 * tcp_is_reno - No SACK
1355 */
1356static inline int tcp_is_sack(const struct tcp_sock *tp)
1357{
1358 return likely(tp->rx_opt.sack_ok);
1359}
1360
1361static inline bool tcp_is_reno(const struct tcp_sock *tp)
1362{
1363 return !tcp_is_sack(tp);
1364}
1365
1366static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1367{
1368 return tp->sacked_out + tp->lost_out;
1369}
1370
1371/* This determines how many packets are "in the network" to the best
1372 * of our knowledge. In many cases it is conservative, but where
1373 * detailed information is available from the receiver (via SACK
1374 * blocks etc.) we can make more aggressive calculations.
1375 *
1376 * Use this for decisions involving congestion control, use just
1377 * tp->packets_out to determine if the send queue is empty or not.
1378 *
1379 * Read this equation as:
1380 *
1381 * "Packets sent once on transmission queue" MINUS
1382 * "Packets left network, but not honestly ACKed yet" PLUS
1383 * "Packets fast retransmitted"
1384 */
1385static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1386{
1387 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1388}
1389
1390#define TCP_INFINITE_SSTHRESH 0x7fffffff
1391
1392static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp)
1393{
1394 return tp->snd_cwnd;
1395}
1396
1397static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val)
1398{
1399 WARN_ON_ONCE((int)val <= 0);
1400 tp->snd_cwnd = val;
1401}
1402
1403static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1404{
1405 return tcp_snd_cwnd(tp) < tp->snd_ssthresh;
1406}
1407
1408static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1409{
1410 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1411}
1412
1413static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1414{
1415 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1416 (1 << inet_csk(sk)->icsk_ca_state);
1417}
1418
1419/* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1420 * The exception is cwnd reduction phase, when cwnd is decreasing towards
1421 * ssthresh.
1422 */
1423static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1424{
1425 const struct tcp_sock *tp = tcp_sk(sk);
1426
1427 if (tcp_in_cwnd_reduction(sk))
1428 return tp->snd_ssthresh;
1429 else
1430 return max(tp->snd_ssthresh,
1431 ((tcp_snd_cwnd(tp) >> 1) +
1432 (tcp_snd_cwnd(tp) >> 2)));
1433}
1434
1435/* Use define here intentionally to get WARN_ON location shown at the caller */
1436#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
1437
1438void tcp_enter_cwr(struct sock *sk);
1439__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1440
1441/* The maximum number of MSS of available cwnd for which TSO defers
1442 * sending if not using sysctl_tcp_tso_win_divisor.
1443 */
1444static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1445{
1446 return 3;
1447}
1448
1449/* Returns end sequence number of the receiver's advertised window */
1450static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1451{
1452 return tp->snd_una + tp->snd_wnd;
1453}
1454
1455/* We follow the spirit of RFC2861 to validate cwnd but implement a more
1456 * flexible approach. The RFC suggests cwnd should not be raised unless
1457 * it was fully used previously. And that's exactly what we do in
1458 * congestion avoidance mode. But in slow start we allow cwnd to grow
1459 * as long as the application has used half the cwnd.
1460 * Example :
1461 * cwnd is 10 (IW10), but application sends 9 frames.
1462 * We allow cwnd to reach 18 when all frames are ACKed.
1463 * This check is safe because it's as aggressive as slow start which already
1464 * risks 100% overshoot. The advantage is that we discourage application to
1465 * either send more filler packets or data to artificially blow up the cwnd
1466 * usage, and allow application-limited process to probe bw more aggressively.
1467 */
1468static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1469{
1470 const struct tcp_sock *tp = tcp_sk(sk);
1471
1472 if (tp->is_cwnd_limited)
1473 return true;
1474
1475 /* If in slow start, ensure cwnd grows to twice what was ACKed. */
1476 if (tcp_in_slow_start(tp))
1477 return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;
1478
1479 return false;
1480}
1481
1482/* BBR congestion control needs pacing.
1483 * Same remark for SO_MAX_PACING_RATE.
1484 * sch_fq packet scheduler is efficiently handling pacing,
1485 * but is not always installed/used.
1486 * Return true if TCP stack should pace packets itself.
1487 */
1488static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1489{
1490 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1491}
1492
1493/* Estimates in how many jiffies next packet for this flow can be sent.
1494 * Scheduling a retransmit timer too early would be silly.
1495 */
1496static inline unsigned long tcp_pacing_delay(const struct sock *sk)
1497{
1498 s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
1499
1500 return delay > 0 ? nsecs_to_jiffies(n: delay) : 0;
1501}
1502
1503static inline void tcp_reset_xmit_timer(struct sock *sk,
1504 const int what,
1505 unsigned long when,
1506 bool pace_delay)
1507{
1508 if (pace_delay)
1509 when += tcp_pacing_delay(sk);
1510 inet_csk_reset_xmit_timer(sk, what, when,
1511 max_when: tcp_rto_max(sk));
1512}
1513
1514/* Something is really bad, we could not queue an additional packet,
1515 * because qdisc is full or receiver sent a 0 window, or we are paced.
1516 * We do not want to add fuel to the fire, or abort too early,
1517 * so make sure the timer we arm now is at least 200ms in the future,
1518 * regardless of current icsk_rto value (as it could be ~2ms)
1519 */
1520static inline unsigned long tcp_probe0_base(const struct sock *sk)
1521{
1522 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1523}
1524
1525/* Variant of inet_csk_rto_backoff() used for zero window probes */
1526static inline unsigned long tcp_probe0_when(const struct sock *sk,
1527 unsigned long max_when)
1528{
1529 u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1,
1530 inet_csk(sk)->icsk_backoff);
1531 u64 when = (u64)tcp_probe0_base(sk) << backoff;
1532
1533 return (unsigned long)min_t(u64, when, max_when);
1534}
1535
1536static inline void tcp_check_probe_timer(struct sock *sk)
1537{
1538 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1539 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1540 when: tcp_probe0_base(sk), pace_delay: true);
1541}
1542
1543static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1544{
1545 tp->snd_wl1 = seq;
1546}
1547
1548static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1549{
1550 tp->snd_wl1 = seq;
1551}
1552
1553/*
1554 * Calculate(/check) TCP checksum
1555 */
1556static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1557 __be32 daddr, __wsum base)
1558{
1559 return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, sum: base);
1560}
1561
1562static inline bool tcp_checksum_complete(struct sk_buff *skb)
1563{
1564 return !skb_csum_unnecessary(skb) &&
1565 __skb_checksum_complete(skb);
1566}
1567
1568bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1569 enum skb_drop_reason *reason);
1570
1571
1572int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason);
1573void tcp_set_state(struct sock *sk, int state);
1574void tcp_done(struct sock *sk);
1575int tcp_abort(struct sock *sk, int err);
1576
1577static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1578{
1579 rx_opt->dsack = 0;
1580 rx_opt->num_sacks = 0;
1581}
1582
1583void tcp_cwnd_restart(struct sock *sk, s32 delta);
1584
1585static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1586{
1587 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1588 struct tcp_sock *tp = tcp_sk(sk);
1589 s32 delta;
1590
1591 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||
1592 tp->packets_out || ca_ops->cong_control)
1593 return;
1594 delta = tcp_jiffies32 - tp->lsndtime;
1595 if (delta > inet_csk(sk)->icsk_rto)
1596 tcp_cwnd_restart(sk, delta);
1597}
1598
1599/* Determine a window scaling and initial window to offer. */
1600void tcp_select_initial_window(const struct sock *sk, int __space,
1601 __u32 mss, __u32 *rcv_wnd,
1602 __u32 *window_clamp, int wscale_ok,
1603 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1604
1605static inline int __tcp_win_from_space(u8 scaling_ratio, int space)
1606{
1607 s64 scaled_space = (s64)space * scaling_ratio;
1608
1609 return scaled_space >> TCP_RMEM_TO_WIN_SCALE;
1610}
1611
1612static inline int tcp_win_from_space(const struct sock *sk, int space)
1613{
1614 return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space);
1615}
1616
1617/* inverse of __tcp_win_from_space() */
1618static inline int __tcp_space_from_win(u8 scaling_ratio, int win)
1619{
1620 u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE;
1621
1622 do_div(val, scaling_ratio);
1623 return val;
1624}
1625
1626static inline int tcp_space_from_win(const struct sock *sk, int win)
1627{
1628 return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win);
1629}
1630
1631/* Assume a 50% default for skb->len/skb->truesize ratio.
1632 * This may be adjusted later in tcp_measure_rcv_mss().
1633 */
1634#define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1))
1635
1636static inline void tcp_scaling_ratio_init(struct sock *sk)
1637{
1638 tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
1639}
1640
1641/* Note: caller must be prepared to deal with negative returns */
1642static inline int tcp_space(const struct sock *sk)
1643{
1644 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1645 READ_ONCE(sk->sk_backlog.len) -
1646 atomic_read(v: &sk->sk_rmem_alloc));
1647}
1648
1649static inline int tcp_full_space(const struct sock *sk)
1650{
1651 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1652}
1653
1654static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh)
1655{
1656 int unused_mem = sk_unused_reserved_mem(sk);
1657 struct tcp_sock *tp = tcp_sk(sk);
1658
1659 tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh);
1660 if (unused_mem)
1661 tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,
1662 tcp_win_from_space(sk, unused_mem));
1663}
1664
1665static inline void tcp_adjust_rcv_ssthresh(struct sock *sk)
1666{
1667 __tcp_adjust_rcv_ssthresh(sk, new_ssthresh: 4U * tcp_sk(sk)->advmss);
1668}
1669
1670void tcp_cleanup_rbuf(struct sock *sk, int copied);
1671void __tcp_cleanup_rbuf(struct sock *sk, int copied);
1672
1673
1674/* We provision sk_rcvbuf around 200% of sk_rcvlowat.
1675 * If 87.5 % (7/8) of the space has been consumed, we want to override
1676 * SO_RCVLOWAT constraint, since we are receiving skbs with too small
1677 * len/truesize ratio.
1678 */
1679static inline bool tcp_rmem_pressure(const struct sock *sk)
1680{
1681 int rcvbuf, threshold;
1682
1683 if (tcp_under_memory_pressure(sk))
1684 return true;
1685
1686 rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1687 threshold = rcvbuf - (rcvbuf >> 3);
1688
1689 return atomic_read(v: &sk->sk_rmem_alloc) > threshold;
1690}
1691
1692static inline bool tcp_epollin_ready(const struct sock *sk, int target)
1693{
1694 const struct tcp_sock *tp = tcp_sk(sk);
1695 int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);
1696
1697 if (avail <= 0)
1698 return false;
1699
1700 return (avail >= target) || tcp_rmem_pressure(sk) ||
1701 (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);
1702}
1703
1704extern void tcp_openreq_init_rwin(struct request_sock *req,
1705 const struct sock *sk_listener,
1706 const struct dst_entry *dst);
1707
1708void tcp_enter_memory_pressure(struct sock *sk);
1709void tcp_leave_memory_pressure(struct sock *sk);
1710
1711static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1712{
1713 struct net *net = sock_net(sk: (struct sock *)tp);
1714 int val;
1715
1716 /* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl()
1717 * and do_tcp_setsockopt().
1718 */
1719 val = READ_ONCE(tp->keepalive_intvl);
1720
1721 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);
1722}
1723
1724static inline int keepalive_time_when(const struct tcp_sock *tp)
1725{
1726 struct net *net = sock_net(sk: (struct sock *)tp);
1727 int val;
1728
1729 /* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */
1730 val = READ_ONCE(tp->keepalive_time);
1731
1732 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
1733}
1734
1735static inline int keepalive_probes(const struct tcp_sock *tp)
1736{
1737 struct net *net = sock_net(sk: (struct sock *)tp);
1738 int val;
1739
1740 /* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt()
1741 * and do_tcp_setsockopt().
1742 */
1743 val = READ_ONCE(tp->keepalive_probes);
1744
1745 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);
1746}
1747
1748static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1749{
1750 const struct inet_connection_sock *icsk = &tp->inet_conn;
1751
1752 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1753 tcp_jiffies32 - tp->rcv_tstamp);
1754}
1755
1756static inline int tcp_fin_time(const struct sock *sk)
1757{
1758 int fin_timeout = tcp_sk(sk)->linger2 ? :
1759 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);
1760 const int rto = inet_csk(sk)->icsk_rto;
1761
1762 if (fin_timeout < (rto << 2) - (rto >> 1))
1763 fin_timeout = (rto << 2) - (rto >> 1);
1764
1765 return fin_timeout;
1766}
1767
1768static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1769 int paws_win)
1770{
1771 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1772 return true;
1773 if (unlikely(!time_before32(ktime_get_seconds(),
1774 rx_opt->ts_recent_stamp + TCP_PAWS_WRAP)))
1775 return true;
1776 /*
1777 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
1778 * then following tcp messages have valid values. Ignore 0 value,
1779 * or else 'negative' tsval might forbid us to accept their packets.
1780 */
1781 if (!rx_opt->ts_recent)
1782 return true;
1783 return false;
1784}
1785
1786static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1787 int rst)
1788{
1789 if (tcp_paws_check(rx_opt, paws_win: 0))
1790 return false;
1791
1792 /* RST segments are not recommended to carry timestamp,
1793 and, if they do, it is recommended to ignore PAWS because
1794 "their cleanup function should take precedence over timestamps."
1795 Certainly, it is mistake. It is necessary to understand the reasons
1796 of this constraint to relax it: if peer reboots, clock may go
1797 out-of-sync and half-open connections will not be reset.
1798 Actually, the problem would be not existing if all
1799 the implementations followed draft about maintaining clock
1800 via reboots. Linux-2.2 DOES NOT!
1801
1802 However, we can relax time bounds for RST segments to MSL.
1803 */
1804 if (rst && !time_before32(ktime_get_seconds(),
1805 rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1806 return false;
1807 return true;
1808}
1809
1810static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
1811{
1812 u32 ace;
1813
1814 /* mptcp hooks are only on the slow path */
1815 if (sk_is_mptcp(sk: (struct sock *)tp))
1816 return;
1817
1818 ace = tcp_ecn_mode_accecn(tp) ?
1819 ((tp->delivered_ce + TCP_ACCECN_CEP_INIT_OFFSET) &
1820 TCP_ACCECN_CEP_ACE_MASK) : 0;
1821
1822 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
1823 (ace << 22) |
1824 ntohl(TCP_FLAG_ACK) |
1825 snd_wnd);
1826}
1827
1828static inline void tcp_fast_path_on(struct tcp_sock *tp)
1829{
1830 __tcp_fast_path_on(tp, snd_wnd: tp->snd_wnd >> tp->rx_opt.snd_wscale);
1831}
1832
1833static inline void tcp_fast_path_check(struct sock *sk)
1834{
1835 struct tcp_sock *tp = tcp_sk(sk);
1836
1837 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
1838 tp->rcv_wnd &&
1839 atomic_read(v: &sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
1840 !tp->urg_data)
1841 tcp_fast_path_on(tp);
1842}
1843
1844bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1845 int mib_idx, u32 *last_oow_ack_time);
1846
1847static inline void tcp_mib_init(struct net *net)
1848{
1849 /* See RFC 2012 */
1850 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1851 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1852 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1853 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1854}
1855
1856/* from STCP */
1857static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1858{
1859 tp->retransmit_skb_hint = NULL;
1860}
1861
1862#define tcp_md5_addr tcp_ao_addr
1863
1864/* - key database */
1865struct tcp_md5sig_key {
1866 struct hlist_node node;
1867 u8 keylen;
1868 u8 family; /* AF_INET or AF_INET6 */
1869 u8 prefixlen;
1870 u8 flags;
1871 union tcp_md5_addr addr;
1872 int l3index; /* set if key added with L3 scope */
1873 u8 key[TCP_MD5SIG_MAXKEYLEN];
1874 struct rcu_head rcu;
1875};
1876
1877/* - sock block */
1878struct tcp_md5sig_info {
1879 struct hlist_head head;
1880 struct rcu_head rcu;
1881};
1882
1883/* - pseudo header */
1884struct tcp4_pseudohdr {
1885 __be32 saddr;
1886 __be32 daddr;
1887 __u8 pad;
1888 __u8 protocol;
1889 __be16 len;
1890};
1891
1892struct tcp6_pseudohdr {
1893 struct in6_addr saddr;
1894 struct in6_addr daddr;
1895 __be32 len;
1896 __be32 protocol; /* including padding */
1897};
1898
1899union tcp_md5sum_block {
1900 struct tcp4_pseudohdr ip4;
1901#if IS_ENABLED(CONFIG_IPV6)
1902 struct tcp6_pseudohdr ip6;
1903#endif
1904};
1905
1906/*
1907 * struct tcp_sigpool - per-CPU pool of ahash_requests
1908 * @scratch: per-CPU temporary area, that can be used between
1909 * tcp_sigpool_start() and tcp_sigpool_end() to perform
1910 * crypto request
1911 * @req: pre-allocated ahash request
1912 */
1913struct tcp_sigpool {
1914 void *scratch;
1915 struct ahash_request *req;
1916};
1917
1918int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size);
1919void tcp_sigpool_get(unsigned int id);
1920void tcp_sigpool_release(unsigned int id);
1921int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
1922 const struct sk_buff *skb,
1923 unsigned int header_len);
1924
1925/**
1926 * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash
1927 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
1928 * @c: returned tcp_sigpool for usage (uninitialized on failure)
1929 *
1930 * Returns: 0 on success, error otherwise.
1931 */
1932int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c);
1933/**
1934 * tcp_sigpool_end - enable bh and stop using tcp_sigpool
1935 * @c: tcp_sigpool context that was returned by tcp_sigpool_start()
1936 */
1937void tcp_sigpool_end(struct tcp_sigpool *c);
1938size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len);
1939/* - functions */
1940int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1941 const struct sock *sk, const struct sk_buff *skb);
1942int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1943 int family, u8 prefixlen, int l3index, u8 flags,
1944 const u8 *newkey, u8 newkeylen);
1945int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1946 int family, u8 prefixlen, int l3index,
1947 struct tcp_md5sig_key *key);
1948
1949int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1950 int family, u8 prefixlen, int l3index, u8 flags);
1951void tcp_clear_md5_list(struct sock *sk);
1952struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1953 const struct sock *addr_sk);
1954
1955#ifdef CONFIG_TCP_MD5SIG
1956struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1957 const union tcp_md5_addr *addr,
1958 int family, bool any_l3index);
1959static inline struct tcp_md5sig_key *
1960tcp_md5_do_lookup(const struct sock *sk, int l3index,
1961 const union tcp_md5_addr *addr, int family)
1962{
1963 if (!static_branch_unlikely(&tcp_md5_needed.key))
1964 return NULL;
1965 return __tcp_md5_do_lookup(sk, l3index, addr, family, any_l3index: false);
1966}
1967
1968static inline struct tcp_md5sig_key *
1969tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1970 const union tcp_md5_addr *addr, int family)
1971{
1972 if (!static_branch_unlikely(&tcp_md5_needed.key))
1973 return NULL;
1974 return __tcp_md5_do_lookup(sk, l3index: 0, addr, family, any_l3index: true);
1975}
1976
1977#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1978void tcp_md5_destruct_sock(struct sock *sk);
1979#else
1980static inline struct tcp_md5sig_key *
1981tcp_md5_do_lookup(const struct sock *sk, int l3index,
1982 const union tcp_md5_addr *addr, int family)
1983{
1984 return NULL;
1985}
1986
1987static inline struct tcp_md5sig_key *
1988tcp_md5_do_lookup_any_l3index(const struct sock *sk,
1989 const union tcp_md5_addr *addr, int family)
1990{
1991 return NULL;
1992}
1993
1994#define tcp_twsk_md5_key(twsk) NULL
1995static inline void tcp_md5_destruct_sock(struct sock *sk)
1996{
1997}
1998#endif
1999
2000int tcp_md5_alloc_sigpool(void);
2001void tcp_md5_release_sigpool(void);
2002void tcp_md5_add_sigpool(void);
2003extern int tcp_md5_sigpool_id;
2004
2005int tcp_md5_hash_key(struct tcp_sigpool *hp,
2006 const struct tcp_md5sig_key *key);
2007
2008/* From tcp_fastopen.c */
2009void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
2010 struct tcp_fastopen_cookie *cookie);
2011void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
2012 struct tcp_fastopen_cookie *cookie, bool syn_lost,
2013 u16 try_exp);
2014struct tcp_fastopen_request {
2015 /* Fast Open cookie. Size 0 means a cookie request */
2016 struct tcp_fastopen_cookie cookie;
2017 struct msghdr *data; /* data in MSG_FASTOPEN */
2018 size_t size;
2019 int copied; /* queued in tcp_connect() */
2020 struct ubuf_info *uarg;
2021};
2022void tcp_free_fastopen_req(struct tcp_sock *tp);
2023void tcp_fastopen_destroy_cipher(struct sock *sk);
2024void tcp_fastopen_ctx_destroy(struct net *net);
2025int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
2026 void *primary_key, void *backup_key);
2027int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
2028 u64 *key);
2029void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
2030struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
2031 struct request_sock *req,
2032 struct tcp_fastopen_cookie *foc,
2033 const struct dst_entry *dst);
2034void tcp_fastopen_init_key_once(struct net *net);
2035bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
2036 struct tcp_fastopen_cookie *cookie);
2037bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
2038#define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
2039#define TCP_FASTOPEN_KEY_MAX 2
2040#define TCP_FASTOPEN_KEY_BUF_LENGTH \
2041 (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
2042
2043/* Fastopen key context */
2044struct tcp_fastopen_context {
2045 siphash_key_t key[TCP_FASTOPEN_KEY_MAX];
2046 int num;
2047 struct rcu_head rcu;
2048};
2049
2050void tcp_fastopen_active_disable(struct sock *sk);
2051bool tcp_fastopen_active_should_disable(struct sock *sk);
2052void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
2053void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
2054
2055/* Caller needs to wrap with rcu_read_(un)lock() */
2056static inline
2057struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
2058{
2059 struct tcp_fastopen_context *ctx;
2060
2061 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
2062 if (!ctx)
2063 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
2064 return ctx;
2065}
2066
2067static inline
2068bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
2069 const struct tcp_fastopen_cookie *orig)
2070{
2071 if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
2072 orig->len == foc->len &&
2073 !memcmp(orig->val, foc->val, foc->len))
2074 return true;
2075 return false;
2076}
2077
2078static inline
2079int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
2080{
2081 return ctx->num;
2082}
2083
2084/* Latencies incurred by various limits for a sender. They are
2085 * chronograph-like stats that are mutually exclusive.
2086 */
2087enum tcp_chrono {
2088 TCP_CHRONO_UNSPEC,
2089 TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
2090 TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */
2091 TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */
2092 __TCP_CHRONO_MAX,
2093};
2094
2095void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
2096void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
2097
2098/* This helper is needed, because skb->tcp_tsorted_anchor uses
2099 * the same memory storage than skb->destructor/_skb_refdst
2100 */
2101static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
2102{
2103 skb->destructor = NULL;
2104 skb->_skb_refdst = 0UL;
2105}
2106
2107#define tcp_skb_tsorted_save(skb) { \
2108 unsigned long _save = skb->_skb_refdst; \
2109 skb->_skb_refdst = 0UL;
2110
2111#define tcp_skb_tsorted_restore(skb) \
2112 skb->_skb_refdst = _save; \
2113}
2114
2115void tcp_write_queue_purge(struct sock *sk);
2116
2117static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
2118{
2119 return skb_rb_first(&sk->tcp_rtx_queue);
2120}
2121
2122static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
2123{
2124 return skb_rb_last(&sk->tcp_rtx_queue);
2125}
2126
2127static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
2128{
2129 return skb_peek_tail(list_: &sk->sk_write_queue);
2130}
2131
2132#define tcp_for_write_queue_from_safe(skb, tmp, sk) \
2133 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
2134
2135static inline struct sk_buff *tcp_send_head(const struct sock *sk)
2136{
2137 return skb_peek(list_: &sk->sk_write_queue);
2138}
2139
2140static inline bool tcp_skb_is_last(const struct sock *sk,
2141 const struct sk_buff *skb)
2142{
2143 return skb_queue_is_last(list: &sk->sk_write_queue, skb);
2144}
2145
2146/**
2147 * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
2148 * @sk: socket
2149 *
2150 * Since the write queue can have a temporary empty skb in it,
2151 * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
2152 */
2153static inline bool tcp_write_queue_empty(const struct sock *sk)
2154{
2155 const struct tcp_sock *tp = tcp_sk(sk);
2156
2157 return tp->write_seq == tp->snd_nxt;
2158}
2159
2160static inline bool tcp_rtx_queue_empty(const struct sock *sk)
2161{
2162 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
2163}
2164
2165static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
2166{
2167 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
2168}
2169
2170static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
2171{
2172 __skb_queue_tail(list: &sk->sk_write_queue, newsk: skb);
2173
2174 /* Queue it, remembering where we must start sending. */
2175 if (sk->sk_write_queue.next == skb)
2176 tcp_chrono_start(sk, type: TCP_CHRONO_BUSY);
2177}
2178
2179/* Insert new before skb on the write queue of sk. */
2180static inline void tcp_insert_write_queue_before(struct sk_buff *new,
2181 struct sk_buff *skb,
2182 struct sock *sk)
2183{
2184 __skb_queue_before(list: &sk->sk_write_queue, next: skb, newsk: new);
2185}
2186
2187static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
2188{
2189 tcp_skb_tsorted_anchor_cleanup(skb);
2190 __skb_unlink(skb, list: &sk->sk_write_queue);
2191}
2192
2193void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
2194
2195static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
2196{
2197 tcp_skb_tsorted_anchor_cleanup(skb);
2198 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
2199}
2200
2201static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
2202{
2203 list_del(entry: &skb->tcp_tsorted_anchor);
2204 tcp_rtx_queue_unlink(skb, sk);
2205 tcp_wmem_free_skb(sk, skb);
2206}
2207
2208static inline void tcp_write_collapse_fence(struct sock *sk)
2209{
2210 struct sk_buff *skb = tcp_write_queue_tail(sk);
2211
2212 if (skb)
2213 TCP_SKB_CB(skb)->eor = 1;
2214}
2215
2216static inline void tcp_push_pending_frames(struct sock *sk)
2217{
2218 if (tcp_send_head(sk)) {
2219 struct tcp_sock *tp = tcp_sk(sk);
2220
2221 __tcp_push_pending_frames(sk, cur_mss: tcp_current_mss(sk), nonagle: tp->nonagle);
2222 }
2223}
2224
2225/* Start sequence of the skb just after the highest skb with SACKed
2226 * bit, valid only if sacked_out > 0 or when the caller has ensured
2227 * validity by itself.
2228 */
2229static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
2230{
2231 if (!tp->sacked_out)
2232 return tp->snd_una;
2233
2234 if (tp->highest_sack == NULL)
2235 return tp->snd_nxt;
2236
2237 return TCP_SKB_CB(tp->highest_sack)->seq;
2238}
2239
2240static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
2241{
2242 tcp_sk(sk)->highest_sack = skb_rb_next(skb);
2243}
2244
2245static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
2246{
2247 return tcp_sk(sk)->highest_sack;
2248}
2249
2250static inline void tcp_highest_sack_reset(struct sock *sk)
2251{
2252 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
2253}
2254
2255/* Called when old skb is about to be deleted and replaced by new skb */
2256static inline void tcp_highest_sack_replace(struct sock *sk,
2257 struct sk_buff *old,
2258 struct sk_buff *new)
2259{
2260 if (old == tcp_highest_sack(sk))
2261 tcp_sk(sk)->highest_sack = new;
2262}
2263
2264/* This helper checks if socket has IP_TRANSPARENT set */
2265static inline bool inet_sk_transparent(const struct sock *sk)
2266{
2267 switch (sk->sk_state) {
2268 case TCP_TIME_WAIT:
2269 return inet_twsk(sk)->tw_transparent;
2270 case TCP_NEW_SYN_RECV:
2271 return inet_rsk(sk: inet_reqsk(sk))->no_srccheck;
2272 }
2273 return inet_test_bit(TRANSPARENT, sk);
2274}
2275
2276/* Determines whether this is a thin stream (which may suffer from
2277 * increased latency). Used to trigger latency-reducing mechanisms.
2278 */
2279static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
2280{
2281 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
2282}
2283
2284/* /proc */
2285enum tcp_seq_states {
2286 TCP_SEQ_STATE_LISTENING,
2287 TCP_SEQ_STATE_ESTABLISHED,
2288};
2289
2290void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
2291void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
2292void tcp_seq_stop(struct seq_file *seq, void *v);
2293
2294struct tcp_seq_afinfo {
2295 sa_family_t family;
2296};
2297
2298struct tcp_iter_state {
2299 struct seq_net_private p;
2300 enum tcp_seq_states state;
2301 struct sock *syn_wait_sk;
2302 int bucket, offset, sbucket, num;
2303 loff_t last_pos;
2304};
2305
2306extern struct request_sock_ops tcp_request_sock_ops;
2307extern struct request_sock_ops tcp6_request_sock_ops;
2308
2309void tcp_v4_destroy_sock(struct sock *sk);
2310
2311struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
2312 netdev_features_t features);
2313struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb);
2314struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th);
2315struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
2316 struct tcphdr *th);
2317INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
2318INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
2319INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
2320INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb));
2321#ifdef CONFIG_INET
2322void tcp_gro_complete(struct sk_buff *skb);
2323#else
2324static inline void tcp_gro_complete(struct sk_buff *skb) { }
2325#endif
2326
2327void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
2328
2329static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
2330{
2331 struct net *net = sock_net(sk: (struct sock *)tp);
2332 u32 val;
2333
2334 val = READ_ONCE(tp->notsent_lowat);
2335
2336 return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);
2337}
2338
2339bool tcp_stream_memory_free(const struct sock *sk, int wake);
2340
2341#ifdef CONFIG_PROC_FS
2342int tcp4_proc_init(void);
2343void tcp4_proc_exit(void);
2344#endif
2345
2346int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
2347int tcp_conn_request(struct request_sock_ops *rsk_ops,
2348 const struct tcp_request_sock_ops *af_ops,
2349 struct sock *sk, struct sk_buff *skb);
2350
2351/* TCP af-specific functions */
2352struct tcp_sock_af_ops {
2353#ifdef CONFIG_TCP_MD5SIG
2354 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
2355 const struct sock *addr_sk);
2356 int (*calc_md5_hash)(char *location,
2357 const struct tcp_md5sig_key *md5,
2358 const struct sock *sk,
2359 const struct sk_buff *skb);
2360 int (*md5_parse)(struct sock *sk,
2361 int optname,
2362 sockptr_t optval,
2363 int optlen);
2364#endif
2365#ifdef CONFIG_TCP_AO
2366 int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen);
2367 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2368 struct sock *addr_sk,
2369 int sndid, int rcvid);
2370 int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
2371 const struct sock *sk,
2372 __be32 sisn, __be32 disn, bool send);
2373 int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao,
2374 const struct sock *sk, const struct sk_buff *skb,
2375 const u8 *tkey, int hash_offset, u32 sne);
2376#endif
2377};
2378
2379struct tcp_request_sock_ops {
2380 u16 mss_clamp;
2381#ifdef CONFIG_TCP_MD5SIG
2382 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2383 const struct sock *addr_sk);
2384 int (*calc_md5_hash) (char *location,
2385 const struct tcp_md5sig_key *md5,
2386 const struct sock *sk,
2387 const struct sk_buff *skb);
2388#endif
2389#ifdef CONFIG_TCP_AO
2390 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2391 struct request_sock *req,
2392 int sndid, int rcvid);
2393 int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
2394 int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt,
2395 struct request_sock *req, const struct sk_buff *skb,
2396 int hash_offset, u32 sne);
2397#endif
2398#ifdef CONFIG_SYN_COOKIES
2399 __u32 (*cookie_init_seq)(const struct sk_buff *skb,
2400 __u16 *mss);
2401#endif
2402 struct dst_entry *(*route_req)(const struct sock *sk,
2403 struct sk_buff *skb,
2404 struct flowi *fl,
2405 struct request_sock *req,
2406 u32 tw_isn);
2407 u32 (*init_seq)(const struct sk_buff *skb);
2408 u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
2409 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2410 struct flowi *fl, struct request_sock *req,
2411 struct tcp_fastopen_cookie *foc,
2412 enum tcp_synack_type synack_type,
2413 struct sk_buff *syn_skb);
2414};
2415
2416extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
2417#if IS_ENABLED(CONFIG_IPV6)
2418extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
2419#endif
2420
2421#ifdef CONFIG_SYN_COOKIES
2422static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2423 const struct sock *sk, struct sk_buff *skb,
2424 __u16 *mss)
2425{
2426 tcp_synq_overflow(sk);
2427 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2428 return ops->cookie_init_seq(skb, mss);
2429}
2430#else
2431static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2432 const struct sock *sk, struct sk_buff *skb,
2433 __u16 *mss)
2434{
2435 return 0;
2436}
2437#endif
2438
2439struct tcp_key {
2440 union {
2441 struct {
2442 struct tcp_ao_key *ao_key;
2443 char *traffic_key;
2444 u32 sne;
2445 u8 rcv_next;
2446 };
2447 struct tcp_md5sig_key *md5_key;
2448 };
2449 enum {
2450 TCP_KEY_NONE = 0,
2451 TCP_KEY_MD5,
2452 TCP_KEY_AO,
2453 } type;
2454};
2455
2456static inline void tcp_get_current_key(const struct sock *sk,
2457 struct tcp_key *out)
2458{
2459#if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG)
2460 const struct tcp_sock *tp = tcp_sk(sk);
2461#endif
2462
2463#ifdef CONFIG_TCP_AO
2464 if (static_branch_unlikely(&tcp_ao_needed.key)) {
2465 struct tcp_ao_info *ao;
2466
2467 ao = rcu_dereference_protected(tp->ao_info,
2468 lockdep_sock_is_held(sk));
2469 if (ao) {
2470 out->ao_key = READ_ONCE(ao->current_key);
2471 out->type = TCP_KEY_AO;
2472 return;
2473 }
2474 }
2475#endif
2476#ifdef CONFIG_TCP_MD5SIG
2477 if (static_branch_unlikely(&tcp_md5_needed.key) &&
2478 rcu_access_pointer(tp->md5sig_info)) {
2479 out->md5_key = tp->af_specific->md5_lookup(sk, sk);
2480 if (out->md5_key) {
2481 out->type = TCP_KEY_MD5;
2482 return;
2483 }
2484 }
2485#endif
2486 out->type = TCP_KEY_NONE;
2487}
2488
2489static inline bool tcp_key_is_md5(const struct tcp_key *key)
2490{
2491 if (static_branch_tcp_md5())
2492 return key->type == TCP_KEY_MD5;
2493 return false;
2494}
2495
2496static inline bool tcp_key_is_ao(const struct tcp_key *key)
2497{
2498 if (static_branch_tcp_ao())
2499 return key->type == TCP_KEY_AO;
2500 return false;
2501}
2502
2503int tcpv4_offload_init(void);
2504
2505void tcp_v4_init(void);
2506void tcp_init(void);
2507
2508/* tcp_recovery.c */
2509void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2510void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2511extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2512 u32 reo_wnd);
2513extern bool tcp_rack_mark_lost(struct sock *sk);
2514extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2515 u64 xmit_time);
2516extern void tcp_rack_reo_timeout(struct sock *sk);
2517extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2518
2519/* tcp_plb.c */
2520
2521/*
2522 * Scaling factor for fractions in PLB. For example, tcp_plb_update_state
2523 * expects cong_ratio which represents fraction of traffic that experienced
2524 * congestion over a single RTT. In order to avoid floating point operations,
2525 * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in.
2526 */
2527#define TCP_PLB_SCALE 8
2528
2529/* State for PLB (Protective Load Balancing) for a single TCP connection. */
2530struct tcp_plb_state {
2531 u8 consec_cong_rounds:5, /* consecutive congested rounds */
2532 unused:3;
2533 u32 pause_until; /* jiffies32 when PLB can resume rerouting */
2534};
2535
2536static inline void tcp_plb_init(const struct sock *sk,
2537 struct tcp_plb_state *plb)
2538{
2539 plb->consec_cong_rounds = 0;
2540 plb->pause_until = 0;
2541}
2542void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
2543 const int cong_ratio);
2544void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
2545void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
2546
2547static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str)
2548{
2549 WARN_ONCE(cond,
2550 "%scwn:%u out:%u sacked:%u lost:%u retrans:%u tlp_high_seq:%u sk_state:%u ca_state:%u advmss:%u mss_cache:%u pmtu:%u\n",
2551 str,
2552 tcp_snd_cwnd(tcp_sk(sk)),
2553 tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out,
2554 tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out,
2555 tcp_sk(sk)->tlp_high_seq, sk->sk_state,
2556 inet_csk(sk)->icsk_ca_state,
2557 tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache,
2558 inet_csk(sk)->icsk_pmtu_cookie);
2559}
2560
2561/* At how many usecs into the future should the RTO fire? */
2562static inline s64 tcp_rto_delta_us(const struct sock *sk)
2563{
2564 const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2565 u32 rto = inet_csk(sk)->icsk_rto;
2566
2567 if (likely(skb)) {
2568 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(j: rto);
2569
2570 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2571 } else {
2572 tcp_warn_once(sk, cond: 1, str: "rtx queue empty: ");
2573 return jiffies_to_usecs(j: rto);
2574 }
2575
2576}
2577
2578/*
2579 * Save and compile IPv4 options, return a pointer to it
2580 */
2581static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2582 struct sk_buff *skb)
2583{
2584 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2585 struct ip_options_rcu *dopt = NULL;
2586
2587 if (opt->optlen) {
2588 int opt_size = sizeof(*dopt) + opt->optlen;
2589
2590 dopt = kmalloc(opt_size, GFP_ATOMIC);
2591 if (dopt && __ip_options_echo(net, dopt: &dopt->opt, skb, sopt: opt)) {
2592 kfree(objp: dopt);
2593 dopt = NULL;
2594 }
2595 }
2596 return dopt;
2597}
2598
2599/* locally generated TCP pure ACKs have skb->truesize == 2
2600 * (check tcp_send_ack() in net/ipv4/tcp_output.c )
2601 * This is much faster than dissecting the packet to find out.
2602 * (Think of GRE encapsulations, IPv4, IPv6, ...)
2603 */
2604static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2605{
2606 return skb->truesize == 2;
2607}
2608
2609static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2610{
2611 skb->truesize = 2;
2612}
2613
2614static inline int tcp_inq(struct sock *sk)
2615{
2616 struct tcp_sock *tp = tcp_sk(sk);
2617 int answ;
2618
2619 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2620 answ = 0;
2621 } else if (sock_flag(sk, flag: SOCK_URGINLINE) ||
2622 !tp->urg_data ||
2623 before(seq1: tp->urg_seq, seq2: tp->copied_seq) ||
2624 !before(seq1: tp->urg_seq, seq2: tp->rcv_nxt)) {
2625
2626 answ = tp->rcv_nxt - tp->copied_seq;
2627
2628 /* Subtract 1, if FIN was received */
2629 if (answ && sock_flag(sk, flag: SOCK_DONE))
2630 answ--;
2631 } else {
2632 answ = tp->urg_seq - tp->copied_seq;
2633 }
2634
2635 return answ;
2636}
2637
2638int tcp_peek_len(struct socket *sock);
2639
2640static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2641{
2642 u16 segs_in;
2643
2644 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2645
2646 /* We update these fields while other threads might
2647 * read them from tcp_get_info()
2648 */
2649 WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);
2650 if (skb->len > tcp_hdrlen(skb))
2651 WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);
2652}
2653
2654/*
2655 * TCP listen path runs lockless.
2656 * We forced "struct sock" to be const qualified to make sure
2657 * we don't modify one of its field by mistake.
2658 * Here, we increment sk_drops which is an atomic_t, so we can safely
2659 * make sock writable again.
2660 */
2661static inline void tcp_listendrop(const struct sock *sk)
2662{
2663 sk_drops_inc(sk: (struct sock *)sk);
2664 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2665}
2666
2667enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2668
2669/*
2670 * Interface for adding Upper Level Protocols over TCP
2671 */
2672
2673#define TCP_ULP_NAME_MAX 16
2674#define TCP_ULP_MAX 128
2675#define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2676
2677struct tcp_ulp_ops {
2678 struct list_head list;
2679
2680 /* initialize ulp */
2681 int (*init)(struct sock *sk);
2682 /* update ulp */
2683 void (*update)(struct sock *sk, struct proto *p,
2684 void (*write_space)(struct sock *sk));
2685 /* cleanup ulp */
2686 void (*release)(struct sock *sk);
2687 /* diagnostic */
2688 int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin);
2689 size_t (*get_info_size)(const struct sock *sk, bool net_admin);
2690 /* clone ulp */
2691 void (*clone)(const struct request_sock *req, struct sock *newsk,
2692 const gfp_t priority);
2693
2694 char name[TCP_ULP_NAME_MAX];
2695 struct module *owner;
2696};
2697int tcp_register_ulp(struct tcp_ulp_ops *type);
2698void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2699int tcp_set_ulp(struct sock *sk, const char *name);
2700void tcp_get_available_ulp(char *buf, size_t len);
2701void tcp_cleanup_ulp(struct sock *sk);
2702void tcp_update_ulp(struct sock *sk, struct proto *p,
2703 void (*write_space)(struct sock *sk));
2704
2705#define MODULE_ALIAS_TCP_ULP(name) \
2706 MODULE_INFO(alias, name); \
2707 MODULE_INFO(alias, "tcp-ulp-" name)
2708
2709#ifdef CONFIG_NET_SOCK_MSG
2710struct sk_msg;
2711struct sk_psock;
2712
2713#ifdef CONFIG_BPF_SYSCALL
2714int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2715void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2716#ifdef CONFIG_BPF_STREAM_PARSER
2717struct strparser;
2718int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc,
2719 sk_read_actor_t recv_actor);
2720#endif /* CONFIG_BPF_STREAM_PARSER */
2721#endif /* CONFIG_BPF_SYSCALL */
2722
2723#ifdef CONFIG_INET
2724void tcp_eat_skb(struct sock *sk, struct sk_buff *skb);
2725#else
2726static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb)
2727{
2728}
2729#endif
2730
2731int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
2732 struct sk_msg *msg, u32 bytes, int flags);
2733#endif /* CONFIG_NET_SOCK_MSG */
2734
2735#if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)
2736static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
2737{
2738}
2739#endif
2740
2741#ifdef CONFIG_CGROUP_BPF
2742static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2743 struct sk_buff *skb,
2744 unsigned int end_offset)
2745{
2746 skops->skb = skb;
2747 skops->skb_data_end = skb->data + end_offset;
2748}
2749#else
2750static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2751 struct sk_buff *skb,
2752 unsigned int end_offset)
2753{
2754}
2755#endif
2756
2757/* Call BPF_SOCK_OPS program that returns an int. If the return value
2758 * is < 0, then the BPF op failed (for example if the loaded BPF
2759 * program does not support the chosen operation or there is no BPF
2760 * program loaded).
2761 */
2762#ifdef CONFIG_BPF
2763static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2764{
2765 struct bpf_sock_ops_kern sock_ops;
2766 int ret;
2767
2768 memset(s: &sock_ops, c: 0, offsetof(struct bpf_sock_ops_kern, temp));
2769 if (sk_fullsock(sk)) {
2770 sock_ops.is_fullsock = 1;
2771 sock_ops.is_locked_tcp_sock = 1;
2772 sock_owned_by_me(sk);
2773 }
2774
2775 sock_ops.sk = sk;
2776 sock_ops.op = op;
2777 if (nargs > 0)
2778 memcpy(to: sock_ops.args, from: args, len: nargs * sizeof(*args));
2779
2780 ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2781 if (ret == 0)
2782 ret = sock_ops.reply;
2783 else
2784 ret = -1;
2785 return ret;
2786}
2787
2788static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2789{
2790 u32 args[2] = {arg1, arg2};
2791
2792 return tcp_call_bpf(sk, op, nargs: 2, args);
2793}
2794
2795static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2796 u32 arg3)
2797{
2798 u32 args[3] = {arg1, arg2, arg3};
2799
2800 return tcp_call_bpf(sk, op, nargs: 3, args);
2801}
2802
2803#else
2804static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2805{
2806 return -EPERM;
2807}
2808
2809static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2810{
2811 return -EPERM;
2812}
2813
2814static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2815 u32 arg3)
2816{
2817 return -EPERM;
2818}
2819
2820#endif
2821
2822static inline u32 tcp_timeout_init(struct sock *sk)
2823{
2824 int timeout;
2825
2826 timeout = tcp_call_bpf(sk, op: BPF_SOCK_OPS_TIMEOUT_INIT, nargs: 0, NULL);
2827
2828 if (timeout <= 0)
2829 timeout = TCP_TIMEOUT_INIT;
2830 return min_t(int, timeout, TCP_RTO_MAX);
2831}
2832
2833static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2834{
2835 int rwnd;
2836
2837 rwnd = tcp_call_bpf(sk, op: BPF_SOCK_OPS_RWND_INIT, nargs: 0, NULL);
2838
2839 if (rwnd < 0)
2840 rwnd = 0;
2841 return rwnd;
2842}
2843
2844static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2845{
2846 return (tcp_call_bpf(sk, op: BPF_SOCK_OPS_NEEDS_ECN, nargs: 0, NULL) == 1);
2847}
2848
2849static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
2850{
2851 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2852 tcp_call_bpf_2arg(sk, op: BPF_SOCK_OPS_RTT_CB, arg1: mrtt, arg2: srtt);
2853}
2854
2855#if IS_ENABLED(CONFIG_SMC)
2856extern struct static_key_false tcp_have_smc;
2857#endif
2858
2859#if IS_ENABLED(CONFIG_TLS_DEVICE)
2860void clean_acked_data_enable(struct tcp_sock *tp,
2861 void (*cad)(struct sock *sk, u32 ack_seq));
2862void clean_acked_data_disable(struct tcp_sock *tp);
2863void clean_acked_data_flush(void);
2864#endif
2865
2866DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2867static inline void tcp_add_tx_delay(struct sk_buff *skb,
2868 const struct tcp_sock *tp)
2869{
2870 if (static_branch_unlikely(&tcp_tx_delay_enabled))
2871 skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2872}
2873
2874/* Compute Earliest Departure Time for some control packets
2875 * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets.
2876 */
2877static inline u64 tcp_transmit_time(const struct sock *sk)
2878{
2879 if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2880 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2881 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2882
2883 return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2884 }
2885 return 0;
2886}
2887
2888static inline int tcp_parse_auth_options(const struct tcphdr *th,
2889 const u8 **md5_hash, const struct tcp_ao_hdr **aoh)
2890{
2891 const u8 *md5_tmp, *ao_tmp;
2892 int ret;
2893
2894 ret = tcp_do_parse_auth_options(th, md5_hash: &md5_tmp, ao_hash: &ao_tmp);
2895 if (ret)
2896 return ret;
2897
2898 if (md5_hash)
2899 *md5_hash = md5_tmp;
2900
2901 if (aoh) {
2902 if (!ao_tmp)
2903 *aoh = NULL;
2904 else
2905 *aoh = (struct tcp_ao_hdr *)(ao_tmp - 2);
2906 }
2907
2908 return 0;
2909}
2910
2911static inline bool tcp_ao_required(struct sock *sk, const void *saddr,
2912 int family, int l3index, bool stat_inc)
2913{
2914#ifdef CONFIG_TCP_AO
2915 struct tcp_ao_info *ao_info;
2916 struct tcp_ao_key *ao_key;
2917
2918 if (!static_branch_unlikely(&tcp_ao_needed.key))
2919 return false;
2920
2921 ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info,
2922 lockdep_sock_is_held(sk));
2923 if (!ao_info)
2924 return false;
2925
2926 ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1);
2927 if (ao_info->ao_required || ao_key) {
2928 if (stat_inc) {
2929 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED);
2930 atomic64_inc(&ao_info->counters.ao_required);
2931 }
2932 return true;
2933 }
2934#endif
2935 return false;
2936}
2937
2938enum skb_drop_reason tcp_inbound_hash(struct sock *sk,
2939 const struct request_sock *req, const struct sk_buff *skb,
2940 const void *saddr, const void *daddr,
2941 int family, int dif, int sdif);
2942
2943#endif /* _TCP_H */
2944