xfrm_policy.c source code [Linux/net/xfrm/xfrm_policy.c]

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* xfrm_policy.c
4	*
5	* Changes:
6	* Mitsuru KANDA @USAGI
7	* Kazunori MIYAZAWA @USAGI
8	* Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9	* IPv6 support
10	* Kazunori MIYAZAWA @USAGI
11	* YOSHIFUJI Hideaki
12	* Split up af-specific portion
13	* Derek Atkins <derek@ihtfp.com> Add the post_input processor
14	*
15	*/
16
17	#include <linux/err.h>
18	#include <linux/slab.h>
19	#include <linux/kmod.h>
20	#include <linux/list.h>
21	#include <linux/spinlock.h>
22	#include <linux/workqueue.h>
23	#include <linux/notifier.h>
24	#include <linux/netdevice.h>
25	#include <linux/netfilter.h>
26	#include <linux/module.h>
27	#include <linux/cache.h>
28	#include <linux/cpu.h>
29	#include <linux/audit.h>
30	#include <linux/rhashtable.h>
31	#include <linux/if_tunnel.h>
32	#include <linux/icmp.h>
33	#include <net/dst.h>
34	#include <net/flow.h>
35	#include <net/inet_ecn.h>
36	#include <net/xfrm.h>
37	#include <net/ip.h>
38	#include <net/gre.h>
39	#if IS_ENABLED(CONFIG_IPV6_MIP6)
40	#include <net/mip6.h>
41	#endif
42	#ifdef CONFIG_XFRM_STATISTICS
43	#include <net/snmp.h>
44	#endif
45	#ifdef CONFIG_XFRM_ESPINTCP
46	#include <net/espintcp.h>
47	#endif
48	#include <net/inet_dscp.h>
49
50	#include "xfrm_hash.h"
51
52	#define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
53	#define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
54	#define XFRM_MAX_QUEUE_LEN 100
55
56	struct xfrm_flo {
57	struct dst_entry *dst_orig;
58	u8 flags;
59	};
60
61	/ prefixes smaller than this are stored in lists, not trees. /
62	#define INEXACT_PREFIXLEN_IPV4 16
63	#define INEXACT_PREFIXLEN_IPV6 48
64
65	struct xfrm_pol_inexact_node {
66	struct rb_node node;
67	union {
68	xfrm_address_t addr;
69	struct rcu_head rcu;
70	};
71	u8 prefixlen;
72
73	struct rb_root root;
74
75	/ the policies matching this node, can be empty list /
76	struct hlist_head hhead;
77	};
78
79	/ xfrm inexact policy search tree:*
80	* xfrm_pol_inexact_bin = hash(dir,type,family,if_id);
81	* \|
82	* +---- root_d: sorted by daddr:prefix
83	* \| \|
84	* \| xfrm_pol_inexact_node
85	* \| \|
86	* \| +- root: sorted by saddr/prefix
87	* \| \| \|
88	* \| \| xfrm_pol_inexact_node
89	* \| \| \|
90	* \| \| + root: unused
91	* \| \| \|
92	* \| \| + hhead: saddr:daddr policies
93	* \| \|
94	* \| +- coarse policies and all any:daddr policies
95	* \|
96	* +---- root_s: sorted by saddr:prefix
97	* \| \|
98	* \| xfrm_pol_inexact_node
99	* \| \|
100	* \| + root: unused
101	* \| \|
102	* \| + hhead: saddr:any policies
103	* \|
104	* +---- coarse policies and all any:any policies
105	*
106	* Lookups return four candidate lists:
107	* 1. any:any list from top-level xfrm_pol_inexact_bin
108	* 2. any:daddr list from daddr tree
109	* 3. saddr:daddr list from 2nd level daddr tree
110	* 4. saddr:any list from saddr tree
111	*
112	* This result set then needs to be searched for the policy with
113	* the lowest priority. If two candidates have the same priority, the
114	* struct xfrm_policy pos member with the lower number is used.
115	*
116	* This replicates previous single-list-search algorithm which would
117	* return first matching policy in the (ordered-by-priority) list.
118	*/
119
120	struct xfrm_pol_inexact_key {
121	possible_net_t net;
122	u32 if_id;
123	u16 family;
124	u8 dir, type;
125	};
126
127	struct xfrm_pol_inexact_bin {
128	struct xfrm_pol_inexact_key k;
129	struct rhash_head head;
130	/ list containing ':' policies /
131	struct hlist_head hhead;
132
133	seqcount_spinlock_t count;
134	/ tree sorted by daddr/prefix /
135	struct rb_root root_d;
136
137	/ tree sorted by saddr/prefix /
138	struct rb_root root_s;
139
140	/ slow path below /
141	struct list_head inexact_bins;
142	struct rcu_head rcu;
143	};
144
145	enum xfrm_pol_inexact_candidate_type {
146	XFRM_POL_CAND_BOTH,
147	XFRM_POL_CAND_SADDR,
148	XFRM_POL_CAND_DADDR,
149	XFRM_POL_CAND_ANY,
150
151	XFRM_POL_CAND_MAX,
152	};
153
154	struct xfrm_pol_inexact_candidates {
155	struct hlist_head *res[XFRM_POL_CAND_MAX];
156	};
157
158	struct xfrm_flow_keys {
159	struct flow_dissector_key_basic basic;
160	struct flow_dissector_key_control control;
161	union {
162	struct flow_dissector_key_ipv4_addrs ipv4;
163	struct flow_dissector_key_ipv6_addrs ipv6;
164	} addrs;
165	struct flow_dissector_key_ip ip;
166	struct flow_dissector_key_icmp icmp;
167	struct flow_dissector_key_ports ports;
168	struct flow_dissector_key_keyid gre;
169	};
170
171	static struct flow_dissector xfrm_session_dissector __ro_after_init;
172
173	static DEFINE_SPINLOCK(xfrm_if_cb_lock);
174	static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
175
176	static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
177	static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + `1`]
178	__read_mostly;
179
180	static struct kmem_cache *xfrm_dst_cache __ro_after_init;
181
182	static struct rhashtable xfrm_policy_inexact_table;
183	static const struct rhashtable_params xfrm_pol_inexact_params;
184
185	static void xfrm_init_pmtu(struct xfrm_dst *bundle, int* nr);
186	static int stale_bundle(struct dst_entry *dst);
187	static int xfrm_bundle_ok(struct xfrm_dst *xdst);
188	static void xfrm_policy_queue_process(struct timer_list *t);
189
190	static void __xfrm_policy_link(struct xfrm_policy pol, int* dir);
191	static struct xfrm_policy __xfrm_policy_unlink(struct* xfrm_policy *pol,
192	int dir);
193
194	static struct xfrm_pol_inexact_bin *
195	xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
196	u32 if_id);
197
198	static struct xfrm_pol_inexact_bin *
199	xfrm_policy_inexact_lookup_rcu(struct net *net,
200	u8 type, u16 family, u8 dir, u32 if_id);
201	static struct xfrm_policy *
202	xfrm_policy_insert_list(struct hlist_head chain, struct* xfrm_policy *policy,
203	bool excl);
204
205	static bool
206	xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
207	struct xfrm_pol_inexact_bin *b,
208	const xfrm_address_t *saddr,
209	const xfrm_address_t *daddr);
210
211	static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
212	{
213	return refcount_inc_not_zero(r: &policy->refcnt);
214	}
215
216	static inline bool
217	__xfrm4_selector_match(const struct xfrm_selector sel, const* struct flowi *fl)
218	{
219	const struct flowi4 *fl4 = &fl->u.ip4;
220
221	return addr4_match(a1: fl4->daddr, a2: sel->daddr.a4, prefixlen: sel->prefixlen_d) &&
222	addr4_match(a1: fl4->saddr, a2: sel->saddr.a4, prefixlen: sel->prefixlen_s) &&
223	!((xfrm_flowi_dport(fl, uli: &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
224	!((xfrm_flowi_sport(fl, uli: &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
225	(fl4->flowi4_proto == sel->proto \|\| !sel->proto) &&
226	(fl4->flowi4_oif == sel->ifindex \|\| !sel->ifindex);
227	}
228
229	static inline bool
230	__xfrm6_selector_match(const struct xfrm_selector sel, const* struct flowi *fl)
231	{
232	const struct flowi6 *fl6 = &fl->u.ip6;
233
234	return addr_match(token1: &fl6->daddr, token2: &sel->daddr, prefixlen: sel->prefixlen_d) &&
235	addr_match(token1: &fl6->saddr, token2: &sel->saddr, prefixlen: sel->prefixlen_s) &&
236	!((xfrm_flowi_dport(fl, uli: &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
237	!((xfrm_flowi_sport(fl, uli: &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
238	(fl6->flowi6_proto == sel->proto \|\| !sel->proto) &&
239	(fl6->flowi6_oif == sel->ifindex \|\| !sel->ifindex);
240	}
241
242	bool xfrm_selector_match(const struct xfrm_selector sel, const* struct flowi *fl,
243	unsigned short family)
244	{
245	switch (family) {
246	case AF_INET:
247	return __xfrm4_selector_match(sel, fl);
248	case AF_INET6:
249	return __xfrm6_selector_match(sel, fl);
250	}
251	return false;
252	}
253
254	static const struct xfrm_policy_afinfo xfrm_policy_get_afinfo(unsigned* short family)
255	{
256	const struct xfrm_policy_afinfo *afinfo;
257
258	if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
259	return NULL;
260	rcu_read_lock();
261	afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
262	if (unlikely(!afinfo))
263	rcu_read_unlock();
264	return afinfo;
265	}
266
267	/ Called with rcu_read_lock(). /
268	static const struct xfrm_if_cb xfrm_if_get_cb(void*)
269	{
270	return rcu_dereference(xfrm_if_cb);
271	}
272
273	struct dst_entry __xfrm_dst_lookup(int* family,
274	const struct xfrm_dst_lookup_params *params)
275	{
276	const struct xfrm_policy_afinfo *afinfo;
277	struct dst_entry *dst;
278
279	afinfo = xfrm_policy_get_afinfo(family);
280	if (unlikely(afinfo == NULL))
281	return ERR_PTR(error: -EAFNOSUPPORT);
282
283	dst = afinfo->dst_lookup(params);
284
285	rcu_read_unlock();
286
287	return dst;
288	}
289	EXPORT_SYMBOL(__xfrm_dst_lookup);
290
291	static inline struct dst_entry xfrm_dst_lookup(struct* xfrm_state *x,
292	dscp_t dscp, int oif,
293	xfrm_address_t *prev_saddr,
294	xfrm_address_t *prev_daddr,
295	int family, u32 mark)
296	{
297	struct xfrm_dst_lookup_params params;
298	struct net *net = xs_net(x);
299	xfrm_address_t *saddr = &x->props.saddr;
300	xfrm_address_t *daddr = &x->id.daddr;
301	struct dst_entry *dst;
302
303	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
304	saddr = x->coaddr;
305	daddr = prev_daddr;
306	}
307	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
308	saddr = prev_saddr;
309	daddr = x->coaddr;
310	}
311
312	params.net = net;
313	params.saddr = saddr;
314	params.daddr = daddr;
315	params.dscp = dscp;
316	params.oif = oif;
317	params.mark = mark;
318	params.ipproto = x->id.proto;
319	if (x->encap) {
320	switch (x->encap->encap_type) {
321	case UDP_ENCAP_ESPINUDP:
322	params.ipproto = IPPROTO_UDP;
323	params.uli.ports.sport = x->encap->encap_sport;
324	params.uli.ports.dport = x->encap->encap_dport;
325	break;
326	case TCP_ENCAP_ESPINTCP:
327	params.ipproto = IPPROTO_TCP;
328	params.uli.ports.sport = x->encap->encap_sport;
329	params.uli.ports.dport = x->encap->encap_dport;
330	break;
331	}
332	}
333
334	dst = __xfrm_dst_lookup(family, &params);
335
336	if (!IS_ERR(ptr: dst)) {
337	if (prev_saddr != saddr)
338	memcpy(to: prev_saddr, from: saddr, len: sizeof(*prev_saddr));
339	if (prev_daddr != daddr)
340	memcpy(to: prev_daddr, from: daddr, len: sizeof(*prev_daddr));
341	}
342
343	return dst;
344	}
345
346	static inline unsigned long make_jiffies(long secs)
347	{
348	if (secs >= (MAX_SCHEDULE_TIMEOUT-`1`)/HZ)
349	return MAX_SCHEDULE_TIMEOUT-`1`;
350	else
351	return secs*HZ;
352	}
353
354	static void xfrm_policy_timer(struct timer_list *t)
355	{
356	struct xfrm_policy *xp = timer_container_of(xp, t, timer);
357	time64_t now = ktime_get_real_seconds();
358	time64_t next = TIME64_MAX;
359	int warn = `0`;
360	int dir;
361
362	read_lock(&xp->lock);
363
364	if (unlikely(xp->walk.dead))
365	goto out;
366
367	dir = xfrm_policy_id2dir(index: xp->index);
368
369	if (xp->lft.hard_add_expires_seconds) {
370	time64_t tmo = xp->lft.hard_add_expires_seconds +
371	xp->curlft.add_time - now;
372	if (tmo <= `0`)
373	goto expired;
374	if (tmo < next)
375	next = tmo;
376	}
377	if (xp->lft.hard_use_expires_seconds) {
378	time64_t tmo = xp->lft.hard_use_expires_seconds +
379	(READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
380	if (tmo <= `0`)
381	goto expired;
382	if (tmo < next)
383	next = tmo;
384	}
385	if (xp->lft.soft_add_expires_seconds) {
386	time64_t tmo = xp->lft.soft_add_expires_seconds +
387	xp->curlft.add_time - now;
388	if (tmo <= `0`) {
389	warn = `1`;
390	tmo = XFRM_KM_TIMEOUT;
391	}
392	if (tmo < next)
393	next = tmo;
394	}
395	if (xp->lft.soft_use_expires_seconds) {
396	time64_t tmo = xp->lft.soft_use_expires_seconds +
397	(READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
398	if (tmo <= `0`) {
399	warn = `1`;
400	tmo = XFRM_KM_TIMEOUT;
401	}
402	if (tmo < next)
403	next = tmo;
404	}
405
406	if (warn)
407	km_policy_expired(pol: xp, dir, hard: `0`, portid: `0`);
408	if (next != TIME64_MAX &&
409	!mod_timer(timer: &xp->timer, expires: jiffies + make_jiffies(secs: next)))
410	xfrm_pol_hold(policy: xp);
411
412	out:
413	read_unlock(&xp->lock);
414	xfrm_pol_put(policy: xp);
415	return;
416
417	expired:
418	read_unlock(&xp->lock);
419	if (!xfrm_policy_delete(pol: xp, dir))
420	km_policy_expired(pol: xp, dir, hard: `1`, portid: `0`);
421	xfrm_pol_put(policy: xp);
422	}
423
424	/ Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2*
425	* SPD calls.
426	*/
427
428	struct xfrm_policy xfrm_policy_alloc(struct* net *net, gfp_t gfp)
429	{
430	struct xfrm_policy *policy;
431
432	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
433
434	if (policy) {
435	write_pnet(pnet: &policy->xp_net, net);
436	INIT_LIST_HEAD(list: &policy->walk.all);
437	INIT_HLIST_HEAD(&policy->state_cache_list);
438	INIT_HLIST_NODE(h: &policy->bydst);
439	INIT_HLIST_NODE(h: &policy->byidx);
440	rwlock_init(&policy->lock);
441	refcount_set(r: &policy->refcnt, n: `1`);
442	skb_queue_head_init(list: &policy->polq.hold_queue);
443	timer_setup(&policy->timer, xfrm_policy_timer, `0`);
444	timer_setup(&policy->polq.hold_timer,
445	xfrm_policy_queue_process, `0`);
446	}
447	return policy;
448	}
449	EXPORT_SYMBOL(xfrm_policy_alloc);
450
451	static void xfrm_policy_destroy_rcu(struct rcu_head *head)
452	{
453	struct xfrm_policy policy = container_of(head, struct* xfrm_policy, rcu);
454
455	security_xfrm_policy_free(ctx: policy->security);
456	kfree(objp: policy);
457	}
458
459	/ Destroy xfrm_policy: descendant resources must be released to this moment. /
460
461	void xfrm_policy_destroy(struct xfrm_policy *policy)
462	{
463	BUG_ON(!policy->walk.dead);
464
465	if (timer_delete(timer: &policy->timer) \|\| timer_delete(timer: &policy->polq.hold_timer))
466	BUG();
467
468	xfrm_dev_policy_free(x: policy);
469	call_rcu(head: &policy->rcu, func: xfrm_policy_destroy_rcu);
470	}
471	EXPORT_SYMBOL(xfrm_policy_destroy);
472
473	/ Rule must be locked. Release descendant resources, announce*
474	* entry dead. The rule must be unlinked from lists to the moment.
475	*/
476
477	static void xfrm_policy_kill(struct xfrm_policy *policy)
478	{
479	struct net *net = xp_net(xp: policy);
480	struct xfrm_state *x;
481
482	xfrm_dev_policy_delete(x: policy);
483
484	write_lock_bh(&policy->lock);
485	policy->walk.dead = `1`;
486	write_unlock_bh(&policy->lock);
487
488	atomic_inc(v: &policy->genid);
489
490	if (timer_delete(timer: &policy->polq.hold_timer))
491	xfrm_pol_put(policy);
492	skb_queue_purge(list: &policy->polq.hold_queue);
493
494	if (timer_delete(timer: &policy->timer))
495	xfrm_pol_put(policy);
496
497	/ XXX: Flush state cache /
498	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
499	hlist_for_each_entry_rcu(x, &policy->state_cache_list, state_cache) {
500	hlist_del_init_rcu(n: &x->state_cache);
501	}
502	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
503
504	xfrm_pol_put(policy);
505	}
506
507	static unsigned int xfrm_policy_hashmax __read_mostly = `1` * `1024` * `1024`;
508
509	static inline unsigned int idx_hash(struct net *net, u32 index)
510	{
511	return __idx_hash(index, hmask: net->xfrm.policy_idx_hmask);
512	}
513
514	/ calculate policy hash thresholds /
515	static void __get_hash_thresh(struct net *net,
516	unsigned short family, int dir,
517	u8 dbits, u8 sbits)
518	{
519	switch (family) {
520	case AF_INET:
521	*dbits = net->xfrm.policy_bydst[dir].dbits4;
522	*sbits = net->xfrm.policy_bydst[dir].sbits4;
523	break;
524
525	case AF_INET6:
526	*dbits = net->xfrm.policy_bydst[dir].dbits6;
527	*sbits = net->xfrm.policy_bydst[dir].sbits6;
528	break;
529
530	default:
531	*dbits = `0`;
532	*sbits = `0`;
533	}
534	}
535
536	static struct hlist_head policy_hash_bysel(struct* net *net,
537	const struct xfrm_selector *sel,
538	unsigned short family, int dir)
539	{
540	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
541	unsigned int hash;
542	u8 dbits;
543	u8 sbits;
544
545	__get_hash_thresh(net, family, dir, dbits: &dbits, sbits: &sbits);
546	hash = __sel_hash(sel, family, hmask, dbits, sbits);
547
548	if (hash == hmask + `1`)
549	return NULL;
550
551	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
552	lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
553	}
554
555	static struct hlist_head policy_hash_direct(struct* net *net,
556	const xfrm_address_t *daddr,
557	const xfrm_address_t *saddr,
558	unsigned short family, int dir)
559	{
560	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
561	unsigned int hash;
562	u8 dbits;
563	u8 sbits;
564
565	__get_hash_thresh(net, family, dir, dbits: &dbits, sbits: &sbits);
566	hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
567
568	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
569	lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
570	}
571
572	static void xfrm_dst_hash_transfer(struct net *net,
573	struct hlist_head *list,
574	struct hlist_head *ndsttable,
575	unsigned int nhashmask,
576	int dir)
577	{
578	struct hlist_node tmp, entry0 = NULL;
579	struct xfrm_policy *pol;
580	unsigned int h0 = `0`;
581	u8 dbits;
582	u8 sbits;
583
584	redo:
585	hlist_for_each_entry_safe(pol, tmp, list, bydst) {
586	unsigned int h;
587
588	__get_hash_thresh(net, family: pol->family, dir, dbits: &dbits, sbits: &sbits);
589	h = __addr_hash(daddr: &pol->selector.daddr, saddr: &pol->selector.saddr,
590	family: pol->family, hmask: nhashmask, dbits, sbits);
591	if (!entry0 \|\| pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
592	hlist_del_rcu(n: &pol->bydst);
593	hlist_add_head_rcu(n: &pol->bydst, h: ndsttable + h);
594	h0 = h;
595	} else {
596	if (h != h0)
597	continue;
598	hlist_del_rcu(n: &pol->bydst);
599	hlist_add_behind_rcu(n: &pol->bydst, prev: entry0);
600	}
601	entry0 = &pol->bydst;
602	}
603	if (!hlist_empty(h: list)) {
604	entry0 = NULL;
605	goto redo;
606	}
607	}
608
609	static void xfrm_idx_hash_transfer(struct hlist_head *list,
610	struct hlist_head *nidxtable,
611	unsigned int nhashmask)
612	{
613	struct hlist_node *tmp;
614	struct xfrm_policy *pol;
615
616	hlist_for_each_entry_safe(pol, tmp, list, byidx) {
617	unsigned int h;
618
619	h = __idx_hash(index: pol->index, hmask: nhashmask);
620	hlist_add_head(n: &pol->byidx, h: nidxtable+h);
621	}
622	}
623
624	static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
625	{
626	return ((old_hmask + `1`) << `1`) - `1`;
627	}
628
629	static void xfrm_bydst_resize(struct net net, int* dir)
630	{
631	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
632	unsigned int nhashmask = xfrm_new_hash_mask(old_hmask: hmask);
633	unsigned int nsize = (nhashmask + `1`) * sizeof(struct hlist_head);
634	struct hlist_head *ndst = xfrm_hash_alloc(sz: nsize);
635	struct hlist_head *odst;
636	int i;
637
638	if (!ndst)
639	return;
640
641	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
642	write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
643
644	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
645	lockdep_is_held(&net->xfrm.xfrm_policy_lock));
646
647	for (i = hmask; i >= `0`; i--)
648	xfrm_dst_hash_transfer(net, list: odst + i, ndsttable: ndst, nhashmask, dir);
649
650	rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
651	net->xfrm.policy_bydst[dir].hmask = nhashmask;
652
653	write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
654	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
655
656	synchronize_rcu();
657
658	xfrm_hash_free(n: odst, sz: (hmask + `1`) * sizeof(struct hlist_head));
659	}
660
661	static void xfrm_byidx_resize(struct net *net)
662	{
663	unsigned int hmask = net->xfrm.policy_idx_hmask;
664	unsigned int nhashmask = xfrm_new_hash_mask(old_hmask: hmask);
665	unsigned int nsize = (nhashmask + `1`) * sizeof(struct hlist_head);
666	struct hlist_head *oidx = net->xfrm.policy_byidx;
667	struct hlist_head *nidx = xfrm_hash_alloc(sz: nsize);
668	int i;
669
670	if (!nidx)
671	return;
672
673	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
674
675	for (i = hmask; i >= `0`; i--)
676	xfrm_idx_hash_transfer(list: oidx + i, nidxtable: nidx, nhashmask);
677
678	net->xfrm.policy_byidx = nidx;
679	net->xfrm.policy_idx_hmask = nhashmask;
680
681	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
682
683	xfrm_hash_free(n: oidx, sz: (hmask + `1`) * sizeof(struct hlist_head));
684	}
685
686	static inline int xfrm_bydst_should_resize(struct net net, int* dir, int *total)
687	{
688	unsigned int cnt = net->xfrm.policy_count[dir];
689	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
690
691	if (total)
692	*total += cnt;
693
694	if ((hmask + `1`) < xfrm_policy_hashmax &&
695	cnt > hmask)
696	return `1`;
697
698	return `0`;
699	}
700
701	static inline int xfrm_byidx_should_resize(struct net net, int* total)
702	{
703	unsigned int hmask = net->xfrm.policy_idx_hmask;
704
705	if ((hmask + `1`) < xfrm_policy_hashmax &&
706	total > hmask)
707	return `1`;
708
709	return `0`;
710	}
711
712	void xfrm_spd_getinfo(struct net net, struct* xfrmk_spdinfo *si)
713	{
714	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
715	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
716	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
717	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
718	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
719	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
720	si->spdhcnt = net->xfrm.policy_idx_hmask;
721	si->spdhmcnt = xfrm_policy_hashmax;
722	}
723	EXPORT_SYMBOL(xfrm_spd_getinfo);
724
725	static DEFINE_MUTEX(hash_resize_mutex);
726	static void xfrm_hash_resize(struct work_struct *work)
727	{
728	struct net net = container_of(work, struct* net, xfrm.policy_hash_work);
729	int dir, total;
730
731	mutex_lock(lock: &hash_resize_mutex);
732
733	total = `0`;
734	for (dir = `0`; dir < XFRM_POLICY_MAX; dir++) {
735	if (xfrm_bydst_should_resize(net, dir, total: &total))
736	xfrm_bydst_resize(net, dir);
737	}
738	if (xfrm_byidx_should_resize(net, total))
739	xfrm_byidx_resize(net);
740
741	mutex_unlock(lock: &hash_resize_mutex);
742	}
743
744	/ Make sure pol can be inserted into fastbin.
745	* Useful to check that later insert requests will be successful
746	* (provided xfrm_policy_lock is held throughout).
747	*/
748	static struct xfrm_pol_inexact_bin *
749	xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
750	{
751	struct xfrm_pol_inexact_bin bin, prev;
752	struct xfrm_pol_inexact_key k = {
753	.family = pol->family,
754	.type = pol->type,
755	.dir = dir,
756	.if_id = pol->if_id,
757	};
758	struct net *net = xp_net(xp: pol);
759
760	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
761
762	write_pnet(pnet: &k.net, net);
763	bin = rhashtable_lookup_fast(ht: &xfrm_policy_inexact_table, key: &k,
764	params: xfrm_pol_inexact_params);
765	if (bin)
766	return bin;
767
768	bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
769	if (!bin)
770	return NULL;
771
772	bin->k = k;
773	INIT_HLIST_HEAD(&bin->hhead);
774	bin->root_d = RB_ROOT;
775	bin->root_s = RB_ROOT;
776	seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock);
777
778	prev = rhashtable_lookup_get_insert_key(ht: &xfrm_policy_inexact_table,
779	key: &bin->k, obj: &bin->head,
780	params: xfrm_pol_inexact_params);
781	if (!prev) {
782	list_add(new: &bin->inexact_bins, head: &net->xfrm.inexact_bins);
783	return bin;
784	}
785
786	kfree(objp: bin);
787
788	return IS_ERR(ptr: prev) ? NULL : prev;
789	}
790
791	static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
792	int family, u8 prefixlen)
793	{
794	if (xfrm_addr_any(addr, family))
795	return true;
796
797	if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6)
798	return true;
799
800	if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4)
801	return true;
802
803	return false;
804	}
805
806	static bool
807	xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
808	{
809	const xfrm_address_t *addr;
810	bool saddr_any, daddr_any;
811	u8 prefixlen;
812
813	addr = &policy->selector.saddr;
814	prefixlen = policy->selector.prefixlen_s;
815
816	saddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
817	family: policy->family,
818	prefixlen);
819	addr = &policy->selector.daddr;
820	prefixlen = policy->selector.prefixlen_d;
821	daddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
822	family: policy->family,
823	prefixlen);
824	return saddr_any && daddr_any;
825	}
826
827	static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
828	const xfrm_address_t *addr, u8 prefixlen)
829	{
830	node->addr = *addr;
831	node->prefixlen = prefixlen;
832	}
833
834	static struct xfrm_pol_inexact_node *
835	xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
836	{
837	struct xfrm_pol_inexact_node *node;
838
839	node = kzalloc(sizeof(*node), GFP_ATOMIC);
840	if (node)
841	xfrm_pol_inexact_node_init(node, addr, prefixlen);
842
843	return node;
844	}
845
846	static int xfrm_policy_addr_delta(const xfrm_address_t *a,
847	const xfrm_address_t *b,
848	u8 prefixlen, u16 family)
849	{
850	u32 ma, mb, mask;
851	unsigned int pdw, pbi;
852	int delta = `0`;
853
854	switch (family) {
855	case AF_INET:
856	if (prefixlen == `0`)
857	return `0`;
858	mask = ~`0U` << (`32` - prefixlen);
859	ma = ntohl(a->a4) & mask;
860	mb = ntohl(b->a4) & mask;
861	if (ma < mb)
862	delta = -`1`;
863	else if (ma > mb)
864	delta = `1`;
865	break;
866	case AF_INET6:
867	pdw = prefixlen >> `5`;
868	pbi = prefixlen & `0x1f`;
869
870	if (pdw) {
871	delta = memcmp(a->a6, b->a6, pdw << `2`);
872	if (delta)
873	return delta;
874	}
875	if (pbi) {
876	mask = ~`0U` << (`32` - pbi);
877	ma = ntohl(a->a6[pdw]) & mask;
878	mb = ntohl(b->a6[pdw]) & mask;
879	if (ma < mb)
880	delta = -`1`;
881	else if (ma > mb)
882	delta = `1`;
883	}
884	break;
885	default:
886	break;
887	}
888
889	return delta;
890	}
891
892	static void xfrm_policy_inexact_list_reinsert(struct net *net,
893	struct xfrm_pol_inexact_node *n,
894	u16 family)
895	{
896	unsigned int matched_s, matched_d;
897	struct xfrm_policy policy, p;
898
899	matched_s = `0`;
900	matched_d = `0`;
901
902	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
903	struct hlist_node *newpos = NULL;
904	bool matches_s, matches_d;
905
906	if (policy->walk.dead \|\| !policy->bydst_reinsert)
907	continue;
908
909	WARN_ON_ONCE(policy->family != family);
910
911	policy->bydst_reinsert = false;
912	hlist_for_each_entry(p, &n->hhead, bydst) {
913	if (policy->priority > p->priority)
914	newpos = &p->bydst;
915	else if (policy->priority == p->priority &&
916	policy->pos > p->pos)
917	newpos = &p->bydst;
918	else
919	break;
920	}
921
922	if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
923	hlist_add_behind_rcu(n: &policy->bydst, prev: newpos);
924	else
925	hlist_add_head_rcu(n: &policy->bydst, h: &n->hhead);
926
927	/ paranoia checks follow.*
928	* Check that the reinserted policy matches at least
929	* saddr or daddr for current node prefix.
930	*
931	* Matching both is fine, matching saddr in one policy
932	* (but not daddr) and then matching only daddr in another
933	* is a bug.
934	*/
935	matches_s = xfrm_policy_addr_delta(a: &policy->selector.saddr,
936	b: &n->addr,
937	prefixlen: n->prefixlen,
938	family) == `0`;
939	matches_d = xfrm_policy_addr_delta(a: &policy->selector.daddr,
940	b: &n->addr,
941	prefixlen: n->prefixlen,
942	family) == `0`;
943	if (matches_s && matches_d)
944	continue;
945
946	WARN_ON_ONCE(!matches_s && !matches_d);
947	if (matches_s)
948	matched_s++;
949	if (matches_d)
950	matched_d++;
951	WARN_ON_ONCE(matched_s && matched_d);
952	}
953	}
954
955	static void xfrm_policy_inexact_node_reinsert(struct net *net,
956	struct xfrm_pol_inexact_node *n,
957	struct rb_root *new,
958	u16 family)
959	{
960	struct xfrm_pol_inexact_node *node;
961	struct rb_node *p, parent;
962
963	/ we should not have another subtree here /
964	WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root));
965	restart:
966	parent = NULL;
967	p = &new->rb_node;
968	while (*p) {
969	u8 prefixlen;
970	int delta;
971
972	parent = *p;
973	node = rb_entry(p, struct* xfrm_pol_inexact_node, node);
974
975	prefixlen = min(node->prefixlen, n->prefixlen);
976
977	delta = xfrm_policy_addr_delta(a: &n->addr, b: &node->addr,
978	prefixlen, family);
979	if (delta < `0`) {
980	p = &parent->rb_left;
981	} else if (delta > `0`) {
982	p = &parent->rb_right;
983	} else {
984	bool same_prefixlen = node->prefixlen == n->prefixlen;
985	struct xfrm_policy *tmp;
986
987	hlist_for_each_entry(tmp, &n->hhead, bydst) {
988	tmp->bydst_reinsert = true;
989	hlist_del_rcu(n: &tmp->bydst);
990	}
991
992	node->prefixlen = prefixlen;
993
994	xfrm_policy_inexact_list_reinsert(net, n: node, family);
995
996	if (same_prefixlen) {
997	kfree_rcu(n, rcu);
998	return;
999	}
1000
1001	rb_erase(*p, new);
1002	kfree_rcu(n, rcu);
1003	n = node;
1004	goto restart;
1005	}
1006	}
1007
1008	rb_link_node_rcu(node: &n->node, parent, rb_link: p);
1009	rb_insert_color(&n->node, new);
1010	}
1011
1012	/ merge nodes v and n /
1013	static void xfrm_policy_inexact_node_merge(struct net *net,
1014	struct xfrm_pol_inexact_node *v,
1015	struct xfrm_pol_inexact_node *n,
1016	u16 family)
1017	{
1018	struct xfrm_pol_inexact_node *node;
1019	struct xfrm_policy *tmp;
1020	struct rb_node *rnode;
1021
1022	/ To-be-merged node v has a subtree.*
1023	*
1024	* Dismantle it and insert its nodes to n->root.
1025	*/
1026	while ((rnode = rb_first(&v->root)) != NULL) {
1027	node = rb_entry(rnode, struct xfrm_pol_inexact_node, node);
1028	rb_erase(&node->node, &v->root);
1029	xfrm_policy_inexact_node_reinsert(net, n: node, new: &n->root,
1030	family);
1031	}
1032
1033	hlist_for_each_entry(tmp, &v->hhead, bydst) {
1034	tmp->bydst_reinsert = true;
1035	hlist_del_rcu(n: &tmp->bydst);
1036	}
1037
1038	xfrm_policy_inexact_list_reinsert(net, n, family);
1039	}
1040
1041	static struct xfrm_pol_inexact_node *
1042	xfrm_policy_inexact_insert_node(struct net *net,
1043	struct rb_root *root,
1044	xfrm_address_t *addr,
1045	u16 family, u8 prefixlen, u8 dir)
1046	{
1047	struct xfrm_pol_inexact_node *cached = NULL;
1048	struct rb_node *p, parent = NULL;
1049	struct xfrm_pol_inexact_node *node;
1050
1051	p = &root->rb_node;
1052	while (*p) {
1053	int delta;
1054
1055	parent = *p;
1056	node = rb_entry(p, struct* xfrm_pol_inexact_node, node);
1057
1058	delta = xfrm_policy_addr_delta(a: addr, b: &node->addr,
1059	prefixlen: node->prefixlen,
1060	family);
1061	if (delta == `0` && prefixlen >= node->prefixlen) {
1062	WARN_ON_ONCE(cached); / ipsec policies got lost /
1063	return node;
1064	}
1065
1066	if (delta < `0`)
1067	p = &parent->rb_left;
1068	else
1069	p = &parent->rb_right;
1070
1071	if (prefixlen < node->prefixlen) {
1072	delta = xfrm_policy_addr_delta(a: addr, b: &node->addr,
1073	prefixlen,
1074	family);
1075	if (delta)
1076	continue;
1077
1078	/ This node is a subnet of the new prefix. It needs*
1079	* to be removed and re-inserted with the smaller
1080	* prefix and all nodes that are now also covered
1081	* by the reduced prefixlen.
1082	*/
1083	rb_erase(&node->node, root);
1084
1085	if (!cached) {
1086	xfrm_pol_inexact_node_init(node, addr,
1087	prefixlen);
1088	cached = node;
1089	} else {
1090	/ This node also falls within the new*
1091	* prefixlen. Merge the to-be-reinserted
1092	* node and this one.
1093	*/
1094	xfrm_policy_inexact_node_merge(net, v: node,
1095	n: cached, family);
1096	kfree_rcu(node, rcu);
1097	}
1098
1099	/ restart /
1100	p = &root->rb_node;
1101	parent = NULL;
1102	}
1103	}
1104
1105	node = cached;
1106	if (!node) {
1107	node = xfrm_pol_inexact_node_alloc(addr, prefixlen);
1108	if (!node)
1109	return NULL;
1110	}
1111
1112	rb_link_node_rcu(node: &node->node, parent, rb_link: p);
1113	rb_insert_color(&node->node, root);
1114
1115	return node;
1116	}
1117
1118	static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
1119	{
1120	struct xfrm_pol_inexact_node *node;
1121	struct rb_node *rn = rb_first(r);
1122
1123	while (rn) {
1124	node = rb_entry(rn, struct xfrm_pol_inexact_node, node);
1125
1126	xfrm_policy_inexact_gc_tree(r: &node->root, rm);
1127	rn = rb_next(rn);
1128
1129	if (!hlist_empty(h: &node->hhead) \|\| !RB_EMPTY_ROOT(&node->root)) {
1130	WARN_ON_ONCE(rm);
1131	continue;
1132	}
1133
1134	rb_erase(&node->node, r);
1135	kfree_rcu(node, rcu);
1136	}
1137	}
1138
1139	static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
1140	{
1141	write_seqcount_begin(&b->count);
1142	xfrm_policy_inexact_gc_tree(r: &b->root_d, rm: net_exit);
1143	xfrm_policy_inexact_gc_tree(r: &b->root_s, rm: net_exit);
1144	write_seqcount_end(&b->count);
1145
1146	if (!RB_EMPTY_ROOT(&b->root_d) \|\| !RB_EMPTY_ROOT(&b->root_s) \|\|
1147	!hlist_empty(h: &b->hhead)) {
1148	WARN_ON_ONCE(net_exit);
1149	return;
1150	}
1151
1152	if (rhashtable_remove_fast(ht: &xfrm_policy_inexact_table, obj: &b->head,
1153	params: xfrm_pol_inexact_params) == `0`) {
1154	list_del(entry: &b->inexact_bins);
1155	kfree_rcu(b, rcu);
1156	}
1157	}
1158
1159	static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
1160	{
1161	struct net *net = read_pnet(pnet: &b->k.net);
1162
1163	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1164	__xfrm_policy_inexact_prune_bin(b, net_exit: false);
1165	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1166	}
1167
1168	static void __xfrm_policy_inexact_flush(struct net *net)
1169	{
1170	struct xfrm_pol_inexact_bin bin, t;
1171
1172	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1173
1174	list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)
1175	__xfrm_policy_inexact_prune_bin(b: bin, net_exit: false);
1176	}
1177
1178	static struct hlist_head *
1179	xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
1180	struct xfrm_policy *policy, u8 dir)
1181	{
1182	struct xfrm_pol_inexact_node *n;
1183	struct net *net;
1184
1185	net = xp_net(xp: policy);
1186	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1187
1188	if (xfrm_policy_inexact_insert_use_any_list(policy))
1189	return &bin->hhead;
1190
1191	if (xfrm_pol_inexact_addr_use_any_list(addr: &policy->selector.daddr,
1192	family: policy->family,
1193	prefixlen: policy->selector.prefixlen_d)) {
1194	write_seqcount_begin(&bin->count);
1195	n = xfrm_policy_inexact_insert_node(net,
1196	root: &bin->root_s,
1197	addr: &policy->selector.saddr,
1198	family: policy->family,
1199	prefixlen: policy->selector.prefixlen_s,
1200	dir);
1201	write_seqcount_end(&bin->count);
1202	if (!n)
1203	return NULL;
1204
1205	return &n->hhead;
1206	}
1207
1208	/ daddr is fixed /
1209	write_seqcount_begin(&bin->count);
1210	n = xfrm_policy_inexact_insert_node(net,
1211	root: &bin->root_d,
1212	addr: &policy->selector.daddr,
1213	family: policy->family,
1214	prefixlen: policy->selector.prefixlen_d, dir);
1215	write_seqcount_end(&bin->count);
1216	if (!n)
1217	return NULL;
1218
1219	/ saddr is wildcard /
1220	if (xfrm_pol_inexact_addr_use_any_list(addr: &policy->selector.saddr,
1221	family: policy->family,
1222	prefixlen: policy->selector.prefixlen_s))
1223	return &n->hhead;
1224
1225	write_seqcount_begin(&bin->count);
1226	n = xfrm_policy_inexact_insert_node(net,
1227	root: &n->root,
1228	addr: &policy->selector.saddr,
1229	family: policy->family,
1230	prefixlen: policy->selector.prefixlen_s, dir);
1231	write_seqcount_end(&bin->count);
1232	if (!n)
1233	return NULL;
1234
1235	return &n->hhead;
1236	}
1237
1238	static struct xfrm_policy *
1239	xfrm_policy_inexact_insert(struct xfrm_policy policy, u8 dir, int* excl)
1240	{
1241	struct xfrm_pol_inexact_bin *bin;
1242	struct xfrm_policy *delpol;
1243	struct hlist_head *chain;
1244	struct net *net;
1245
1246	bin = xfrm_policy_inexact_alloc_bin(pol: policy, dir);
1247	if (!bin)
1248	return ERR_PTR(error: -ENOMEM);
1249
1250	net = xp_net(xp: policy);
1251	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1252
1253	chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir);
1254	if (!chain) {
1255	__xfrm_policy_inexact_prune_bin(b: bin, net_exit: false);
1256	return ERR_PTR(error: -ENOMEM);
1257	}
1258
1259	delpol = xfrm_policy_insert_list(chain, policy, excl);
1260	if (delpol && excl) {
1261	__xfrm_policy_inexact_prune_bin(b: bin, net_exit: false);
1262	return ERR_PTR(error: -EEXIST);
1263	}
1264
1265	if (delpol)
1266	__xfrm_policy_inexact_prune_bin(b: bin, net_exit: false);
1267
1268	return delpol;
1269	}
1270
1271	static bool xfrm_policy_is_dead_or_sk(const struct xfrm_policy *policy)
1272	{
1273	int dir;
1274
1275	if (policy->walk.dead)
1276	return true;
1277
1278	dir = xfrm_policy_id2dir(index: policy->index);
1279	return dir >= XFRM_POLICY_MAX;
1280	}
1281
1282	static void xfrm_hash_rebuild(struct work_struct *work)
1283	{
1284	struct net net = container_of(work, struct* net,
1285	xfrm.policy_hthresh.work);
1286	struct xfrm_policy *pol;
1287	struct xfrm_policy *policy;
1288	struct hlist_head *chain;
1289	struct hlist_node *newpos;
1290	int dir;
1291	unsigned seq;
1292	u8 lbits4, rbits4, lbits6, rbits6;
1293
1294	mutex_lock(lock: &hash_resize_mutex);
1295
1296	/ read selector prefixlen thresholds /
1297	do {
1298	seq = read_seqbegin(sl: &net->xfrm.policy_hthresh.lock);
1299
1300	lbits4 = net->xfrm.policy_hthresh.lbits4;
1301	rbits4 = net->xfrm.policy_hthresh.rbits4;
1302	lbits6 = net->xfrm.policy_hthresh.lbits6;
1303	rbits6 = net->xfrm.policy_hthresh.rbits6;
1304	} while (read_seqretry(sl: &net->xfrm.policy_hthresh.lock, start: seq));
1305
1306	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1307	write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
1308
1309	/ make sure that we can insert the indirect policies again before*
1310	* we start with destructive action.
1311	*/
1312	list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
1313	struct xfrm_pol_inexact_bin *bin;
1314	u8 dbits, sbits;
1315
1316	if (xfrm_policy_is_dead_or_sk(policy))
1317	continue;
1318
1319	dir = xfrm_policy_id2dir(index: policy->index);
1320	if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1321	if (policy->family == AF_INET) {
1322	dbits = rbits4;
1323	sbits = lbits4;
1324	} else {
1325	dbits = rbits6;
1326	sbits = lbits6;
1327	}
1328	} else {
1329	if (policy->family == AF_INET) {
1330	dbits = lbits4;
1331	sbits = rbits4;
1332	} else {
1333	dbits = lbits6;
1334	sbits = rbits6;
1335	}
1336	}
1337
1338	if (policy->selector.prefixlen_d < dbits \|\|
1339	policy->selector.prefixlen_s < sbits)
1340	continue;
1341
1342	bin = xfrm_policy_inexact_alloc_bin(pol: policy, dir);
1343	if (!bin)
1344	goto out_unlock;
1345
1346	if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir))
1347	goto out_unlock;
1348	}
1349
1350	for (dir = `0`; dir < XFRM_POLICY_MAX; dir++) {
1351	if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1352	/ dir out => dst = remote, src = local /
1353	net->xfrm.policy_bydst[dir].dbits4 = rbits4;
1354	net->xfrm.policy_bydst[dir].sbits4 = lbits4;
1355	net->xfrm.policy_bydst[dir].dbits6 = rbits6;
1356	net->xfrm.policy_bydst[dir].sbits6 = lbits6;
1357	} else {
1358	/ dir in/fwd => dst = local, src = remote /
1359	net->xfrm.policy_bydst[dir].dbits4 = lbits4;
1360	net->xfrm.policy_bydst[dir].sbits4 = rbits4;
1361	net->xfrm.policy_bydst[dir].dbits6 = lbits6;
1362	net->xfrm.policy_bydst[dir].sbits6 = rbits6;
1363	}
1364	}
1365
1366	/ re-insert all policies by order of creation /
1367	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
1368	if (xfrm_policy_is_dead_or_sk(policy))
1369	continue;
1370
1371	hlist_del_rcu(n: &policy->bydst);
1372
1373	newpos = NULL;
1374	dir = xfrm_policy_id2dir(index: policy->index);
1375	chain = policy_hash_bysel(net, sel: &policy->selector,
1376	family: policy->family, dir);
1377
1378	if (!chain) {
1379	void *p = xfrm_policy_inexact_insert(policy, dir, excl: `0`);
1380
1381	WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
1382	continue;
1383	}
1384
1385	hlist_for_each_entry(pol, chain, bydst) {
1386	if (policy->priority >= pol->priority)
1387	newpos = &pol->bydst;
1388	else
1389	break;
1390	}
1391	if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1392	hlist_add_behind_rcu(n: &policy->bydst, prev: newpos);
1393	else
1394	hlist_add_head_rcu(n: &policy->bydst, h: chain);
1395	}
1396
1397	out_unlock:
1398	__xfrm_policy_inexact_flush(net);
1399	write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
1400	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1401
1402	mutex_unlock(lock: &hash_resize_mutex);
1403	}
1404
1405	void xfrm_policy_hash_rebuild(struct net *net)
1406	{
1407	schedule_work(work: &net->xfrm.policy_hthresh.work);
1408	}
1409	EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
1410
1411	/ Generate new index... KAME seems to generate them ordered by cost*
1412	* of an absolute inpredictability of ordering of rules. This will not pass. */
1413	static u32 xfrm_gen_index(struct net net, int* dir, u32 index)
1414	{
1415	for (;;) {
1416	struct hlist_head *list;
1417	struct xfrm_policy *p;
1418	u32 idx;
1419	int found;
1420
1421	if (!index) {
1422	idx = (net->xfrm.idx_generator \| dir);
1423	net->xfrm.idx_generator += `8`;
1424	} else {
1425	idx = index;
1426	index = `0`;
1427	}
1428
1429	if (idx == `0`)
1430	idx = `8`;
1431	list = net->xfrm.policy_byidx + idx_hash(net, index: idx);
1432	found = `0`;
1433	hlist_for_each_entry(p, list, byidx) {
1434	if (p->index == idx) {
1435	found = `1`;
1436	break;
1437	}
1438	}
1439	if (!found)
1440	return idx;
1441	}
1442	}
1443
1444	static inline int selector_cmp(struct xfrm_selector s1, struct* xfrm_selector *s2)
1445	{
1446	u32 p1 = (u32 ) s1;
1447	u32 p2 = (u32 ) s2;
1448	int len = sizeof(struct xfrm_selector) / sizeof(u32);
1449	int i;
1450
1451	for (i = `0`; i < len; i++) {
1452	if (p1[i] != p2[i])
1453	return `1`;
1454	}
1455
1456	return `0`;
1457	}
1458
1459	static void xfrm_policy_requeue(struct xfrm_policy *old,
1460	struct xfrm_policy *new)
1461	{
1462	struct xfrm_policy_queue *pq = &old->polq;
1463	struct sk_buff_head list;
1464
1465	if (skb_queue_empty(list: &pq->hold_queue))
1466	return;
1467
1468	__skb_queue_head_init(list: &list);
1469
1470	spin_lock_bh(lock: &pq->hold_queue.lock);
1471	skb_queue_splice_init(list: &pq->hold_queue, head: &list);
1472	if (timer_delete(timer: &pq->hold_timer))
1473	xfrm_pol_put(policy: old);
1474	spin_unlock_bh(lock: &pq->hold_queue.lock);
1475
1476	pq = &new->polq;
1477
1478	spin_lock_bh(lock: &pq->hold_queue.lock);
1479	skb_queue_splice(list: &list, head: &pq->hold_queue);
1480	pq->timeout = XFRM_QUEUE_TMO_MIN;
1481	if (!mod_timer(timer: &pq->hold_timer, expires: jiffies))
1482	xfrm_pol_hold(policy: new);
1483	spin_unlock_bh(lock: &pq->hold_queue.lock);
1484	}
1485
1486	static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark,
1487	struct xfrm_policy *pol)
1488	{
1489	return mark->v == pol->mark.v && mark->m == pol->mark.m;
1490	}
1491
1492	static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
1493	{
1494	const struct xfrm_pol_inexact_key *k = data;
1495	u32 a = k->type << `24` \| k->dir << `16` \| k->family;
1496
1497	return jhash_3words(a, b: k->if_id, c: net_hash_mix(net: read_pnet(pnet: &k->net)),
1498	initval: seed);
1499	}
1500
1501	static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
1502	{
1503	const struct xfrm_pol_inexact_bin *b = data;
1504
1505	return xfrm_pol_bin_key(data: &b->k, len: `0`, seed);
1506	}
1507
1508	static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
1509	const void *ptr)
1510	{
1511	const struct xfrm_pol_inexact_key *key = arg->key;
1512	const struct xfrm_pol_inexact_bin *b = ptr;
1513	int ret;
1514
1515	if (!net_eq(net1: read_pnet(pnet: &b->k.net), net2: read_pnet(pnet: &key->net)))
1516	return -`1`;
1517
1518	ret = b->k.dir ^ key->dir;
1519	if (ret)
1520	return ret;
1521
1522	ret = b->k.type ^ key->type;
1523	if (ret)
1524	return ret;
1525
1526	ret = b->k.family ^ key->family;
1527	if (ret)
1528	return ret;
1529
1530	return b->k.if_id ^ key->if_id;
1531	}
1532
1533	static const struct rhashtable_params xfrm_pol_inexact_params = {
1534	.head_offset = offsetof(struct xfrm_pol_inexact_bin, head),
1535	.hashfn = xfrm_pol_bin_key,
1536	.obj_hashfn = xfrm_pol_bin_obj,
1537	.obj_cmpfn = xfrm_pol_bin_cmp,
1538	.automatic_shrinking = true,
1539	};
1540
1541	static struct xfrm_policy xfrm_policy_insert_list(struct* hlist_head *chain,
1542	struct xfrm_policy *policy,
1543	bool excl)
1544	{
1545	struct xfrm_policy pol, newpos = NULL, *delpol = NULL;
1546
1547	hlist_for_each_entry(pol, chain, bydst) {
1548	if (pol->type == policy->type &&
1549	pol->if_id == policy->if_id &&
1550	!selector_cmp(s1: &pol->selector, s2: &policy->selector) &&
1551	xfrm_policy_mark_match(mark: &policy->mark, pol) &&
1552	xfrm_sec_ctx_match(s1: pol->security, s2: policy->security) &&
1553	!WARN_ON(delpol)) {
1554	if (excl)
1555	return ERR_PTR(error: -EEXIST);
1556	delpol = pol;
1557	if (policy->priority > pol->priority)
1558	continue;
1559	} else if (policy->priority >= pol->priority) {
1560	newpos = pol;
1561	continue;
1562	}
1563	if (delpol)
1564	break;
1565	}
1566
1567	if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1568	hlist_add_behind_rcu(n: &policy->bydst, prev: &newpos->bydst);
1569	else
1570	/ Packet offload policies enter to the head*
1571	* to speed-up lookups.
1572	*/
1573	hlist_add_head_rcu(n: &policy->bydst, h: chain);
1574
1575	return delpol;
1576	}
1577
1578	int xfrm_policy_insert(int dir, struct xfrm_policy policy, int* excl)
1579	{
1580	struct net *net = xp_net(xp: policy);
1581	struct xfrm_policy *delpol;
1582	struct hlist_head *chain;
1583
1584	/ Sanitize mark before store /
1585	policy->mark.v &= policy->mark.m;
1586
1587	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1588	chain = policy_hash_bysel(net, sel: &policy->selector, family: policy->family, dir);
1589	if (chain)
1590	delpol = xfrm_policy_insert_list(chain, policy, excl);
1591	else
1592	delpol = xfrm_policy_inexact_insert(policy, dir, excl);
1593
1594	if (IS_ERR(ptr: delpol)) {
1595	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1596	return PTR_ERR(ptr: delpol);
1597	}
1598
1599	__xfrm_policy_link(pol: policy, dir);
1600
1601	/ After previous checking, family can either be AF_INET or AF_INET6 /
1602	if (policy->family == AF_INET)
1603	rt_genid_bump_ipv4(net);
1604	else
1605	rt_genid_bump_ipv6(net);
1606
1607	if (delpol) {
1608	xfrm_policy_requeue(old: delpol, new: policy);
1609	__xfrm_policy_unlink(pol: delpol, dir);
1610	}
1611	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, index: policy->index);
1612	hlist_add_head(n: &policy->byidx, h: net->xfrm.policy_byidx+idx_hash(net, index: policy->index));
1613	policy->curlft.add_time = ktime_get_real_seconds();
1614	policy->curlft.use_time = `0`;
1615	if (!mod_timer(timer: &policy->timer, expires: jiffies + HZ))
1616	xfrm_pol_hold(policy);
1617	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1618
1619	if (delpol)
1620	xfrm_policy_kill(policy: delpol);
1621	else if (xfrm_bydst_should_resize(net, dir, NULL))
1622	schedule_work(work: &net->xfrm.policy_hash_work);
1623
1624	return `0`;
1625	}
1626	EXPORT_SYMBOL(xfrm_policy_insert);
1627
1628	static struct xfrm_policy *
1629	__xfrm_policy_bysel_ctx(struct hlist_head chain, const* struct xfrm_mark *mark,
1630	u32 if_id, u8 type, int dir, struct xfrm_selector *sel,
1631	struct xfrm_sec_ctx *ctx)
1632	{
1633	struct xfrm_policy *pol;
1634
1635	if (!chain)
1636	return NULL;
1637
1638	hlist_for_each_entry(pol, chain, bydst) {
1639	if (pol->type == type &&
1640	pol->if_id == if_id &&
1641	xfrm_policy_mark_match(mark, pol) &&
1642	!selector_cmp(s1: sel, s2: &pol->selector) &&
1643	xfrm_sec_ctx_match(s1: ctx, s2: pol->security))
1644	return pol;
1645	}
1646
1647	return NULL;
1648	}
1649
1650	struct xfrm_policy *
1651	xfrm_policy_bysel_ctx(struct net net, const* struct xfrm_mark *mark, u32 if_id,
1652	u8 type, int dir, struct xfrm_selector *sel,
1653	struct xfrm_sec_ctx ctx, int* delete, int *err)
1654	{
1655	struct xfrm_pol_inexact_bin *bin = NULL;
1656	struct xfrm_policy pol, ret = NULL;
1657	struct hlist_head *chain;
1658
1659	*err = `0`;
1660	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1661	chain = policy_hash_bysel(net, sel, family: sel->family, dir);
1662	if (!chain) {
1663	struct xfrm_pol_inexact_candidates cand;
1664	int i;
1665
1666	bin = xfrm_policy_inexact_lookup(net, type,
1667	family: sel->family, dir, if_id);
1668	if (!bin) {
1669	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1670	return NULL;
1671	}
1672
1673	if (!xfrm_policy_find_inexact_candidates(cand: &cand, b: bin,
1674	saddr: &sel->saddr,
1675	daddr: &sel->daddr)) {
1676	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1677	return NULL;
1678	}
1679
1680	pol = NULL;
1681	for (i = `0`; i < ARRAY_SIZE(cand.res); i++) {
1682	struct xfrm_policy *tmp;
1683
1684	tmp = __xfrm_policy_bysel_ctx(chain: cand.res[i], mark,
1685	if_id, type, dir,
1686	sel, ctx);
1687	if (!tmp)
1688	continue;
1689
1690	if (!pol \|\| tmp->pos < pol->pos)
1691	pol = tmp;
1692	}
1693	} else {
1694	pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,
1695	sel, ctx);
1696	}
1697
1698	if (pol) {
1699	xfrm_pol_hold(policy: pol);
1700	if (delete) {
1701	*err = security_xfrm_policy_delete(ctx: pol->security);
1702	if (*err) {
1703	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1704	return pol;
1705	}
1706	__xfrm_policy_unlink(pol, dir);
1707	}
1708	ret = pol;
1709	}
1710	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1711
1712	if (ret && delete)
1713	xfrm_policy_kill(policy: ret);
1714	if (bin && delete)
1715	xfrm_policy_inexact_prune_bin(b: bin);
1716	return ret;
1717	}
1718	EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1719
1720	struct xfrm_policy *
1721	xfrm_policy_byid(struct net net, const* struct xfrm_mark *mark, u32 if_id,
1722	u8 type, int dir, u32 id, int delete, int *err)
1723	{
1724	struct xfrm_policy pol, ret;
1725	struct hlist_head *chain;
1726
1727	*err = -ENOENT;
1728	if (xfrm_policy_id2dir(index: id) != dir)
1729	return NULL;
1730
1731	*err = `0`;
1732	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1733	chain = net->xfrm.policy_byidx + idx_hash(net, index: id);
1734	ret = NULL;
1735	hlist_for_each_entry(pol, chain, byidx) {
1736	if (pol->type == type && pol->index == id &&
1737	pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {
1738	xfrm_pol_hold(policy: pol);
1739	if (delete) {
1740	*err = security_xfrm_policy_delete(
1741	ctx: pol->security);
1742	if (*err) {
1743	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1744	return pol;
1745	}
1746	__xfrm_policy_unlink(pol, dir);
1747	}
1748	ret = pol;
1749	break;
1750	}
1751	}
1752	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1753
1754	if (ret && delete)
1755	xfrm_policy_kill(policy: ret);
1756	return ret;
1757	}
1758	EXPORT_SYMBOL(xfrm_policy_byid);
1759
1760	#ifdef CONFIG_SECURITY_NETWORK_XFRM
1761	static inline int
1762	xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1763	{
1764	struct xfrm_policy *pol;
1765	int err = `0`;
1766
1767	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1768	if (pol->walk.dead \|\|
1769	xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX \|\|
1770	pol->type != type)
1771	continue;
1772
1773	err = security_xfrm_policy_delete(pol->security);
1774	if (err) {
1775	xfrm_audit_policy_delete(pol, `0`, task_valid);
1776	return err;
1777	}
1778	}
1779	return err;
1780	}
1781
1782	static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1783	struct net_device *dev,
1784	bool task_valid)
1785	{
1786	struct xfrm_policy *pol;
1787	int err = `0`;
1788
1789	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1790	if (pol->walk.dead \|\|
1791	xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX \|\|
1792	pol->xdo.dev != dev)
1793	continue;
1794
1795	err = security_xfrm_policy_delete(pol->security);
1796	if (err) {
1797	xfrm_audit_policy_delete(pol, `0`, task_valid);
1798	return err;
1799	}
1800	}
1801	return err;
1802	}
1803	#else
1804	static inline int
1805	xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1806	{
1807	return `0`;
1808	}
1809
1810	static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1811	struct net_device *dev,
1812	bool task_valid)
1813	{
1814	return `0`;
1815	}
1816	#endif
1817
1818	int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
1819	{
1820	int dir, err = `0`, cnt = `0`;
1821	struct xfrm_policy *pol;
1822
1823	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1824
1825	err = xfrm_policy_flush_secctx_check(net, type, task_valid);
1826	if (err)
1827	goto out;
1828
1829	again:
1830	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1831	if (pol->walk.dead)
1832	continue;
1833
1834	dir = xfrm_policy_id2dir(index: pol->index);
1835	if (dir >= XFRM_POLICY_MAX \|\|
1836	pol->type != type)
1837	continue;
1838
1839	__xfrm_policy_unlink(pol, dir);
1840	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1841	cnt++;
1842	xfrm_audit_policy_delete(xp: pol, result: `1`, task_valid);
1843	xfrm_policy_kill(policy: pol);
1844	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1845	goto again;
1846	}
1847	if (cnt)
1848	__xfrm_policy_inexact_flush(net);
1849	else
1850	err = -ESRCH;
1851	out:
1852	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1853	return err;
1854	}
1855	EXPORT_SYMBOL(xfrm_policy_flush);
1856
1857	int xfrm_dev_policy_flush(struct net net, struct* net_device *dev,
1858	bool task_valid)
1859	{
1860	int dir, err = `0`, cnt = `0`;
1861	struct xfrm_policy *pol;
1862
1863	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1864
1865	err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid);
1866	if (err)
1867	goto out;
1868
1869	again:
1870	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1871	if (pol->walk.dead)
1872	continue;
1873
1874	dir = xfrm_policy_id2dir(index: pol->index);
1875	if (dir >= XFRM_POLICY_MAX \|\|
1876	pol->xdo.dev != dev)
1877	continue;
1878
1879	__xfrm_policy_unlink(pol, dir);
1880	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1881	cnt++;
1882	xfrm_audit_policy_delete(xp: pol, result: `1`, task_valid);
1883	xfrm_policy_kill(policy: pol);
1884	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1885	goto again;
1886	}
1887	if (cnt)
1888	__xfrm_policy_inexact_flush(net);
1889	else
1890	err = -ESRCH;
1891	out:
1892	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1893	return err;
1894	}
1895	EXPORT_SYMBOL(xfrm_dev_policy_flush);
1896
1897	int xfrm_policy_walk(struct net net, struct* xfrm_policy_walk *walk,
1898	int (func)(struct* xfrm_policy , int, int, void**),
1899	void *data)
1900	{
1901	struct xfrm_policy *pol;
1902	struct xfrm_policy_walk_entry *x;
1903	int error = `0`;
1904
1905	if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1906	walk->type != XFRM_POLICY_TYPE_ANY)
1907	return -EINVAL;
1908
1909	if (list_empty(head: &walk->walk.all) && walk->seq != `0`)
1910	return `0`;
1911
1912	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
1913	if (list_empty(head: &walk->walk.all))
1914	x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1915	else
1916	x = list_first_entry(&walk->walk.all,
1917	struct xfrm_policy_walk_entry, all);
1918
1919	list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1920	if (x->dead)
1921	continue;
1922	pol = container_of(x, struct xfrm_policy, walk);
1923	if (walk->type != XFRM_POLICY_TYPE_ANY &&
1924	walk->type != pol->type)
1925	continue;
1926	error = func(pol, xfrm_policy_id2dir(index: pol->index),
1927	walk->seq, data);
1928	if (error) {
1929	list_move_tail(list: &walk->walk.all, head: &x->all);
1930	goto out;
1931	}
1932	walk->seq++;
1933	}
1934	if (walk->seq == `0`) {
1935	error = -ENOENT;
1936	goto out;
1937	}
1938	list_del_init(entry: &walk->walk.all);
1939	out:
1940	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1941	return error;
1942	}
1943	EXPORT_SYMBOL(xfrm_policy_walk);
1944
1945	void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1946	{
1947	INIT_LIST_HEAD(list: &walk->walk.all);
1948	walk->walk.dead = `1`;
1949	walk->type = type;
1950	walk->seq = `0`;
1951	}
1952	EXPORT_SYMBOL(xfrm_policy_walk_init);
1953
1954	void xfrm_policy_walk_done(struct xfrm_policy_walk walk, struct* net *net)
1955	{
1956	if (list_empty(head: &walk->walk.all))
1957	return;
1958
1959	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); /FIXME where is net? /
1960	list_del(entry: &walk->walk.all);
1961	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
1962	}
1963	EXPORT_SYMBOL(xfrm_policy_walk_done);
1964
1965	/*
1966	* Find policy to apply to this flow.
1967	*
1968	* Returns 0 if policy found, else an -errno.
1969	*/
1970	static int xfrm_policy_match(const struct xfrm_policy *pol,
1971	const struct flowi *fl,
1972	u8 type, u16 family, u32 if_id)
1973	{
1974	const struct xfrm_selector *sel = &pol->selector;
1975	int ret = -ESRCH;
1976	bool match;
1977
1978	if (pol->family != family \|\|
1979	pol->if_id != if_id \|\|
1980	(fl->flowi_mark & pol->mark.m) != pol->mark.v \|\|
1981	pol->type != type)
1982	return ret;
1983
1984	match = xfrm_selector_match(sel, fl, family);
1985	if (match)
1986	ret = security_xfrm_policy_lookup(ctx: pol->security, fl_secid: fl->flowi_secid);
1987	return ret;
1988	}
1989
1990	static struct xfrm_pol_inexact_node *
1991	xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
1992	seqcount_spinlock_t *count,
1993	const xfrm_address_t *addr, u16 family)
1994	{
1995	const struct rb_node *parent;
1996	int seq;
1997
1998	again:
1999	seq = read_seqcount_begin(count);
2000
2001	parent = rcu_dereference_raw(r->rb_node);
2002	while (parent) {
2003	struct xfrm_pol_inexact_node *node;
2004	int delta;
2005
2006	node = rb_entry(parent, struct xfrm_pol_inexact_node, node);
2007
2008	delta = xfrm_policy_addr_delta(a: addr, b: &node->addr,
2009	prefixlen: node->prefixlen, family);
2010	if (delta < `0`) {
2011	parent = rcu_dereference_raw(parent->rb_left);
2012	continue;
2013	} else if (delta > `0`) {
2014	parent = rcu_dereference_raw(parent->rb_right);
2015	continue;
2016	}
2017
2018	return node;
2019	}
2020
2021	if (read_seqcount_retry(count, seq))
2022	goto again;
2023
2024	return NULL;
2025	}
2026
2027	static bool
2028	xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
2029	struct xfrm_pol_inexact_bin *b,
2030	const xfrm_address_t *saddr,
2031	const xfrm_address_t *daddr)
2032	{
2033	struct xfrm_pol_inexact_node *n;
2034	u16 family;
2035
2036	if (!b)
2037	return false;
2038
2039	family = b->k.family;
2040	memset(s: cand, c: `0`, n: sizeof(*cand));
2041	cand->res[XFRM_POL_CAND_ANY] = &b->hhead;
2042
2043	n = xfrm_policy_lookup_inexact_addr(r: &b->root_d, count: &b->count, addr: daddr,
2044	family);
2045	if (n) {
2046	cand->res[XFRM_POL_CAND_DADDR] = &n->hhead;
2047	n = xfrm_policy_lookup_inexact_addr(r: &n->root, count: &b->count, addr: saddr,
2048	family);
2049	if (n)
2050	cand->res[XFRM_POL_CAND_BOTH] = &n->hhead;
2051	}
2052
2053	n = xfrm_policy_lookup_inexact_addr(r: &b->root_s, count: &b->count, addr: saddr,
2054	family);
2055	if (n)
2056	cand->res[XFRM_POL_CAND_SADDR] = &n->hhead;
2057
2058	return true;
2059	}
2060
2061	static struct xfrm_pol_inexact_bin *
2062	xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
2063	u8 dir, u32 if_id)
2064	{
2065	struct xfrm_pol_inexact_key k = {
2066	.family = family,
2067	.type = type,
2068	.dir = dir,
2069	.if_id = if_id,
2070	};
2071
2072	write_pnet(pnet: &k.net, net);
2073
2074	return rhashtable_lookup(ht: &xfrm_policy_inexact_table, key: &k,
2075	params: xfrm_pol_inexact_params);
2076	}
2077
2078	static struct xfrm_pol_inexact_bin *
2079	xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
2080	u8 dir, u32 if_id)
2081	{
2082	struct xfrm_pol_inexact_bin *bin;
2083
2084	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
2085
2086	rcu_read_lock();
2087	bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2088	rcu_read_unlock();
2089
2090	return bin;
2091	}
2092
2093	static struct xfrm_policy *
2094	__xfrm_policy_eval_candidates(struct hlist_head *chain,
2095	struct xfrm_policy *prefer,
2096	const struct flowi *fl,
2097	u8 type, u16 family, u32 if_id)
2098	{
2099	u32 priority = prefer ? prefer->priority : ~`0u`;
2100	struct xfrm_policy *pol;
2101
2102	if (!chain)
2103	return NULL;
2104
2105	hlist_for_each_entry_rcu(pol, chain, bydst) {
2106	int err;
2107
2108	if (pol->priority > priority)
2109	break;
2110
2111	err = xfrm_policy_match(pol, fl, type, family, if_id);
2112	if (err) {
2113	if (err != -ESRCH)
2114	return ERR_PTR(error: err);
2115
2116	continue;
2117	}
2118
2119	if (prefer) {
2120	/ matches. Is it older than prefer? /*
2121	if (pol->priority == priority &&
2122	prefer->pos < pol->pos)
2123	return prefer;
2124	}
2125
2126	return pol;
2127	}
2128
2129	return NULL;
2130	}
2131
2132	static struct xfrm_policy *
2133	xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
2134	struct xfrm_policy *prefer,
2135	const struct flowi *fl,
2136	u8 type, u16 family, u32 if_id)
2137	{
2138	struct xfrm_policy *tmp;
2139	int i;
2140
2141	for (i = `0`; i < ARRAY_SIZE(cand->res); i++) {
2142	tmp = __xfrm_policy_eval_candidates(chain: cand->res[i],
2143	prefer,
2144	fl, type, family, if_id);
2145	if (!tmp)
2146	continue;
2147
2148	if (IS_ERR(ptr: tmp))
2149	return tmp;
2150	prefer = tmp;
2151	}
2152
2153	return prefer;
2154	}
2155
2156	static struct xfrm_policy xfrm_policy_lookup_bytype(struct* net *net, u8 type,
2157	const struct flowi *fl,
2158	u16 family, u8 dir,
2159	u32 if_id)
2160	{
2161	struct xfrm_pol_inexact_candidates cand;
2162	const xfrm_address_t daddr, saddr;
2163	struct xfrm_pol_inexact_bin *bin;
2164	struct xfrm_policy pol, ret;
2165	struct hlist_head *chain;
2166	unsigned int sequence;
2167	int err;
2168
2169	daddr = xfrm_flowi_daddr(fl, family);
2170	saddr = xfrm_flowi_saddr(fl, family);
2171	if (unlikely(!daddr \|\| !saddr))
2172	return NULL;
2173
2174	rcu_read_lock();
2175	retry:
2176	do {
2177	sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
2178	chain = policy_hash_direct(net, daddr, saddr, family, dir);
2179	} while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence));
2180
2181	ret = NULL;
2182	hlist_for_each_entry_rcu(pol, chain, bydst) {
2183	err = xfrm_policy_match(pol, fl, type, family, if_id);
2184	if (err) {
2185	if (err == -ESRCH)
2186	continue;
2187	else {
2188	ret = ERR_PTR(error: err);
2189	goto fail;
2190	}
2191	} else {
2192	ret = pol;
2193	break;
2194	}
2195	}
2196	if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET)
2197	goto skip_inexact;
2198
2199	bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2200	if (!bin \|\| !xfrm_policy_find_inexact_candidates(cand: &cand, b: bin, saddr,
2201	daddr))
2202	goto skip_inexact;
2203
2204	pol = xfrm_policy_eval_candidates(cand: &cand, prefer: ret, fl, type,
2205	family, if_id);
2206	if (pol) {
2207	ret = pol;
2208	if (IS_ERR(ptr: pol))
2209	goto fail;
2210	}
2211
2212	skip_inexact:
2213	if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence))
2214	goto retry;
2215
2216	if (ret && !xfrm_pol_hold_rcu(policy: ret))
2217	goto retry;
2218	fail:
2219	rcu_read_unlock();
2220
2221	return ret;
2222	}
2223
2224	static struct xfrm_policy xfrm_policy_lookup(struct* net *net,
2225	const struct flowi *fl,
2226	u16 family, u8 dir, u32 if_id)
2227	{
2228	#ifdef CONFIG_XFRM_SUB_POLICY
2229	struct xfrm_policy *pol;
2230
2231	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
2232	dir, if_id);
2233	if (pol != NULL)
2234	return pol;
2235	#endif
2236	return xfrm_policy_lookup_bytype(net, type: XFRM_POLICY_TYPE_MAIN, fl, family,
2237	dir, if_id);
2238	}
2239
2240	static struct xfrm_policy xfrm_sk_policy_lookup(const* struct sock sk, int* dir,
2241	const struct flowi *fl,
2242	u16 family, u32 if_id)
2243	{
2244	struct xfrm_policy *pol;
2245
2246	rcu_read_lock();
2247	again:
2248	pol = rcu_dereference(sk->sk_policy[dir]);
2249	if (pol != NULL) {
2250	bool match;
2251	int err = `0`;
2252
2253	if (pol->family != family) {
2254	pol = NULL;
2255	goto out;
2256	}
2257
2258	match = xfrm_selector_match(sel: &pol->selector, fl, family);
2259	if (match) {
2260	if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v \|\|
2261	pol->if_id != if_id) {
2262	pol = NULL;
2263	goto out;
2264	}
2265	err = security_xfrm_policy_lookup(ctx: pol->security,
2266	fl_secid: fl->flowi_secid);
2267	if (!err) {
2268	if (!xfrm_pol_hold_rcu(policy: pol))
2269	goto again;
2270	} else if (err == -ESRCH) {
2271	pol = NULL;
2272	} else {
2273	pol = ERR_PTR(error: err);
2274	}
2275	} else
2276	pol = NULL;
2277	}
2278	out:
2279	rcu_read_unlock();
2280	return pol;
2281	}
2282
2283	static u32 xfrm_gen_pos_slow(struct net *net)
2284	{
2285	struct xfrm_policy *policy;
2286	u32 i = `0`;
2287
2288	/ oldest entry is last in list /
2289	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
2290	if (!xfrm_policy_is_dead_or_sk(policy))
2291	policy->pos = ++i;
2292	}
2293
2294	return i;
2295	}
2296
2297	static u32 xfrm_gen_pos(struct net *net)
2298	{
2299	const struct xfrm_policy *policy;
2300	u32 i = `0`;
2301
2302	/ most recently added policy is at the head of the list /
2303	list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
2304	if (xfrm_policy_is_dead_or_sk(policy))
2305	continue;
2306
2307	if (policy->pos == UINT_MAX)
2308	return xfrm_gen_pos_slow(net);
2309
2310	i = policy->pos + `1`;
2311	break;
2312	}
2313
2314	return i;
2315	}
2316
2317	static void __xfrm_policy_link(struct xfrm_policy pol, int* dir)
2318	{
2319	struct net *net = xp_net(xp: pol);
2320
2321	switch (dir) {
2322	case XFRM_POLICY_IN:
2323	case XFRM_POLICY_FWD:
2324	case XFRM_POLICY_OUT:
2325	pol->pos = xfrm_gen_pos(net);
2326	break;
2327	}
2328
2329	list_add(new: &pol->walk.all, head: &net->xfrm.policy_all);
2330	net->xfrm.policy_count[dir]++;
2331	xfrm_pol_hold(policy: pol);
2332	}
2333
2334	static struct xfrm_policy __xfrm_policy_unlink(struct* xfrm_policy *pol,
2335	int dir)
2336	{
2337	struct net *net = xp_net(xp: pol);
2338
2339	if (list_empty(head: &pol->walk.all))
2340	return NULL;
2341
2342	/ Socket policies are not hashed. /
2343	if (!hlist_unhashed(h: &pol->bydst)) {
2344	hlist_del_rcu(n: &pol->bydst);
2345	hlist_del(n: &pol->byidx);
2346	}
2347
2348	list_del_init(entry: &pol->walk.all);
2349	net->xfrm.policy_count[dir]--;
2350
2351	return pol;
2352	}
2353
2354	static void xfrm_sk_policy_link(struct xfrm_policy pol, int* dir)
2355	{
2356	__xfrm_policy_link(pol, dir: XFRM_POLICY_MAX + dir);
2357	}
2358
2359	static void xfrm_sk_policy_unlink(struct xfrm_policy pol, int* dir)
2360	{
2361	__xfrm_policy_unlink(pol, dir: XFRM_POLICY_MAX + dir);
2362	}
2363
2364	int xfrm_policy_delete(struct xfrm_policy pol, int* dir)
2365	{
2366	struct net *net = xp_net(xp: pol);
2367
2368	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
2369	pol = __xfrm_policy_unlink(pol, dir);
2370	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
2371	if (pol) {
2372	xfrm_policy_kill(policy: pol);
2373	return `0`;
2374	}
2375	return -ENOENT;
2376	}
2377	EXPORT_SYMBOL(xfrm_policy_delete);
2378
2379	int xfrm_sk_policy_insert(struct sock sk, int* dir, struct xfrm_policy *pol)
2380	{
2381	struct net *net = sock_net(sk);
2382	struct xfrm_policy *old_pol;
2383
2384	#ifdef CONFIG_XFRM_SUB_POLICY
2385	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
2386	return -EINVAL;
2387	#endif
2388
2389	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
2390	old_pol = rcu_dereference_protected(sk->sk_policy[dir],
2391	lockdep_is_held(&net->xfrm.xfrm_policy_lock));
2392	if (pol) {
2393	pol->curlft.add_time = ktime_get_real_seconds();
2394	pol->index = xfrm_gen_index(net, dir: XFRM_POLICY_MAX+dir, index: `0`);
2395	xfrm_sk_policy_link(pol, dir);
2396	}
2397	rcu_assign_pointer(sk->sk_policy[dir], pol);
2398	if (old_pol) {
2399	if (pol)
2400	xfrm_policy_requeue(old: old_pol, new: pol);
2401
2402	/ Unlinking succeeds always. This is the only function*
2403	* allowed to delete or replace socket policy.
2404	*/
2405	xfrm_sk_policy_unlink(pol: old_pol, dir);
2406	}
2407	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
2408
2409	if (old_pol) {
2410	xfrm_policy_kill(policy: old_pol);
2411	}
2412	return `0`;
2413	}
2414
2415	static struct xfrm_policy clone_policy(const* struct xfrm_policy old, int* dir)
2416	{
2417	struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(xp: old), GFP_ATOMIC);
2418	struct net *net = xp_net(xp: old);
2419
2420	if (newp) {
2421	newp->selector = old->selector;
2422	if (security_xfrm_policy_clone(old: old->security,
2423	new_ctxp: &newp->security)) {
2424	kfree(objp: newp);
2425	return NULL; / ENOMEM /
2426	}
2427	newp->lft = old->lft;
2428	newp->curlft = old->curlft;
2429	newp->mark = old->mark;
2430	newp->if_id = old->if_id;
2431	newp->action = old->action;
2432	newp->flags = old->flags;
2433	newp->xfrm_nr = old->xfrm_nr;
2434	newp->index = old->index;
2435	newp->type = old->type;
2436	newp->family = old->family;
2437	memcpy(to: newp->xfrm_vec, from: old->xfrm_vec,
2438	len: newp->xfrm_nr*sizeof(struct xfrm_tmpl));
2439	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
2440	xfrm_sk_policy_link(pol: newp, dir);
2441	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
2442	xfrm_pol_put(policy: newp);
2443	}
2444	return newp;
2445	}
2446
2447	int __xfrm_sk_clone_policy(struct sock sk, const* struct sock *osk)
2448	{
2449	const struct xfrm_policy *p;
2450	struct xfrm_policy *np;
2451	int i, ret = `0`;
2452
2453	rcu_read_lock();
2454	for (i = `0`; i < `2`; i++) {
2455	p = rcu_dereference(osk->sk_policy[i]);
2456	if (p) {
2457	np = clone_policy(old: p, dir: i);
2458	if (unlikely(!np)) {
2459	ret = -ENOMEM;
2460	break;
2461	}
2462	rcu_assign_pointer(sk->sk_policy[i], np);
2463	}
2464	}
2465	rcu_read_unlock();
2466	return ret;
2467	}
2468
2469	static int
2470	xfrm_get_saddr(unsigned short family, xfrm_address_t *saddr,
2471	const struct xfrm_dst_lookup_params *params)
2472	{
2473	int err;
2474	const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2475
2476	if (unlikely(afinfo == NULL))
2477	return -EINVAL;
2478	err = afinfo->get_saddr(saddr, params);
2479	rcu_read_unlock();
2480	return err;
2481	}
2482
2483	/ Resolve list of templates for the flow, given policy. /
2484
2485	static int
2486	xfrm_tmpl_resolve_one(struct xfrm_policy policy, const* struct flowi *fl,
2487	struct xfrm_state *xfrm, unsigned* short family)
2488	{
2489	struct net *net = xp_net(xp: policy);
2490	int nx;
2491	int i, error;
2492	xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
2493	xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
2494	xfrm_address_t tmp;
2495
2496	for (nx = `0`, i = `0`; i < policy->xfrm_nr; i++) {
2497	struct xfrm_state *x;
2498	xfrm_address_t *remote = daddr;
2499	xfrm_address_t *local = saddr;
2500	struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
2501
2502	if (tmpl->mode == XFRM_MODE_TUNNEL \|\|
2503	tmpl->mode == XFRM_MODE_IPTFS \|\|
2504	tmpl->mode == XFRM_MODE_BEET) {
2505	remote = &tmpl->id.daddr;
2506	local = &tmpl->saddr;
2507	if (xfrm_addr_any(addr: local, family: tmpl->encap_family)) {
2508	struct xfrm_dst_lookup_params params;
2509
2510	memset(s: &params, c: `0`, n: sizeof(params));
2511	params.net = net;
2512	params.oif = fl->flowi_oif;
2513	params.daddr = remote;
2514	error = xfrm_get_saddr(family: tmpl->encap_family, saddr: &tmp,
2515	params: &params);
2516	if (error)
2517	goto fail;
2518	local = &tmp;
2519	}
2520	}
2521
2522	x = xfrm_state_find(daddr: remote, saddr: local, fl, tmpl, pol: policy, err: &error,
2523	family, if_id: policy->if_id);
2524	if (x && x->dir && x->dir != XFRM_SA_DIR_OUT) {
2525	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEDIRERROR);
2526	xfrm_state_put(x);
2527	error = -EINVAL;
2528	goto fail;
2529	}
2530
2531	if (x && x->km.state == XFRM_STATE_VALID) {
2532	xfrm[nx++] = x;
2533	daddr = remote;
2534	saddr = local;
2535	continue;
2536	}
2537	if (x) {
2538	error = (x->km.state == XFRM_STATE_ERROR ?
2539	-EINVAL : -EAGAIN);
2540	xfrm_state_put(x);
2541	} else if (error == -ESRCH) {
2542	error = -EAGAIN;
2543	}
2544
2545	if (!tmpl->optional)
2546	goto fail;
2547	}
2548	return nx;
2549
2550	fail:
2551	for (nx--; nx >= `0`; nx--)
2552	xfrm_state_put(x: xfrm[nx]);
2553	return error;
2554	}
2555
2556	static int
2557	xfrm_tmpl_resolve(struct xfrm_policy *pols, int* npols, const struct flowi *fl,
2558	struct xfrm_state *xfrm, unsigned* short family)
2559	{
2560	struct xfrm_state *tp[XFRM_MAX_DEPTH];
2561	struct xfrm_state **tpp = (npols > `1`) ? tp : xfrm;
2562	int cnx = `0`;
2563	int error;
2564	int ret;
2565	int i;
2566
2567	for (i = `0`; i < npols; i++) {
2568	if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
2569	error = -ENOBUFS;
2570	goto fail;
2571	}
2572
2573	ret = xfrm_tmpl_resolve_one(policy: pols[i], fl, xfrm: &tpp[cnx], family);
2574	if (ret < `0`) {
2575	error = ret;
2576	goto fail;
2577	} else
2578	cnx += ret;
2579	}
2580
2581	/ found states are sorted for outbound processing /
2582	if (npols > `1`)
2583	xfrm_state_sort(d: xfrm, s: tpp, n: cnx, family);
2584
2585	return cnx;
2586
2587	fail:
2588	for (cnx--; cnx >= `0`; cnx--)
2589	xfrm_state_put(x: tpp[cnx]);
2590	return error;
2591
2592	}
2593
2594	static dscp_t xfrm_get_dscp(const struct flowi fl, int* family)
2595	{
2596	if (family == AF_INET)
2597	return fl->u.ip4.flowi4_dscp;
2598
2599	return `0`;
2600	}
2601
2602	static inline struct xfrm_dst xfrm_alloc_dst(struct* net net, int* family)
2603	{
2604	const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2605	struct dst_ops *dst_ops;
2606	struct xfrm_dst *xdst;
2607
2608	if (!afinfo)
2609	return ERR_PTR(error: -EINVAL);
2610
2611	switch (family) {
2612	case AF_INET:
2613	dst_ops = &net->xfrm.xfrm4_dst_ops;
2614	break;
2615	#if IS_ENABLED(CONFIG_IPV6)
2616	case AF_INET6:
2617	dst_ops = &net->xfrm.xfrm6_dst_ops;
2618	break;
2619	#endif
2620	default:
2621	BUG();
2622	}
2623	xdst = dst_alloc(ops: dst_ops, NULL, DST_OBSOLETE_NONE, flags: `0`);
2624
2625	if (likely(xdst)) {
2626	memset_after(xdst, `0`, u.dst);
2627	} else
2628	xdst = ERR_PTR(error: -ENOBUFS);
2629
2630	rcu_read_unlock();
2631
2632	return xdst;
2633	}
2634
2635	static void xfrm_init_path(struct xfrm_dst path, struct* dst_entry *dst,
2636	int nfheader_len)
2637	{
2638	if (dst->ops->family == AF_INET6) {
2639	path->path_cookie = rt6_get_cookie(dst_rt6_info(dst));
2640	path->u.rt6.rt6i_nfheader_len = nfheader_len;
2641	}
2642	}
2643
2644	static inline int xfrm_fill_dst(struct xfrm_dst xdst, struct* net_device *dev,
2645	const struct flowi *fl)
2646	{
2647	const struct xfrm_policy_afinfo *afinfo =
2648	xfrm_policy_get_afinfo(family: xdst->u.dst.ops->family);
2649	int err;
2650
2651	if (!afinfo)
2652	return -EINVAL;
2653
2654	err = afinfo->fill_dst(xdst, dev, fl);
2655
2656	rcu_read_unlock();
2657
2658	return err;
2659	}
2660
2661
2662	/ Allocate chain of dst_entry's, attach known xfrm's, calculate*
2663	* all the metrics... Shortly, bundle a bundle.
2664	*/
2665
2666	static struct dst_entry xfrm_bundle_create(struct* xfrm_policy *policy,
2667	struct xfrm_state **xfrm,
2668	struct xfrm_dst **bundle,
2669	int nx,
2670	const struct flowi *fl,
2671	struct dst_entry *dst)
2672	{
2673	const struct xfrm_state_afinfo *afinfo;
2674	const struct xfrm_mode *inner_mode;
2675	struct net *net = xp_net(xp: policy);
2676	unsigned long now = jiffies;
2677	struct net_device *dev;
2678	struct xfrm_dst *xdst_prev = NULL;
2679	struct xfrm_dst *xdst0 = NULL;
2680	int i = `0`;
2681	int err;
2682	int header_len = `0`;
2683	int nfheader_len = `0`;
2684	int trailer_len = `0`;
2685	int family = policy->selector.family;
2686	xfrm_address_t saddr, daddr;
2687	dscp_t dscp;
2688
2689	xfrm_flowi_addr_get(fl, saddr: &saddr, daddr: &daddr, family);
2690
2691	dscp = xfrm_get_dscp(fl, family);
2692
2693	dst_hold(dst);
2694
2695	for (; i < nx; i++) {
2696	struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
2697	struct dst_entry *dst1 = &xdst->u.dst;
2698
2699	err = PTR_ERR(ptr: xdst);
2700	if (IS_ERR(ptr: xdst)) {
2701	dst_release(dst);
2702	goto put_states;
2703	}
2704
2705	bundle[i] = xdst;
2706	if (!xdst_prev)
2707	xdst0 = xdst;
2708	else
2709	/ Ref count is taken during xfrm_alloc_dst()*
2710	* No need to do dst_clone() on dst1
2711	*/
2712	xfrm_dst_set_child(xdst: xdst_prev, child: &xdst->u.dst);
2713
2714	if (xfrm[i]->sel.family == AF_UNSPEC) {
2715	inner_mode = xfrm_ip2inner_mode(x: xfrm[i],
2716	ipproto: xfrm_af2proto(family));
2717	if (!inner_mode) {
2718	err = -EAFNOSUPPORT;
2719	dst_release(dst);
2720	goto put_states;
2721	}
2722	} else
2723	inner_mode = &xfrm[i]->inner_mode;
2724
2725	xdst->route = dst;
2726	dst_copy_metrics(dest: dst1, src: dst);
2727
2728	if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
2729	__u32 mark = `0`;
2730	int oif;
2731
2732	if (xfrm[i]->props.smark.v \|\| xfrm[i]->props.smark.m)
2733	mark = xfrm_smark_get(mark: fl->flowi_mark, x: xfrm[i]);
2734
2735	if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET)
2736	family = xfrm[i]->props.family;
2737
2738	oif = fl->flowi_oif ? : fl->flowi_l3mdev;
2739	dst = xfrm_dst_lookup(x: xfrm[i], dscp, oif, prev_saddr: &saddr,
2740	prev_daddr: &daddr, family, mark);
2741	err = PTR_ERR(ptr: dst);
2742	if (IS_ERR(ptr: dst))
2743	goto put_states;
2744	} else
2745	dst_hold(dst);
2746
2747	dst1->xfrm = xfrm[i];
2748	xdst->xfrm_genid = xfrm[i]->genid;
2749
2750	dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2751	dst1->lastuse = now;
2752
2753	dst1->input = dst_discard;
2754
2755	if (xfrm[i]->mode_cbs && xfrm[i]->mode_cbs->output) {
2756	dst1->output = xfrm[i]->mode_cbs->output;
2757	} else {
2758	rcu_read_lock();
2759	afinfo = xfrm_state_afinfo_get_rcu(family: inner_mode->family);
2760	if (likely(afinfo))
2761	dst1->output = afinfo->output;
2762	else
2763	dst1->output = dst_discard_out;
2764	rcu_read_unlock();
2765	}
2766
2767	xdst_prev = xdst;
2768
2769	header_len += xfrm[i]->props.header_len;
2770	if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
2771	nfheader_len += xfrm[i]->props.header_len;
2772	trailer_len += xfrm[i]->props.trailer_len;
2773	}
2774
2775	xfrm_dst_set_child(xdst: xdst_prev, child: dst);
2776	xdst0->path = dst;
2777
2778	err = -ENODEV;
2779	dev = dst->dev;
2780	if (!dev)
2781	goto free_dst;
2782
2783	xfrm_init_path(path: xdst0, dst, nfheader_len);
2784	xfrm_init_pmtu(bundle, nr: nx);
2785
2786	for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
2787	xdst_prev = (struct xfrm_dst *) xfrm_dst_child(dst: &xdst_prev->u.dst)) {
2788	err = xfrm_fill_dst(xdst: xdst_prev, dev, fl);
2789	if (err)
2790	goto free_dst;
2791
2792	xdst_prev->u.dst.header_len = header_len;
2793	xdst_prev->u.dst.trailer_len = trailer_len;
2794	header_len -= xdst_prev->u.dst.xfrm->props.header_len;
2795	trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
2796	}
2797
2798	return &xdst0->u.dst;
2799
2800	put_states:
2801	for (; i < nx; i++)
2802	xfrm_state_put(x: xfrm[i]);
2803	free_dst:
2804	if (xdst0)
2805	dst_release_immediate(dst: &xdst0->u.dst);
2806
2807	return ERR_PTR(error: err);
2808	}
2809
2810	static int xfrm_expand_policies(const struct flowi *fl, u16 family,
2811	struct xfrm_policy **pols,
2812	int num_pols, int* *num_xfrms)
2813	{
2814	int i;
2815
2816	if (*num_pols == `0` \|\| !pols[`0`]) {
2817	*num_pols = `0`;
2818	*num_xfrms = `0`;
2819	return `0`;
2820	}
2821	if (IS_ERR(ptr: pols[`0`])) {
2822	*num_pols = `0`;
2823	return PTR_ERR(ptr: pols[`0`]);
2824	}
2825
2826	*num_xfrms = pols[`0`]->xfrm_nr;
2827
2828	#ifdef CONFIG_XFRM_SUB_POLICY
2829	if (pols[`0`]->action == XFRM_POLICY_ALLOW &&
2830	pols[`0`]->type != XFRM_POLICY_TYPE_MAIN) {
2831	pols[`1`] = xfrm_policy_lookup_bytype(xp_net(pols[`0`]),
2832	XFRM_POLICY_TYPE_MAIN,
2833	fl, family,
2834	XFRM_POLICY_OUT,
2835	pols[`0`]->if_id);
2836	if (pols[`1`]) {
2837	if (IS_ERR(pols[`1`])) {
2838	xfrm_pols_put(pols, *num_pols);
2839	*num_pols = `0`;
2840	return PTR_ERR(pols[`1`]);
2841	}
2842	(*num_pols)++;
2843	(*num_xfrms) += pols[`1`]->xfrm_nr;
2844	}
2845	}
2846	#endif
2847	for (i = `0`; i < *num_pols; i++) {
2848	if (pols[i]->action != XFRM_POLICY_ALLOW) {
2849	*num_xfrms = -`1`;
2850	break;
2851	}
2852	}
2853
2854	return `0`;
2855
2856	}
2857
2858	static struct xfrm_dst *
2859	xfrm_resolve_and_create_bundle(struct xfrm_policy *pols, int* num_pols,
2860	const struct flowi *fl, u16 family,
2861	struct dst_entry *dst_orig)
2862	{
2863	struct net *net = xp_net(xp: pols[`0`]);
2864	struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
2865	struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2866	struct xfrm_dst *xdst;
2867	struct dst_entry *dst;
2868	int err;
2869
2870	/ Try to instantiate a bundle /
2871	err = xfrm_tmpl_resolve(pols, npols: num_pols, fl, xfrm, family);
2872	if (err <= `0`) {
2873	if (err == `0`)
2874	return NULL;
2875
2876	if (err != -EAGAIN)
2877	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2878	return ERR_PTR(error: err);
2879	}
2880
2881	dst = xfrm_bundle_create(policy: pols[`0`], xfrm, bundle, nx: err, fl, dst: dst_orig);
2882	if (IS_ERR(ptr: dst)) {
2883	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
2884	return ERR_CAST(ptr: dst);
2885	}
2886
2887	xdst = (struct xfrm_dst *)dst;
2888	xdst->num_xfrms = err;
2889	xdst->num_pols = num_pols;
2890	memcpy(to: xdst->pols, from: pols, len: sizeof(struct xfrm_policy ) num_pols);
2891	xdst->policy_genid = atomic_read(v: &pols[`0`]->genid);
2892
2893	return xdst;
2894	}
2895
2896	static void xfrm_policy_queue_process(struct timer_list *t)
2897	{
2898	struct sk_buff *skb;
2899	struct sock *sk;
2900	struct dst_entry *dst;
2901	struct xfrm_policy *pol = timer_container_of(pol, t, polq.hold_timer);
2902	struct net *net = xp_net(xp: pol);
2903	struct xfrm_policy_queue *pq = &pol->polq;
2904	struct flowi fl;
2905	struct sk_buff_head list;
2906	__u32 skb_mark;
2907
2908	spin_lock(lock: &pq->hold_queue.lock);
2909	skb = skb_peek(list_: &pq->hold_queue);
2910	if (!skb) {
2911	spin_unlock(lock: &pq->hold_queue.lock);
2912	goto out;
2913	}
2914	dst = skb_dst(skb);
2915	sk = skb->sk;
2916
2917	/ Fixup the mark to support VTI. /
2918	skb_mark = skb->mark;
2919	skb->mark = pol->mark.v;
2920	xfrm_decode_session(net, skb, fl: &fl, family: dst->ops->family);
2921	skb->mark = skb_mark;
2922	spin_unlock(lock: &pq->hold_queue.lock);
2923
2924	dst_hold(dst: xfrm_dst_path(dst));
2925	dst = xfrm_lookup(net, dst_orig: xfrm_dst_path(dst), fl: &fl, sk, flags: XFRM_LOOKUP_QUEUE);
2926	if (IS_ERR(ptr: dst))
2927	goto purge_queue;
2928
2929	if (dst->flags & DST_XFRM_QUEUE) {
2930	dst_release(dst);
2931
2932	if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
2933	goto purge_queue;
2934
2935	pq->timeout = pq->timeout << `1`;
2936	if (!mod_timer(timer: &pq->hold_timer, expires: jiffies + pq->timeout))
2937	xfrm_pol_hold(policy: pol);
2938	goto out;
2939	}
2940
2941	dst_release(dst);
2942
2943	__skb_queue_head_init(list: &list);
2944
2945	spin_lock(lock: &pq->hold_queue.lock);
2946	pq->timeout = `0`;
2947	skb_queue_splice_init(list: &pq->hold_queue, head: &list);
2948	spin_unlock(lock: &pq->hold_queue.lock);
2949
2950	while (!skb_queue_empty(list: &list)) {
2951	skb = __skb_dequeue(list: &list);
2952
2953	/ Fixup the mark to support VTI. /
2954	skb_mark = skb->mark;
2955	skb->mark = pol->mark.v;
2956	xfrm_decode_session(net, skb, fl: &fl, family: skb_dst(skb)->ops->family);
2957	skb->mark = skb_mark;
2958
2959	dst_hold(dst: xfrm_dst_path(dst: skb_dst(skb)));
2960	dst = xfrm_lookup(net, dst_orig: xfrm_dst_path(dst: skb_dst(skb)), fl: &fl, sk: skb->sk, flags: `0`);
2961	if (IS_ERR(ptr: dst)) {
2962	kfree_skb(skb);
2963	continue;
2964	}
2965
2966	nf_reset_ct(skb);
2967	skb_dst_drop(skb);
2968	skb_dst_set(skb, dst);
2969
2970	dst_output(net, sk: skb_to_full_sk(skb), skb);
2971	}
2972
2973	out:
2974	xfrm_pol_put(policy: pol);
2975	return;
2976
2977	purge_queue:
2978	pq->timeout = `0`;
2979	skb_queue_purge(list: &pq->hold_queue);
2980	xfrm_pol_put(policy: pol);
2981	}
2982
2983	static int xdst_queue_output(struct net net, struct* sock sk, struct* sk_buff *skb)
2984	{
2985	unsigned long sched_next;
2986	struct dst_entry *dst = skb_dst(skb);
2987	struct xfrm_dst xdst = (struct* xfrm_dst *) dst;
2988	struct xfrm_policy *pol = xdst->pols[`0`];
2989	struct xfrm_policy_queue *pq = &pol->polq;
2990
2991	if (unlikely(skb_fclone_busy(sk, skb))) {
2992	kfree_skb(skb);
2993	return `0`;
2994	}
2995
2996	if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2997	kfree_skb(skb);
2998	return -EAGAIN;
2999	}
3000
3001	skb_dst_force(skb);
3002
3003	spin_lock_bh(lock: &pq->hold_queue.lock);
3004
3005	if (!pq->timeout)
3006	pq->timeout = XFRM_QUEUE_TMO_MIN;
3007
3008	sched_next = jiffies + pq->timeout;
3009
3010	if (timer_delete(timer: &pq->hold_timer)) {
3011	if (time_before(pq->hold_timer.expires, sched_next))
3012	sched_next = pq->hold_timer.expires;
3013	xfrm_pol_put(policy: pol);
3014	}
3015
3016	__skb_queue_tail(list: &pq->hold_queue, newsk: skb);
3017	if (!mod_timer(timer: &pq->hold_timer, expires: sched_next))
3018	xfrm_pol_hold(policy: pol);
3019
3020	spin_unlock_bh(lock: &pq->hold_queue.lock);
3021
3022	return `0`;
3023	}
3024
3025	static struct xfrm_dst xfrm_create_dummy_bundle(struct* net *net,
3026	struct xfrm_flo *xflo,
3027	const struct flowi *fl,
3028	int num_xfrms,
3029	u16 family)
3030	{
3031	int err;
3032	struct net_device *dev;
3033	struct dst_entry *dst;
3034	struct dst_entry *dst1;
3035	struct xfrm_dst *xdst;
3036
3037	xdst = xfrm_alloc_dst(net, family);
3038	if (IS_ERR(ptr: xdst))
3039	return xdst;
3040
3041	if (!(xflo->flags & XFRM_LOOKUP_QUEUE) \|\|
3042	net->xfrm.sysctl_larval_drop \|\|
3043	num_xfrms <= `0`)
3044	return xdst;
3045
3046	dst = xflo->dst_orig;
3047	dst1 = &xdst->u.dst;
3048	dst_hold(dst);
3049	xdst->route = dst;
3050
3051	dst_copy_metrics(dest: dst1, src: dst);
3052
3053	dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
3054	dst1->flags \|= DST_XFRM_QUEUE;
3055	dst1->lastuse = jiffies;
3056
3057	dst1->input = dst_discard;
3058	dst1->output = xdst_queue_output;
3059
3060	dst_hold(dst);
3061	xfrm_dst_set_child(xdst, child: dst);
3062	xdst->path = dst;
3063
3064	xfrm_init_path(path: (struct xfrm_dst *)dst1, dst, nfheader_len: `0`);
3065
3066	err = -ENODEV;
3067	dev = dst->dev;
3068	if (!dev)
3069	goto free_dst;
3070
3071	err = xfrm_fill_dst(xdst, dev, fl);
3072	if (err)
3073	goto free_dst;
3074
3075	out:
3076	return xdst;
3077
3078	free_dst:
3079	dst_release(dst: dst1);
3080	xdst = ERR_PTR(error: err);
3081	goto out;
3082	}
3083
3084	static struct xfrm_dst xfrm_bundle_lookup(struct* net *net,
3085	const struct flowi *fl,
3086	u16 family, u8 dir,
3087	struct xfrm_flo *xflo, u32 if_id)
3088	{
3089	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3090	int num_pols = `0`, num_xfrms = `0`, err;
3091	struct xfrm_dst *xdst;
3092
3093	/ Resolve policies to use if we couldn't get them from*
3094	* previous cache entry */
3095	num_pols = `1`;
3096	pols[`0`] = xfrm_policy_lookup(net, fl, family, dir, if_id);
3097	err = xfrm_expand_policies(fl, family, pols,
3098	num_pols: &num_pols, num_xfrms: &num_xfrms);
3099	if (err < `0`)
3100	goto inc_error;
3101	if (num_pols == `0`)
3102	return NULL;
3103	if (num_xfrms <= `0`)
3104	goto make_dummy_bundle;
3105
3106	xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
3107	dst_orig: xflo->dst_orig);
3108	if (IS_ERR(ptr: xdst)) {
3109	err = PTR_ERR(ptr: xdst);
3110	if (err == -EREMOTE) {
3111	xfrm_pols_put(pols, npols: num_pols);
3112	return NULL;
3113	}
3114
3115	if (err != -EAGAIN)
3116	goto error;
3117	goto make_dummy_bundle;
3118	} else if (xdst == NULL) {
3119	num_xfrms = `0`;
3120	goto make_dummy_bundle;
3121	}
3122
3123	return xdst;
3124
3125	make_dummy_bundle:
3126	/ We found policies, but there's no bundles to instantiate:*
3127	* either because the policy blocks, has no transformations or
3128	* we could not build template (no xfrm_states).*/
3129	xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
3130	if (IS_ERR(ptr: xdst)) {
3131	xfrm_pols_put(pols, npols: num_pols);
3132	return ERR_CAST(ptr: xdst);
3133	}
3134	xdst->num_pols = num_pols;
3135	xdst->num_xfrms = num_xfrms;
3136	memcpy(to: xdst->pols, from: pols, len: sizeof(struct xfrm_policy ) num_pols);
3137
3138	return xdst;
3139
3140	inc_error:
3141	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
3142	error:
3143	xfrm_pols_put(pols, npols: num_pols);
3144	return ERR_PTR(error: err);
3145	}
3146
3147	static struct dst_entry make_blackhole(struct* net *net, u16 family,
3148	struct dst_entry *dst_orig)
3149	{
3150	const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
3151	struct dst_entry *ret;
3152
3153	if (!afinfo) {
3154	dst_release(dst: dst_orig);
3155	return ERR_PTR(error: -EINVAL);
3156	} else {
3157	ret = afinfo->blackhole_route(net, dst_orig);
3158	}
3159	rcu_read_unlock();
3160
3161	return ret;
3162	}
3163
3164	/ Finds/creates a bundle for given flow and if_id*
3165	*
3166	* At the moment we eat a raw IP route. Mostly to speed up lookups
3167	* on interfaces with disabled IPsec.
3168	*
3169	* xfrm_lookup uses an if_id of 0 by default, and is provided for
3170	* compatibility
3171	*/
3172	struct dst_entry xfrm_lookup_with_ifid(struct* net *net,
3173	struct dst_entry *dst_orig,
3174	const struct flowi *fl,
3175	const struct sock *sk,
3176	int flags, u32 if_id)
3177	{
3178	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3179	struct xfrm_dst *xdst;
3180	struct dst_entry dst, route;
3181	u16 family = dst_orig->ops->family;
3182	u8 dir = XFRM_POLICY_OUT;
3183	int i, err, num_pols, num_xfrms = `0`, drop_pols = `0`;
3184
3185	dst = NULL;
3186	xdst = NULL;
3187	route = NULL;
3188
3189	sk = sk_const_to_full_sk(sk);
3190	if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
3191	num_pols = `1`;
3192	pols[`0`] = xfrm_sk_policy_lookup(sk, dir: XFRM_POLICY_OUT, fl, family,
3193	if_id);
3194	err = xfrm_expand_policies(fl, family, pols,
3195	num_pols: &num_pols, num_xfrms: &num_xfrms);
3196	if (err < `0`)
3197	goto dropdst;
3198
3199	if (num_pols) {
3200	if (num_xfrms <= `0`) {
3201	drop_pols = num_pols;
3202	goto no_transform;
3203	}
3204
3205	xdst = xfrm_resolve_and_create_bundle(
3206	pols, num_pols, fl,
3207	family, dst_orig);
3208
3209	if (IS_ERR(ptr: xdst)) {
3210	xfrm_pols_put(pols, npols: num_pols);
3211	err = PTR_ERR(ptr: xdst);
3212	if (err == -EREMOTE)
3213	goto nopol;
3214
3215	goto dropdst;
3216	} else if (xdst == NULL) {
3217	num_xfrms = `0`;
3218	drop_pols = num_pols;
3219	goto no_transform;
3220	}
3221
3222	route = xdst->route;
3223	}
3224	}
3225
3226	if (xdst == NULL) {
3227	struct xfrm_flo xflo;
3228
3229	xflo.dst_orig = dst_orig;
3230	xflo.flags = flags;
3231
3232	/ To accelerate a bit... /
3233	if (!if_id && ((dst_orig->flags & DST_NOXFRM) \|\|
3234	!net->xfrm.policy_count[XFRM_POLICY_OUT]))
3235	goto nopol;
3236
3237	xdst = xfrm_bundle_lookup(net, fl, family, dir, xflo: &xflo, if_id);
3238	if (xdst == NULL)
3239	goto nopol;
3240	if (IS_ERR(ptr: xdst)) {
3241	err = PTR_ERR(ptr: xdst);
3242	goto dropdst;
3243	}
3244
3245	num_pols = xdst->num_pols;
3246	num_xfrms = xdst->num_xfrms;
3247	memcpy(to: pols, from: xdst->pols, len: sizeof(struct xfrm_policy ) num_pols);
3248	route = xdst->route;
3249	}
3250
3251	dst = &xdst->u.dst;
3252	if (route == NULL && num_xfrms > `0`) {
3253	/ The only case when xfrm_bundle_lookup() returns a*
3254	* bundle with null route, is when the template could
3255	* not be resolved. It means policies are there, but
3256	* bundle could not be created, since we don't yet
3257	* have the xfrm_state's. We need to wait for KM to
3258	* negotiate new SA's or bail out with error.*/
3259	if (net->xfrm.sysctl_larval_drop) {
3260	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3261	err = -EREMOTE;
3262	goto error;
3263	}
3264
3265	err = -EAGAIN;
3266
3267	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3268	goto error;
3269	}
3270
3271	no_transform:
3272	if (num_pols == `0`)
3273	goto nopol;
3274
3275	if ((flags & XFRM_LOOKUP_ICMP) &&
3276	!(pols[`0`]->flags & XFRM_POLICY_ICMP)) {
3277	err = -ENOENT;
3278	goto error;
3279	}
3280
3281	for (i = `0`; i < num_pols; i++)
3282	WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds());
3283
3284	if (num_xfrms < `0`) {
3285	/ Prohibit the flow /
3286	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
3287	err = -EPERM;
3288	goto error;
3289	} else if (num_xfrms > `0`) {
3290	/ Flow transformed /
3291	dst_release(dst: dst_orig);
3292	} else {
3293	/ Flow passes untransformed /
3294	dst_release(dst);
3295	dst = dst_orig;
3296	}
3297
3298	ok:
3299	xfrm_pols_put(pols, npols: drop_pols);
3300	if (dst->xfrm &&
3301	(dst->xfrm->props.mode == XFRM_MODE_TUNNEL \|\|
3302	dst->xfrm->props.mode == XFRM_MODE_IPTFS))
3303	dst->flags \|= DST_XFRM_TUNNEL;
3304	return dst;
3305
3306	nopol:
3307	if ((!dst_orig->dev \|\| !(dst_orig->dev->flags & IFF_LOOPBACK)) &&
3308	net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3309	err = -EPERM;
3310	goto error;
3311	}
3312	if (!(flags & XFRM_LOOKUP_ICMP)) {
3313	dst = dst_orig;
3314	goto ok;
3315	}
3316	err = -ENOENT;
3317	error:
3318	dst_release(dst);
3319	dropdst:
3320	if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
3321	dst_release(dst: dst_orig);
3322	xfrm_pols_put(pols, npols: drop_pols);
3323	return ERR_PTR(error: err);
3324	}
3325	EXPORT_SYMBOL(xfrm_lookup_with_ifid);
3326
3327	/ Main function: finds/creates a bundle for given flow.*
3328	*
3329	* At the moment we eat a raw IP route. Mostly to speed up lookups
3330	* on interfaces with disabled IPsec.
3331	*/
3332	struct dst_entry xfrm_lookup(struct* net net, struct* dst_entry *dst_orig,
3333	const struct flowi fl, const* struct sock *sk,
3334	int flags)
3335	{
3336	return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, `0`);
3337	}
3338	EXPORT_SYMBOL(xfrm_lookup);
3339
3340	/ Callers of xfrm_lookup_route() must ensure a call to dst_output().*
3341	* Otherwise we may send out blackholed packets.
3342	*/
3343	struct dst_entry xfrm_lookup_route(struct* net net, struct* dst_entry *dst_orig,
3344	const struct flowi *fl,
3345	const struct sock sk, int* flags)
3346	{
3347	struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
3348	flags \| XFRM_LOOKUP_QUEUE \|
3349	XFRM_LOOKUP_KEEP_DST_REF);
3350
3351	if (PTR_ERR(ptr: dst) == -EREMOTE)
3352	return make_blackhole(net, family: dst_orig->ops->family, dst_orig);
3353
3354	if (IS_ERR(ptr: dst))
3355	dst_release(dst: dst_orig);
3356
3357	return dst;
3358	}
3359	EXPORT_SYMBOL(xfrm_lookup_route);
3360
3361	static inline int
3362	xfrm_secpath_reject(int idx, struct sk_buff skb, const* struct flowi *fl)
3363	{
3364	struct sec_path *sp = skb_sec_path(skb);
3365	struct xfrm_state *x;
3366
3367	if (!sp \|\| idx < `0` \|\| idx >= sp->len)
3368	return `0`;
3369	x = sp->xvec[idx];
3370	if (!x->type->reject)
3371	return `0`;
3372	return x->type->reject(x, skb, fl);
3373	}
3374
3375	/ When skb is transformed back to its "native" form, we have to*
3376	* check policy restrictions. At the moment we make this in maximally
3377	* stupid way. Shame on me. :-) Of course, connected sockets must
3378	* have policy cached at them.
3379	*/
3380
3381	static inline int
3382	xfrm_state_ok(const struct xfrm_tmpl tmpl, const* struct xfrm_state *x,
3383	unsigned short family, u32 if_id)
3384	{
3385	if (xfrm_state_kern(x))
3386	return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, family: tmpl->encap_family);
3387	return x->id.proto == tmpl->id.proto &&
3388	(x->id.spi == tmpl->id.spi \|\| !tmpl->id.spi) &&
3389	(x->props.reqid == tmpl->reqid \|\| !tmpl->reqid) &&
3390	x->props.mode == tmpl->mode &&
3391	(tmpl->allalgs \|\| (tmpl->aalgos & (`1`<<x->props.aalgo)) \|\|
3392	!(xfrm_id_proto_match(proto: tmpl->id.proto, IPSEC_PROTO_ANY))) &&
3393	!(x->props.mode != XFRM_MODE_TRANSPORT &&
3394	xfrm_state_addr_cmp(tmpl, x, family)) &&
3395	(if_id == `0` \|\| if_id == x->if_id);
3396	}
3397
3398	/*
3399	* 0 or more than 0 is returned when validation is succeeded (either bypass
3400	* because of optional transport mode, or next index of the matched secpath
3401	* state with the template.
3402	* -1 is returned when no matching template is found.
3403	* Otherwise "-2 - errored_index" is returned.
3404	*/
3405	static inline int
3406	xfrm_policy_ok(const struct xfrm_tmpl tmpl, const* struct sec_path sp, int* start,
3407	unsigned short family, u32 if_id)
3408	{
3409	int idx = start;
3410
3411	if (tmpl->optional) {
3412	if (tmpl->mode == XFRM_MODE_TRANSPORT)
3413	return start;
3414	} else
3415	start = -`1`;
3416	for (; idx < sp->len; idx++) {
3417	if (xfrm_state_ok(tmpl, x: sp->xvec[idx], family, if_id))
3418	return ++idx;
3419	if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
3420	if (idx < sp->verified_cnt) {
3421	/ Secpath entry previously verified, consider optional and*
3422	* continue searching
3423	*/
3424	continue;
3425	}
3426
3427	if (start == -`1`)
3428	start = -`2`-idx;
3429	break;
3430	}
3431	}
3432	return start;
3433	}
3434
3435	static void
3436	decode_session4(const struct xfrm_flow_keys flkeys, struct* flowi *fl, bool reverse)
3437	{
3438	struct flowi4 *fl4 = &fl->u.ip4;
3439
3440	memset(s: fl4, c: `0`, n: sizeof(struct flowi4));
3441
3442	if (reverse) {
3443	fl4->saddr = flkeys->addrs.ipv4.dst;
3444	fl4->daddr = flkeys->addrs.ipv4.src;
3445	fl4->fl4_sport = flkeys->ports.dst;
3446	fl4->fl4_dport = flkeys->ports.src;
3447	} else {
3448	fl4->saddr = flkeys->addrs.ipv4.src;
3449	fl4->daddr = flkeys->addrs.ipv4.dst;
3450	fl4->fl4_sport = flkeys->ports.src;
3451	fl4->fl4_dport = flkeys->ports.dst;
3452	}
3453
3454	switch (flkeys->basic.ip_proto) {
3455	case IPPROTO_GRE:
3456	fl4->fl4_gre_key = flkeys->gre.keyid;
3457	break;
3458	case IPPROTO_ICMP:
3459	fl4->fl4_icmp_type = flkeys->icmp.type;
3460	fl4->fl4_icmp_code = flkeys->icmp.code;
3461	break;
3462	}
3463
3464	fl4->flowi4_proto = flkeys->basic.ip_proto;
3465	fl4->flowi4_dscp = inet_dsfield_to_dscp(dsfield: flkeys->ip.tos);
3466	}
3467
3468	#if IS_ENABLED(CONFIG_IPV6)
3469	static void
3470	decode_session6(const struct xfrm_flow_keys flkeys, struct* flowi *fl, bool reverse)
3471	{
3472	struct flowi6 *fl6 = &fl->u.ip6;
3473
3474	memset(s: fl6, c: `0`, n: sizeof(struct flowi6));
3475
3476	if (reverse) {
3477	fl6->saddr = flkeys->addrs.ipv6.dst;
3478	fl6->daddr = flkeys->addrs.ipv6.src;
3479	fl6->fl6_sport = flkeys->ports.dst;
3480	fl6->fl6_dport = flkeys->ports.src;
3481	} else {
3482	fl6->saddr = flkeys->addrs.ipv6.src;
3483	fl6->daddr = flkeys->addrs.ipv6.dst;
3484	fl6->fl6_sport = flkeys->ports.src;
3485	fl6->fl6_dport = flkeys->ports.dst;
3486	}
3487
3488	switch (flkeys->basic.ip_proto) {
3489	case IPPROTO_GRE:
3490	fl6->fl6_gre_key = flkeys->gre.keyid;
3491	break;
3492	case IPPROTO_ICMPV6:
3493	fl6->fl6_icmp_type = flkeys->icmp.type;
3494	fl6->fl6_icmp_code = flkeys->icmp.code;
3495	break;
3496	}
3497
3498	fl6->flowi6_proto = flkeys->basic.ip_proto;
3499	}
3500	#endif
3501
3502	int __xfrm_decode_session(struct net net, struct* sk_buff skb, struct* flowi *fl,
3503	unsigned int family, int reverse)
3504	{
3505	struct xfrm_flow_keys flkeys;
3506
3507	memset(s: &flkeys, c: `0`, n: sizeof(flkeys));
3508	__skb_flow_dissect(net, skb, flow_dissector: &xfrm_session_dissector, target_container: &flkeys,
3509	NULL, proto: `0`, nhoff: `0`, hlen: `0`, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
3510
3511	switch (family) {
3512	case AF_INET:
3513	decode_session4(flkeys: &flkeys, fl, reverse);
3514	break;
3515	#if IS_ENABLED(CONFIG_IPV6)
3516	case AF_INET6:
3517	decode_session6(flkeys: &flkeys, fl, reverse);
3518	break;
3519	#endif
3520	default:
3521	return -EAFNOSUPPORT;
3522	}
3523
3524	fl->flowi_mark = skb->mark;
3525	if (reverse) {
3526	fl->flowi_oif = skb->skb_iif;
3527	} else {
3528	int oif = `0`;
3529
3530	if (skb_dst(skb) && skb_dst(skb)->dev)
3531	oif = skb_dst(skb)->dev->ifindex;
3532
3533	fl->flowi_oif = oif;
3534	}
3535
3536	return security_xfrm_decode_session(skb, secid: &fl->flowi_secid);
3537	}
3538	EXPORT_SYMBOL(__xfrm_decode_session);
3539
3540	static inline int secpath_has_nontransport(const struct sec_path sp, int* k, int *idxp)
3541	{
3542	for (; k < sp->len; k++) {
3543	if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
3544	*idxp = k;
3545	return `1`;
3546	}
3547	}
3548
3549	return `0`;
3550	}
3551
3552	static bool icmp_err_packet(const struct flowi fl, unsigned* short family)
3553	{
3554	const struct flowi4 *fl4 = &fl->u.ip4;
3555
3556	if (family == AF_INET &&
3557	fl4->flowi4_proto == IPPROTO_ICMP &&
3558	(fl4->fl4_icmp_type == ICMP_DEST_UNREACH \|\|
3559	fl4->fl4_icmp_type == ICMP_TIME_EXCEEDED))
3560	return true;
3561
3562	#if IS_ENABLED(CONFIG_IPV6)
3563	if (family == AF_INET6) {
3564	const struct flowi6 *fl6 = &fl->u.ip6;
3565
3566	if (fl6->flowi6_proto == IPPROTO_ICMPV6 &&
3567	(fl6->fl6_icmp_type == ICMPV6_DEST_UNREACH \|\|
3568	fl6->fl6_icmp_type == ICMPV6_PKT_TOOBIG \|\|
3569	fl6->fl6_icmp_type == ICMPV6_TIME_EXCEED))
3570	return true;
3571	}
3572	#endif
3573	return false;
3574	}
3575
3576	static bool xfrm_icmp_flow_decode(struct sk_buff skb, unsigned* short family,
3577	const struct flowi fl, struct* flowi *fl1)
3578	{
3579	bool ret = true;
3580	struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
3581	int hl = family == AF_INET ? (sizeof(struct iphdr) + sizeof(struct icmphdr)) :
3582	(sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr));
3583
3584	if (!newskb)
3585	return true;
3586
3587	if (!pskb_pull(skb: newskb, len: hl))
3588	goto out;
3589
3590	skb_reset_network_header(skb: newskb);
3591
3592	if (xfrm_decode_session_reverse(net: dev_net(dev: skb->dev), skb: newskb, fl: fl1, family) < `0`)
3593	goto out;
3594
3595	fl1->flowi_oif = fl->flowi_oif;
3596	fl1->flowi_mark = fl->flowi_mark;
3597	fl1->flowi_dscp = fl->flowi_dscp;
3598	nf_nat_decode_session(skb: newskb, fl: fl1, family);
3599	ret = false;
3600
3601	out:
3602	consume_skb(skb: newskb);
3603	return ret;
3604	}
3605
3606	static bool xfrm_selector_inner_icmp_match(struct sk_buff skb, unsigned* short family,
3607	const struct xfrm_selector *sel,
3608	const struct flowi *fl)
3609	{
3610	bool ret = false;
3611
3612	if (icmp_err_packet(fl, family)) {
3613	struct flowi fl1;
3614
3615	if (xfrm_icmp_flow_decode(skb, family, fl, fl1: &fl1))
3616	return ret;
3617
3618	ret = xfrm_selector_match(sel, fl: &fl1, family);
3619	}
3620
3621	return ret;
3622	}
3623
3624	static inline struct
3625	xfrm_policy xfrm_in_fwd_icmp(struct* sk_buff *skb,
3626	const struct flowi fl, unsigned* short family,
3627	u32 if_id)
3628	{
3629	struct xfrm_policy *pol = NULL;
3630
3631	if (icmp_err_packet(fl, family)) {
3632	struct flowi fl1;
3633	struct net *net = dev_net(dev: skb->dev);
3634
3635	if (xfrm_icmp_flow_decode(skb, family, fl, fl1: &fl1))
3636	return pol;
3637
3638	pol = xfrm_policy_lookup(net, fl: &fl1, family, dir: XFRM_POLICY_FWD, if_id);
3639	if (IS_ERR(ptr: pol))
3640	pol = NULL;
3641	}
3642
3643	return pol;
3644	}
3645
3646	static inline struct
3647	dst_entry xfrm_out_fwd_icmp(struct* sk_buff skb, struct* flowi *fl,
3648	unsigned short family, struct dst_entry *dst)
3649	{
3650	if (icmp_err_packet(fl, family)) {
3651	struct net *net = dev_net(dev: skb->dev);
3652	struct dst_entry *dst2;
3653	struct flowi fl1;
3654
3655	if (xfrm_icmp_flow_decode(skb, family, fl, fl1: &fl1))
3656	return dst;
3657
3658	dst_hold(dst);
3659
3660	dst2 = xfrm_lookup(net, dst, &fl1, NULL, (XFRM_LOOKUP_QUEUE \| XFRM_LOOKUP_ICMP));
3661
3662	if (IS_ERR(ptr: dst2))
3663	return dst;
3664
3665	if (dst2->xfrm) {
3666	dst_release(dst);
3667	dst = dst2;
3668	} else {
3669	dst_release(dst: dst2);
3670	}
3671	}
3672
3673	return dst;
3674	}
3675
3676	int __xfrm_policy_check(struct sock sk, int* dir, struct sk_buff *skb,
3677	unsigned short family)
3678	{
3679	struct net *net = dev_net(dev: skb->dev);
3680	struct xfrm_policy *pol;
3681	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3682	int npols = `0`;
3683	int xfrm_nr;
3684	int pi;
3685	int reverse;
3686	struct flowi fl;
3687	int xerr_idx = -`1`;
3688	const struct xfrm_if_cb *ifcb;
3689	struct sec_path *sp;
3690	u32 if_id = `0`;
3691
3692	rcu_read_lock();
3693	ifcb = xfrm_if_get_cb();
3694
3695	if (ifcb) {
3696	struct xfrm_if_decode_session_result r;
3697
3698	if (ifcb->decode_session(skb, family, &r)) {
3699	if_id = r.if_id;
3700	net = r.net;
3701	}
3702	}
3703	rcu_read_unlock();
3704
3705	reverse = dir & ~XFRM_POLICY_MASK;
3706	dir &= XFRM_POLICY_MASK;
3707
3708	if (__xfrm_decode_session(net, skb, &fl, family, reverse) < `0`) {
3709	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
3710	return `0`;
3711	}
3712
3713	nf_nat_decode_session(skb, fl: &fl, family);
3714
3715	/ First, check used SA against their selectors. /
3716	sp = skb_sec_path(skb);
3717	if (sp) {
3718	int i;
3719
3720	for (i = sp->len - `1`; i >= `0`; i--) {
3721	struct xfrm_state *x = sp->xvec[i];
3722	int ret = `0`;
3723
3724	if (!xfrm_selector_match(sel: &x->sel, fl: &fl, family)) {
3725	ret = `1`;
3726	if (x->props.flags & XFRM_STATE_ICMP &&
3727	xfrm_selector_inner_icmp_match(skb, family, sel: &x->sel, fl: &fl))
3728	ret = `0`;
3729	if (ret) {
3730	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
3731	return `0`;
3732	}
3733	}
3734	}
3735	}
3736
3737	pol = NULL;
3738	sk = sk_to_full_sk(sk);
3739	if (sk && sk->sk_policy[dir]) {
3740	pol = xfrm_sk_policy_lookup(sk, dir, fl: &fl, family, if_id);
3741	if (IS_ERR(ptr: pol)) {
3742	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3743	return `0`;
3744	}
3745	}
3746
3747	if (!pol)
3748	pol = xfrm_policy_lookup(net, fl: &fl, family, dir, if_id);
3749
3750	if (IS_ERR(ptr: pol)) {
3751	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3752	return `0`;
3753	}
3754
3755	if (!pol && dir == XFRM_POLICY_FWD)
3756	pol = xfrm_in_fwd_icmp(skb, fl: &fl, family, if_id);
3757
3758	if (!pol) {
3759	const bool is_crypto_offload = sp &&
3760	(xfrm_input_state(skb)->xso.type == XFRM_DEV_OFFLOAD_CRYPTO);
3761
3762	if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3763	XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3764	return `0`;
3765	}
3766
3767	if (sp && secpath_has_nontransport(sp, k: `0`, idxp: &xerr_idx) && !is_crypto_offload) {
3768	xfrm_secpath_reject(idx: xerr_idx, skb, fl: &fl);
3769	XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3770	return `0`;
3771	}
3772	return `1`;
3773	}
3774
3775	/ This lockless write can happen from different cpus. /
3776	WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds());
3777
3778	pols[`0`] = pol;
3779	npols++;
3780	#ifdef CONFIG_XFRM_SUB_POLICY
3781	if (pols[`0`]->type != XFRM_POLICY_TYPE_MAIN) {
3782	pols[`1`] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
3783	&fl, family,
3784	XFRM_POLICY_IN, if_id);
3785	if (pols[`1`]) {
3786	if (IS_ERR(pols[`1`])) {
3787	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3788	xfrm_pol_put(pols[`0`]);
3789	return `0`;
3790	}
3791	/ This write can happen from different cpus. /
3792	WRITE_ONCE(pols[`1`]->curlft.use_time,
3793	ktime_get_real_seconds());
3794	npols++;
3795	}
3796	}
3797	#endif
3798
3799	if (pol->action == XFRM_POLICY_ALLOW) {
3800	static struct sec_path dummy;
3801	struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
3802	struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
3803	struct xfrm_tmpl **tpp = tp;
3804	int ti = `0`;
3805	int i, k;
3806
3807	sp = skb_sec_path(skb);
3808	if (!sp)
3809	sp = &dummy;
3810
3811	for (pi = `0`; pi < npols; pi++) {
3812	if (pols[pi] != pol &&
3813	pols[pi]->action != XFRM_POLICY_ALLOW) {
3814	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3815	goto reject;
3816	}
3817	if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
3818	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
3819	goto reject_error;
3820	}
3821	for (i = `0`; i < pols[pi]->xfrm_nr; i++)
3822	tpp[ti++] = &pols[pi]->xfrm_vec[i];
3823	}
3824	xfrm_nr = ti;
3825
3826	if (npols > `1`) {
3827	xfrm_tmpl_sort(d: stp, s: tpp, n: xfrm_nr, family);
3828	tpp = stp;
3829	}
3830
3831	/ For each tunnel xfrm, find the first matching tmpl.*
3832	* For each tmpl before that, find corresponding xfrm.
3833	* Order is _important_. Later we will implement
3834	* some barriers, but at the moment barriers
3835	* are implied between each two transformations.
3836	* Upon success, marks secpath entries as having been
3837	* verified to allow them to be skipped in future policy
3838	* checks (e.g. nested tunnels).
3839	*/
3840	for (i = xfrm_nr-`1`, k = `0`; i >= `0`; i--) {
3841	k = xfrm_policy_ok(tmpl: tpp[i], sp, start: k, family, if_id);
3842	if (k < `0`) {
3843	if (k < -`1`)
3844	/ "-2 - errored_index" returned /
3845	xerr_idx = -(`2`+k);
3846	XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3847	goto reject;
3848	}
3849	}
3850
3851	if (secpath_has_nontransport(sp, k, idxp: &xerr_idx)) {
3852	XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3853	goto reject;
3854	}
3855
3856	xfrm_pols_put(pols, npols);
3857	sp->verified_cnt = k;
3858
3859	return `1`;
3860	}
3861	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3862
3863	reject:
3864	xfrm_secpath_reject(idx: xerr_idx, skb, fl: &fl);
3865	reject_error:
3866	xfrm_pols_put(pols, npols);
3867	return `0`;
3868	}
3869	EXPORT_SYMBOL(__xfrm_policy_check);
3870
3871	int __xfrm_route_forward(struct sk_buff skb, unsigned* short family)
3872	{
3873	struct net *net = dev_net(dev: skb->dev);
3874	struct flowi fl;
3875	struct dst_entry *dst;
3876	int res = `1`;
3877
3878	if (xfrm_decode_session(net, skb, fl: &fl, family) < `0`) {
3879	XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3880	return `0`;
3881	}
3882
3883	skb_dst_force(skb);
3884	dst = skb_dst(skb);
3885	if (!dst) {
3886	XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3887	return `0`;
3888	}
3889
3890	/ ignore return value from skb_dstref_steal, xfrm_lookup takes*
3891	* care of dropping the refcnt if needed.
3892	*/
3893	skb_dstref_steal(skb);
3894
3895	dst = xfrm_lookup(net, dst, &fl, NULL, XFRM_LOOKUP_QUEUE);
3896	if (IS_ERR(ptr: dst)) {
3897	res = `0`;
3898	dst = NULL;
3899	}
3900
3901	if (dst && !dst->xfrm)
3902	dst = xfrm_out_fwd_icmp(skb, fl: &fl, family, dst);
3903
3904	skb_dst_set(skb, dst);
3905	return res;
3906	}
3907	EXPORT_SYMBOL(__xfrm_route_forward);
3908
3909	/ Optimize later using cookies and generation ids. /
3910
3911	static struct dst_entry xfrm_dst_check(struct* dst_entry *dst, u32 cookie)
3912	{
3913	/ Code (such as __xfrm4_bundle_create()) sets dst->obsolete*
3914	* to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
3915	* get validated by dst_ops->check on every use. We do this
3916	* because when a normal route referenced by an XFRM dst is
3917	* obsoleted we do not go looking around for all parent
3918	* referencing XFRM dsts so that we can invalidate them. It
3919	* is just too much work. Instead we make the checks here on
3920	* every use. For example:
3921	*
3922	* XFRM dst A --> IPv4 dst X
3923	*
3924	* X is the "xdst->route" of A (X is also the "dst->path" of A
3925	* in this example). If X is marked obsolete, "A" will not
3926	* notice. That's what we are validating here via the
3927	* stale_bundle() check.
3928	*
3929	* When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
3930	* be marked on it.
3931	* This will force stale_bundle() to fail on any xdst bundle with
3932	* this dst linked in it.
3933	*/
3934	if (READ_ONCE(dst->obsolete) < `0` && !stale_bundle(dst))
3935	return dst;
3936
3937	return NULL;
3938	}
3939
3940	static int stale_bundle(struct dst_entry *dst)
3941	{
3942	return !xfrm_bundle_ok(xdst: (struct xfrm_dst *)dst);
3943	}
3944
3945	void xfrm_dst_ifdown(struct dst_entry dst, struct* net_device *dev)
3946	{
3947	while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
3948	dst->dev = blackhole_netdev;
3949	dev_hold(dev: dst->dev);
3950	dev_put(dev);
3951	}
3952	}
3953	EXPORT_SYMBOL(xfrm_dst_ifdown);
3954
3955	static void xfrm_link_failure(struct sk_buff *skb)
3956	{
3957	/ Impossible. Such dst must be popped before reaches point of failure. /
3958	}
3959
3960	static void xfrm_negative_advice(struct sock sk, struct* dst_entry *dst)
3961	{
3962	if (READ_ONCE(dst->obsolete))
3963	sk_dst_reset(sk);
3964	}
3965
3966	static void xfrm_init_pmtu(struct xfrm_dst *bundle, int* nr)
3967	{
3968	while (nr--) {
3969	struct xfrm_dst *xdst = bundle[nr];
3970	u32 pmtu, route_mtu_cached;
3971	struct dst_entry *dst;
3972
3973	dst = &xdst->u.dst;
3974	pmtu = dst_mtu(dst: xfrm_dst_child(dst));
3975	xdst->child_mtu_cached = pmtu;
3976
3977	pmtu = xfrm_state_mtu(x: dst->xfrm, mtu: pmtu);
3978
3979	route_mtu_cached = dst_mtu(dst: xdst->route);
3980	xdst->route_mtu_cached = route_mtu_cached;
3981
3982	if (pmtu > route_mtu_cached)
3983	pmtu = route_mtu_cached;
3984
3985	dst_metric_set(dst, RTAX_MTU, val: pmtu);
3986	}
3987	}
3988
3989	/ Check that the bundle accepts the flow and its components are*
3990	* still valid.
3991	*/
3992
3993	static int xfrm_bundle_ok(struct xfrm_dst *first)
3994	{
3995	struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
3996	struct dst_entry *dst = &first->u.dst;
3997	struct xfrm_dst *xdst;
3998	int start_from, nr;
3999	u32 mtu;
4000
4001	if (!dst_check(dst: xfrm_dst_path(dst), cookie: ((struct xfrm_dst *)dst)->path_cookie) \|\|
4002	(dst->dev && !netif_running(dev: dst->dev)))
4003	return `0`;
4004
4005	if (dst->flags & DST_XFRM_QUEUE)
4006	return `1`;
4007
4008	start_from = nr = `0`;
4009	do {
4010	struct xfrm_dst xdst = (struct* xfrm_dst *)dst;
4011
4012	if (dst->xfrm->km.state != XFRM_STATE_VALID)
4013	return `0`;
4014	if (xdst->xfrm_genid != dst->xfrm->genid)
4015	return `0`;
4016	if (xdst->num_pols > `0` &&
4017	xdst->policy_genid != atomic_read(v: &xdst->pols[`0`]->genid))
4018	return `0`;
4019
4020	bundle[nr++] = xdst;
4021
4022	mtu = dst_mtu(dst: xfrm_dst_child(dst));
4023	if (xdst->child_mtu_cached != mtu) {
4024	start_from = nr;
4025	xdst->child_mtu_cached = mtu;
4026	}
4027
4028	if (!dst_check(dst: xdst->route, cookie: xdst->route_cookie))
4029	return `0`;
4030	mtu = dst_mtu(dst: xdst->route);
4031	if (xdst->route_mtu_cached != mtu) {
4032	start_from = nr;
4033	xdst->route_mtu_cached = mtu;
4034	}
4035
4036	dst = xfrm_dst_child(dst);
4037	} while (dst->xfrm);
4038
4039	if (likely(!start_from))
4040	return `1`;
4041
4042	xdst = bundle[start_from - `1`];
4043	mtu = xdst->child_mtu_cached;
4044	while (start_from--) {
4045	dst = &xdst->u.dst;
4046
4047	mtu = xfrm_state_mtu(x: dst->xfrm, mtu);
4048	if (mtu > xdst->route_mtu_cached)
4049	mtu = xdst->route_mtu_cached;
4050	dst_metric_set(dst, RTAX_MTU, val: mtu);
4051	if (!start_from)
4052	break;
4053
4054	xdst = bundle[start_from - `1`];
4055	xdst->child_mtu_cached = mtu;
4056	}
4057
4058	return `1`;
4059	}
4060
4061	static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
4062	{
4063	return dst_metric_advmss(dst: xfrm_dst_path(dst));
4064	}
4065
4066	static unsigned int xfrm_mtu(const struct dst_entry *dst)
4067	{
4068	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
4069
4070	return mtu ? : dst_mtu(dst: xfrm_dst_path(dst));
4071	}
4072
4073	static const void xfrm_get_dst_nexthop(const* struct dst_entry *dst,
4074	const void *daddr)
4075	{
4076	while (dst->xfrm) {
4077	const struct xfrm_state *xfrm = dst->xfrm;
4078
4079	dst = xfrm_dst_child(dst);
4080
4081	if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
4082	continue;
4083	if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
4084	daddr = xfrm->coaddr;
4085	else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
4086	daddr = &xfrm->id.daddr;
4087	}
4088	return daddr;
4089	}
4090
4091	static struct neighbour xfrm_neigh_lookup(const* struct dst_entry *dst,
4092	struct sk_buff *skb,
4093	const void *daddr)
4094	{
4095	const struct dst_entry *path = xfrm_dst_path(dst);
4096
4097	if (!skb)
4098	daddr = xfrm_get_dst_nexthop(dst, daddr);
4099	return path->ops->neigh_lookup(path, skb, daddr);
4100	}
4101
4102	static void xfrm_confirm_neigh(const struct dst_entry dst, const* void *daddr)
4103	{
4104	const struct dst_entry *path = xfrm_dst_path(dst);
4105
4106	daddr = xfrm_get_dst_nexthop(dst, daddr);
4107	path->ops->confirm_neigh(path, daddr);
4108	}
4109
4110	int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo afinfo, int* family)
4111	{
4112	int err = `0`;
4113
4114	if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
4115	return -EAFNOSUPPORT;
4116
4117	spin_lock(lock: &xfrm_policy_afinfo_lock);
4118	if (unlikely(xfrm_policy_afinfo[family] != NULL))
4119	err = -EEXIST;
4120	else {
4121	struct dst_ops *dst_ops = afinfo->dst_ops;
4122	if (likely(dst_ops->kmem_cachep == NULL))
4123	dst_ops->kmem_cachep = xfrm_dst_cache;
4124	if (likely(dst_ops->check == NULL))
4125	dst_ops->check = xfrm_dst_check;
4126	if (likely(dst_ops->default_advmss == NULL))
4127	dst_ops->default_advmss = xfrm_default_advmss;
4128	if (likely(dst_ops->mtu == NULL))
4129	dst_ops->mtu = xfrm_mtu;
4130	if (likely(dst_ops->negative_advice == NULL))
4131	dst_ops->negative_advice = xfrm_negative_advice;
4132	if (likely(dst_ops->link_failure == NULL))
4133	dst_ops->link_failure = xfrm_link_failure;
4134	if (likely(dst_ops->neigh_lookup == NULL))
4135	dst_ops->neigh_lookup = xfrm_neigh_lookup;
4136	if (likely(!dst_ops->confirm_neigh))
4137	dst_ops->confirm_neigh = xfrm_confirm_neigh;
4138	rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
4139	}
4140	spin_unlock(lock: &xfrm_policy_afinfo_lock);
4141
4142	return err;
4143	}
4144	EXPORT_SYMBOL(xfrm_policy_register_afinfo);
4145
4146	void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
4147	{
4148	struct dst_ops *dst_ops = afinfo->dst_ops;
4149	int i;
4150
4151	for (i = `0`; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
4152	if (xfrm_policy_afinfo[i] != afinfo)
4153	continue;
4154	RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
4155	break;
4156	}
4157
4158	synchronize_rcu();
4159
4160	dst_ops->kmem_cachep = NULL;
4161	dst_ops->check = NULL;
4162	dst_ops->negative_advice = NULL;
4163	dst_ops->link_failure = NULL;
4164	}
4165	EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
4166
4167	void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
4168	{
4169	spin_lock(lock: &xfrm_if_cb_lock);
4170	rcu_assign_pointer(xfrm_if_cb, ifcb);
4171	spin_unlock(lock: &xfrm_if_cb_lock);
4172	}
4173	EXPORT_SYMBOL(xfrm_if_register_cb);
4174
4175	void xfrm_if_unregister_cb(void)
4176	{
4177	RCU_INIT_POINTER(xfrm_if_cb, NULL);
4178	synchronize_rcu();
4179	}
4180	EXPORT_SYMBOL(xfrm_if_unregister_cb);
4181
4182	#ifdef CONFIG_XFRM_STATISTICS
4183	static int __net_init xfrm_statistics_init(struct net *net)
4184	{
4185	int rv;
4186	net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
4187	if (!net->mib.xfrm_statistics)
4188	return -ENOMEM;
4189	rv = xfrm_proc_init(net);
4190	if (rv < `0`)
4191	free_percpu(net->mib.xfrm_statistics);
4192	return rv;
4193	}
4194
4195	static void xfrm_statistics_fini(struct net *net)
4196	{
4197	xfrm_proc_fini(net);
4198	free_percpu(net->mib.xfrm_statistics);
4199	}
4200	#else
4201	static int __net_init xfrm_statistics_init(struct net *net)
4202	{
4203	return `0`;
4204	}
4205
4206	static void xfrm_statistics_fini(struct net *net)
4207	{
4208	}
4209	#endif
4210
4211	static int __net_init xfrm_policy_init(struct net *net)
4212	{
4213	unsigned int hmask, sz;
4214	int dir, err;
4215
4216	if (net_eq(net1: net, net2: &init_net)) {
4217	xfrm_dst_cache = KMEM_CACHE(xfrm_dst, SLAB_HWCACHE_ALIGN \| SLAB_PANIC);
4218	err = rhashtable_init(&xfrm_policy_inexact_table,
4219	&xfrm_pol_inexact_params);
4220	BUG_ON(err);
4221	}
4222
4223	hmask = `8` - `1`;
4224	sz = (hmask+`1`) * sizeof(struct hlist_head);
4225
4226	net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
4227	if (!net->xfrm.policy_byidx)
4228	goto out_byidx;
4229	net->xfrm.policy_idx_hmask = hmask;
4230
4231	for (dir = `0`; dir < XFRM_POLICY_MAX; dir++) {
4232	struct xfrm_policy_hash *htab;
4233
4234	net->xfrm.policy_count[dir] = `0`;
4235	net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = `0`;
4236
4237	htab = &net->xfrm.policy_bydst[dir];
4238	htab->table = xfrm_hash_alloc(sz);
4239	if (!htab->table)
4240	goto out_bydst;
4241	htab->hmask = hmask;
4242	htab->dbits4 = `32`;
4243	htab->sbits4 = `32`;
4244	htab->dbits6 = `128`;
4245	htab->sbits6 = `128`;
4246	}
4247	net->xfrm.policy_hthresh.lbits4 = `32`;
4248	net->xfrm.policy_hthresh.rbits4 = `32`;
4249	net->xfrm.policy_hthresh.lbits6 = `128`;
4250	net->xfrm.policy_hthresh.rbits6 = `128`;
4251
4252	seqlock_init(&net->xfrm.policy_hthresh.lock);
4253
4254	INIT_LIST_HEAD(list: &net->xfrm.policy_all);
4255	INIT_LIST_HEAD(list: &net->xfrm.inexact_bins);
4256	INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
4257	INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
4258	return `0`;
4259
4260	out_bydst:
4261	for (dir--; dir >= `0`; dir--) {
4262	struct xfrm_policy_hash *htab;
4263
4264	htab = &net->xfrm.policy_bydst[dir];
4265	xfrm_hash_free(n: htab->table, sz);
4266	}
4267	xfrm_hash_free(n: net->xfrm.policy_byidx, sz);
4268	out_byidx:
4269	return -ENOMEM;
4270	}
4271
4272	static void xfrm_policy_fini(struct net *net)
4273	{
4274	struct xfrm_pol_inexact_bin b, t;
4275	unsigned int sz;
4276	int dir;
4277
4278	flush_work(work: &net->xfrm.policy_hash_work);
4279	#ifdef CONFIG_XFRM_SUB_POLICY
4280	xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
4281	#endif
4282	xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
4283
4284	WARN_ON(!list_empty(&net->xfrm.policy_all));
4285
4286	for (dir = `0`; dir < XFRM_POLICY_MAX; dir++) {
4287	struct xfrm_policy_hash *htab;
4288
4289	htab = &net->xfrm.policy_bydst[dir];
4290	sz = (htab->hmask + `1`) * sizeof(struct hlist_head);
4291	WARN_ON(!hlist_empty(htab->table));
4292	xfrm_hash_free(n: htab->table, sz);
4293	}
4294
4295	sz = (net->xfrm.policy_idx_hmask + `1`) * sizeof(struct hlist_head);
4296	WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
4297	xfrm_hash_free(n: net->xfrm.policy_byidx, sz);
4298
4299	spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock);
4300	list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)
4301	__xfrm_policy_inexact_prune_bin(b, net_exit: true);
4302	spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock);
4303	}
4304
4305	static int __net_init xfrm_net_init(struct net *net)
4306	{
4307	int rv;
4308
4309	/ Initialize the per-net locks here /
4310	spin_lock_init(&net->xfrm.xfrm_state_lock);
4311	spin_lock_init(&net->xfrm.xfrm_policy_lock);
4312	seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);
4313	mutex_init(&net->xfrm.xfrm_cfg_mutex);
4314	net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;
4315	net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;
4316	net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;
4317
4318	rv = xfrm_statistics_init(net);
4319	if (rv < `0`)
4320	goto out_statistics;
4321	rv = xfrm_state_init(net);
4322	if (rv < `0`)
4323	goto out_state;
4324	rv = xfrm_policy_init(net);
4325	if (rv < `0`)
4326	goto out_policy;
4327	rv = xfrm_sysctl_init(net);
4328	if (rv < `0`)
4329	goto out_sysctl;
4330
4331	rv = xfrm_nat_keepalive_net_init(net);
4332	if (rv < `0`)
4333	goto out_nat_keepalive;
4334
4335	return `0`;
4336
4337	out_nat_keepalive:
4338	xfrm_sysctl_fini(net);
4339	out_sysctl:
4340	xfrm_policy_fini(net);
4341	out_policy:
4342	xfrm_state_fini(net);
4343	out_state:
4344	xfrm_statistics_fini(net);
4345	out_statistics:
4346	return rv;
4347	}
4348
4349	static void __net_exit xfrm_net_exit(struct net *net)
4350	{
4351	xfrm_nat_keepalive_net_fini(net);
4352	xfrm_sysctl_fini(net);
4353	xfrm_policy_fini(net);
4354	xfrm_state_fini(net);
4355	xfrm_statistics_fini(net);
4356	}
4357
4358	static struct pernet_operations __net_initdata xfrm_net_ops = {
4359	.init = xfrm_net_init,
4360	.exit = xfrm_net_exit,
4361	};
4362
4363	static const struct flow_dissector_key xfrm_flow_dissector_keys[] = {
4364	{
4365	.key_id = FLOW_DISSECTOR_KEY_CONTROL,
4366	.offset = offsetof(struct xfrm_flow_keys, control),
4367	},
4368	{
4369	.key_id = FLOW_DISSECTOR_KEY_BASIC,
4370	.offset = offsetof(struct xfrm_flow_keys, basic),
4371	},
4372	{
4373	.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
4374	.offset = offsetof(struct xfrm_flow_keys, addrs.ipv4),
4375	},
4376	{
4377	.key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
4378	.offset = offsetof(struct xfrm_flow_keys, addrs.ipv6),
4379	},
4380	{
4381	.key_id = FLOW_DISSECTOR_KEY_PORTS,
4382	.offset = offsetof(struct xfrm_flow_keys, ports),
4383	},
4384	{
4385	.key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
4386	.offset = offsetof(struct xfrm_flow_keys, gre),
4387	},
4388	{
4389	.key_id = FLOW_DISSECTOR_KEY_IP,
4390	.offset = offsetof(struct xfrm_flow_keys, ip),
4391	},
4392	{
4393	.key_id = FLOW_DISSECTOR_KEY_ICMP,
4394	.offset = offsetof(struct xfrm_flow_keys, icmp),
4395	},
4396	};
4397
4398	void __init xfrm_init(void)
4399	{
4400	skb_flow_dissector_init(flow_dissector: &xfrm_session_dissector,
4401	key: xfrm_flow_dissector_keys,
4402	ARRAY_SIZE(xfrm_flow_dissector_keys));
4403
4404	register_pernet_subsys(&xfrm_net_ops);
4405	xfrm_dev_init();
4406	xfrm_input_init();
4407
4408	#ifdef CONFIG_XFRM_ESPINTCP
4409	espintcp_init();
4410	#endif
4411
4412	register_xfrm_state_bpf();
4413	xfrm_nat_keepalive_init(AF_INET);
4414	}
4415
4416	#ifdef CONFIG_AUDITSYSCALL
4417	static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
4418	struct audit_buffer *audit_buf)
4419	{
4420	struct xfrm_sec_ctx *ctx = xp->security;
4421	struct xfrm_selector *sel = &xp->selector;
4422
4423	if (ctx)
4424	audit_log_format(ab: audit_buf, fmt: " sec_alg=%u sec_doi=%u sec_obj=%s",
4425	ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
4426
4427	switch (sel->family) {
4428	case AF_INET:
4429	audit_log_format(ab: audit_buf, fmt: " src=%pI4", &sel->saddr.a4);
4430	if (sel->prefixlen_s != `32`)
4431	audit_log_format(ab: audit_buf, fmt: " src_prefixlen=%d",
4432	sel->prefixlen_s);
4433	audit_log_format(ab: audit_buf, fmt: " dst=%pI4", &sel->daddr.a4);
4434	if (sel->prefixlen_d != `32`)
4435	audit_log_format(ab: audit_buf, fmt: " dst_prefixlen=%d",
4436	sel->prefixlen_d);
4437	break;
4438	case AF_INET6:
4439	audit_log_format(ab: audit_buf, fmt: " src=%pI6", sel->saddr.a6);
4440	if (sel->prefixlen_s != `128`)
4441	audit_log_format(ab: audit_buf, fmt: " src_prefixlen=%d",
4442	sel->prefixlen_s);
4443	audit_log_format(ab: audit_buf, fmt: " dst=%pI6", sel->daddr.a6);
4444	if (sel->prefixlen_d != `128`)
4445	audit_log_format(ab: audit_buf, fmt: " dst_prefixlen=%d",
4446	sel->prefixlen_d);
4447	break;
4448	}
4449	}
4450
4451	void xfrm_audit_policy_add(struct xfrm_policy xp, int* result, bool task_valid)
4452	{
4453	struct audit_buffer *audit_buf;
4454
4455	audit_buf = xfrm_audit_start(op: "SPD-add");
4456	if (audit_buf == NULL)
4457	return;
4458	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4459	audit_log_format(ab: audit_buf, fmt: " res=%u", result);
4460	xfrm_audit_common_policyinfo(xp, audit_buf);
4461	audit_log_end(ab: audit_buf);
4462	}
4463	EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
4464
4465	void xfrm_audit_policy_delete(struct xfrm_policy xp, int* result,
4466	bool task_valid)
4467	{
4468	struct audit_buffer *audit_buf;
4469
4470	audit_buf = xfrm_audit_start(op: "SPD-delete");
4471	if (audit_buf == NULL)
4472	return;
4473	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4474	audit_log_format(ab: audit_buf, fmt: " res=%u", result);
4475	xfrm_audit_common_policyinfo(xp, audit_buf);
4476	audit_log_end(ab: audit_buf);
4477	}
4478	EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
4479	#endif
4480
4481	#ifdef CONFIG_XFRM_MIGRATE
4482	static struct xfrm_policy xfrm_migrate_policy_find(const* struct xfrm_selector *sel,
4483	u8 dir, u8 type, struct net *net, u32 if_id)
4484	{
4485	struct xfrm_policy *pol;
4486	struct flowi fl;
4487
4488	memset(&fl, `0`, sizeof(fl));
4489
4490	fl.flowi_proto = sel->proto;
4491
4492	switch (sel->family) {
4493	case AF_INET:
4494	fl.u.ip4.saddr = sel->saddr.a4;
4495	fl.u.ip4.daddr = sel->daddr.a4;
4496	if (sel->proto == IPSEC_ULPROTO_ANY)
4497	break;
4498	fl.u.flowi4_oif = sel->ifindex;
4499	fl.u.ip4.fl4_sport = sel->sport;
4500	fl.u.ip4.fl4_dport = sel->dport;
4501	break;
4502	case AF_INET6:
4503	fl.u.ip6.saddr = sel->saddr.in6;
4504	fl.u.ip6.daddr = sel->daddr.in6;
4505	if (sel->proto == IPSEC_ULPROTO_ANY)
4506	break;
4507	fl.u.flowi6_oif = sel->ifindex;
4508	fl.u.ip6.fl4_sport = sel->sport;
4509	fl.u.ip6.fl4_dport = sel->dport;
4510	break;
4511	default:
4512	return ERR_PTR(-EAFNOSUPPORT);
4513	}
4514
4515	rcu_read_lock();
4516
4517	pol = xfrm_policy_lookup_bytype(net, type, &fl, sel->family, dir, if_id);
4518	if (IS_ERR_OR_NULL(pol))
4519	goto out_unlock;
4520
4521	if (!xfrm_pol_hold_rcu(pol))
4522	pol = NULL;
4523	out_unlock:
4524	rcu_read_unlock();
4525	return pol;
4526	}
4527
4528	static int migrate_tmpl_match(const struct xfrm_migrate m, const* struct xfrm_tmpl *t)
4529	{
4530	int match = `0`;
4531
4532	if (t->mode == m->mode && t->id.proto == m->proto &&
4533	(m->reqid == `0` \|\| t->reqid == m->reqid)) {
4534	switch (t->mode) {
4535	case XFRM_MODE_TUNNEL:
4536	case XFRM_MODE_BEET:
4537	case XFRM_MODE_IPTFS:
4538	if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
4539	m->old_family) &&
4540	xfrm_addr_equal(&t->saddr, &m->old_saddr,
4541	m->old_family)) {
4542	match = `1`;
4543	}
4544	break;
4545	case XFRM_MODE_TRANSPORT:
4546	/ in case of transport mode, template does not store*
4547	any IP addresses, hence we just compare mode and
4548	protocol /*
4549	match = `1`;
4550	break;
4551	default:
4552	break;
4553	}
4554	}
4555	return match;
4556	}
4557
4558	/ update endpoint address(es) of template(s) /
4559	static int xfrm_policy_migrate(struct xfrm_policy *pol,
4560	struct xfrm_migrate m, int* num_migrate,
4561	struct netlink_ext_ack *extack)
4562	{
4563	struct xfrm_migrate *mp;
4564	int i, j, n = `0`;
4565
4566	write_lock_bh(&pol->lock);
4567	if (unlikely(pol->walk.dead)) {
4568	/ target policy has been deleted /
4569	NL_SET_ERR_MSG(extack, "Target policy not found");
4570	write_unlock_bh(&pol->lock);
4571	return -ENOENT;
4572	}
4573
4574	for (i = `0`; i < pol->xfrm_nr; i++) {
4575	for (j = `0`, mp = m; j < num_migrate; j++, mp++) {
4576	if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
4577	continue;
4578	n++;
4579	if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
4580	pol->xfrm_vec[i].mode != XFRM_MODE_BEET &&
4581	pol->xfrm_vec[i].mode != XFRM_MODE_IPTFS)
4582	continue;
4583	/ update endpoints /
4584	memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
4585	sizeof(pol->xfrm_vec[i].id.daddr));
4586	memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
4587	sizeof(pol->xfrm_vec[i].saddr));
4588	pol->xfrm_vec[i].encap_family = mp->new_family;
4589	/ flush bundles /
4590	atomic_inc(&pol->genid);
4591	}
4592	}
4593
4594	write_unlock_bh(&pol->lock);
4595
4596	if (!n)
4597	return -ENODATA;
4598
4599	return `0`;
4600	}
4601
4602	static int xfrm_migrate_check(const struct xfrm_migrate m, int* num_migrate,
4603	struct netlink_ext_ack *extack)
4604	{
4605	int i, j;
4606
4607	if (num_migrate < `1` \|\| num_migrate > XFRM_MAX_DEPTH) {
4608	NL_SET_ERR_MSG(extack, "Invalid number of SAs to migrate, must be 0 < num <= XFRM_MAX_DEPTH (6)");
4609	return -EINVAL;
4610	}
4611
4612	for (i = `0`; i < num_migrate; i++) {
4613	if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) \|\|
4614	xfrm_addr_any(&m[i].new_saddr, m[i].new_family)) {
4615	NL_SET_ERR_MSG(extack, "Addresses in the MIGRATE attribute's list cannot be null");
4616	return -EINVAL;
4617	}
4618
4619	/ check if there is any duplicated entry /
4620	for (j = i + `1`; j < num_migrate; j++) {
4621	if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
4622	sizeof(m[i].old_daddr)) &&
4623	!memcmp(&m[i].old_saddr, &m[j].old_saddr,
4624	sizeof(m[i].old_saddr)) &&
4625	m[i].proto == m[j].proto &&
4626	m[i].mode == m[j].mode &&
4627	m[i].reqid == m[j].reqid &&
4628	m[i].old_family == m[j].old_family) {
4629	NL_SET_ERR_MSG(extack, "Entries in the MIGRATE attribute's list must be unique");
4630	return -EINVAL;
4631	}
4632	}
4633	}
4634
4635	return `0`;
4636	}
4637
4638	int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
4639	struct xfrm_migrate m, int* num_migrate,
4640	struct xfrm_kmaddress k, struct* net *net,
4641	struct xfrm_encap_tmpl *encap, u32 if_id,
4642	struct netlink_ext_ack extack, struct* xfrm_user_offload *xuo)
4643	{
4644	int i, err, nx_cur = `0`, nx_new = `0`;
4645	struct xfrm_policy *pol = NULL;
4646	struct xfrm_state x, xc;
4647	struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
4648	struct xfrm_state *x_new[XFRM_MAX_DEPTH];
4649	struct xfrm_migrate *mp;
4650
4651	/ Stage 0 - sanity checks /
4652	err = xfrm_migrate_check(m, num_migrate, extack);
4653	if (err < `0`)
4654	goto out;
4655
4656	if (dir >= XFRM_POLICY_MAX) {
4657	NL_SET_ERR_MSG(extack, "Invalid policy direction");
4658	err = -EINVAL;
4659	goto out;
4660	}
4661
4662	/ Stage 1 - find policy /
4663	pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id);
4664	if (IS_ERR_OR_NULL(pol)) {
4665	NL_SET_ERR_MSG(extack, "Target policy not found");
4666	err = IS_ERR(pol) ? PTR_ERR(pol) : -ENOENT;
4667	goto out;
4668	}
4669
4670	/ Stage 2 - find and update state(s) /
4671	for (i = `0`, mp = m; i < num_migrate; i++, mp++) {
4672	if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
4673	x_cur[nx_cur] = x;
4674	nx_cur++;
4675	xc = xfrm_state_migrate(x, mp, encap, net, xuo, extack);
4676	if (xc) {
4677	x_new[nx_new] = xc;
4678	nx_new++;
4679	} else {
4680	err = -ENODATA;
4681	goto restore_state;
4682	}
4683	}
4684	}
4685
4686	/ Stage 3 - update policy /
4687	err = xfrm_policy_migrate(pol, m, num_migrate, extack);
4688	if (err < `0`)
4689	goto restore_state;
4690
4691	/ Stage 4 - delete old state(s) /
4692	if (nx_cur) {
4693	xfrm_states_put(x_cur, nx_cur);
4694	xfrm_states_delete(x_cur, nx_cur);
4695	}
4696
4697	/ Stage 5 - announce /
4698	km_migrate(sel, dir, type, m, num_migrate, k, encap);
4699
4700	xfrm_pol_put(pol);
4701
4702	return `0`;
4703	out:
4704	return err;
4705
4706	restore_state:
4707	if (pol)
4708	xfrm_pol_put(pol);
4709	if (nx_cur)
4710	xfrm_states_put(x_cur, nx_cur);
4711	if (nx_new)
4712	xfrm_states_delete(x_new, nx_new);
4713
4714	return err;
4715	}
4716	EXPORT_SYMBOL(xfrm_migrate);
4717	#endif
4718

Browse the source code of Linux/net/xfrm/xfrm_policy.c