1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Linux INET6 implementation
4 * FIB front-end.
5 *
6 * Authors:
7 * Pedro Roque <roque@di.fc.ul.pt>
8 */
9
10/* Changes:
11 *
12 * YOSHIFUJI Hideaki @USAGI
13 * reworked default router selection.
14 * - respect outgoing interface
15 * - select from (probably) reachable routers (i.e.
16 * routers in REACHABLE, STALE, DELAY or PROBE states).
17 * - always select the same router if it is (probably)
18 * reachable. otherwise, round-robin the list.
19 * Ville Nuorvala
20 * Fixed routing subtrees.
21 */
22
23#define pr_fmt(fmt) "IPv6: " fmt
24
25#include <linux/capability.h>
26#include <linux/errno.h>
27#include <linux/export.h>
28#include <linux/types.h>
29#include <linux/times.h>
30#include <linux/socket.h>
31#include <linux/sockios.h>
32#include <linux/net.h>
33#include <linux/route.h>
34#include <linux/netdevice.h>
35#include <linux/in6.h>
36#include <linux/mroute6.h>
37#include <linux/init.h>
38#include <linux/if_arp.h>
39#include <linux/proc_fs.h>
40#include <linux/seq_file.h>
41#include <linux/nsproxy.h>
42#include <linux/slab.h>
43#include <linux/jhash.h>
44#include <linux/siphash.h>
45#include <net/net_namespace.h>
46#include <net/snmp.h>
47#include <net/ipv6.h>
48#include <net/ip6_fib.h>
49#include <net/ip6_route.h>
50#include <net/ndisc.h>
51#include <net/addrconf.h>
52#include <net/tcp.h>
53#include <linux/rtnetlink.h>
54#include <net/dst.h>
55#include <net/dst_metadata.h>
56#include <net/xfrm.h>
57#include <net/netevent.h>
58#include <net/netlink.h>
59#include <net/rtnh.h>
60#include <net/lwtunnel.h>
61#include <net/ip_tunnels.h>
62#include <net/l3mdev.h>
63#include <net/ip.h>
64#include <linux/uaccess.h>
65#include <linux/btf_ids.h>
66
67#ifdef CONFIG_SYSCTL
68#include <linux/sysctl.h>
69#endif
70
71static int ip6_rt_type_to_error(u8 fib6_type);
72
73#define CREATE_TRACE_POINTS
74#include <trace/events/fib6.h>
75EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup);
76#undef CREATE_TRACE_POINTS
77
78enum rt6_nud_state {
79 RT6_NUD_FAIL_HARD = -3,
80 RT6_NUD_FAIL_PROBE = -2,
81 RT6_NUD_FAIL_DO_RR = -1,
82 RT6_NUD_SUCCEED = 1
83};
84
85INDIRECT_CALLABLE_SCOPE
86struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
87static unsigned int ip6_default_advmss(const struct dst_entry *dst);
88INDIRECT_CALLABLE_SCOPE
89unsigned int ip6_mtu(const struct dst_entry *dst);
90static void ip6_negative_advice(struct sock *sk,
91 struct dst_entry *dst);
92static void ip6_dst_destroy(struct dst_entry *);
93static void ip6_dst_ifdown(struct dst_entry *,
94 struct net_device *dev);
95static void ip6_dst_gc(struct dst_ops *ops);
96
97static int ip6_pkt_discard(struct sk_buff *skb);
98static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
99static int ip6_pkt_prohibit(struct sk_buff *skb);
100static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb);
101static void ip6_link_failure(struct sk_buff *skb);
102static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
103 struct sk_buff *skb, u32 mtu,
104 bool confirm_neigh);
105static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk,
106 struct sk_buff *skb);
107static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
108 int strict);
109static size_t rt6_nlmsg_size(struct fib6_info *f6i);
110static int rt6_fill_node(struct net *net, struct sk_buff *skb,
111 struct fib6_info *rt, struct dst_entry *dst,
112 struct in6_addr *dest, struct in6_addr *src,
113 int iif, int type, u32 portid, u32 seq,
114 unsigned int flags);
115static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
116 const struct in6_addr *daddr,
117 const struct in6_addr *saddr);
118
119#ifdef CONFIG_IPV6_ROUTE_INFO
120static struct fib6_info *rt6_add_route_info(struct net *net,
121 const struct in6_addr *prefix, int prefixlen,
122 const struct in6_addr *gwaddr,
123 struct net_device *dev,
124 unsigned int pref);
125static struct fib6_info *rt6_get_route_info(struct net *net,
126 const struct in6_addr *prefix, int prefixlen,
127 const struct in6_addr *gwaddr,
128 struct net_device *dev);
129#endif
130
131struct uncached_list {
132 spinlock_t lock;
133 struct list_head head;
134};
135
136static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list);
137
138void rt6_uncached_list_add(struct rt6_info *rt)
139{
140 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);
141
142 rt->dst.rt_uncached_list = ul;
143
144 spin_lock_bh(lock: &ul->lock);
145 list_add_tail(new: &rt->dst.rt_uncached, head: &ul->head);
146 spin_unlock_bh(lock: &ul->lock);
147}
148
149void rt6_uncached_list_del(struct rt6_info *rt)
150{
151 if (!list_empty(head: &rt->dst.rt_uncached)) {
152 struct uncached_list *ul = rt->dst.rt_uncached_list;
153
154 spin_lock_bh(lock: &ul->lock);
155 list_del_init(entry: &rt->dst.rt_uncached);
156 spin_unlock_bh(lock: &ul->lock);
157 }
158}
159
160static void rt6_uncached_list_flush_dev(struct net_device *dev)
161{
162 int cpu;
163
164 for_each_possible_cpu(cpu) {
165 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
166 struct rt6_info *rt, *safe;
167
168 if (list_empty(head: &ul->head))
169 continue;
170
171 spin_lock_bh(lock: &ul->lock);
172 list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) {
173 struct inet6_dev *rt_idev = rt->rt6i_idev;
174 struct net_device *rt_dev = rt->dst.dev;
175 bool handled = false;
176
177 if (rt_idev && rt_idev->dev == dev) {
178 rt->rt6i_idev = in6_dev_get(dev: blackhole_netdev);
179 in6_dev_put(idev: rt_idev);
180 handled = true;
181 }
182
183 if (rt_dev == dev) {
184 rt->dst.dev = blackhole_netdev;
185 netdev_ref_replace(odev: rt_dev, ndev: blackhole_netdev,
186 tracker: &rt->dst.dev_tracker,
187 GFP_ATOMIC);
188 handled = true;
189 }
190 if (handled)
191 list_del_init(entry: &rt->dst.rt_uncached);
192 }
193 spin_unlock_bh(lock: &ul->lock);
194 }
195}
196
197static inline const void *choose_neigh_daddr(const struct in6_addr *p,
198 struct sk_buff *skb,
199 const void *daddr)
200{
201 if (!ipv6_addr_any(a: p))
202 return (const void *) p;
203 else if (skb)
204 return &ipv6_hdr(skb)->daddr;
205 return daddr;
206}
207
208struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
209 struct net_device *dev,
210 struct sk_buff *skb,
211 const void *daddr)
212{
213 struct neighbour *n;
214
215 daddr = choose_neigh_daddr(p: gw, skb, daddr);
216 n = __ipv6_neigh_lookup(dev, pkey: daddr);
217 if (n)
218 return n;
219
220 n = neigh_create(tbl: &nd_tbl, pkey: daddr, dev);
221 return IS_ERR(ptr: n) ? NULL : n;
222}
223
224static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst,
225 struct sk_buff *skb,
226 const void *daddr)
227{
228 const struct rt6_info *rt = dst_rt6_info(dst);
229
230 return ip6_neigh_lookup(gw: rt6_nexthop(rt, daddr: &in6addr_any),
231 dev: dst_dev(dst), skb, daddr);
232}
233
234static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr)
235{
236 const struct rt6_info *rt = dst_rt6_info(dst);
237 struct net_device *dev = dst_dev(dst);
238
239 daddr = choose_neigh_daddr(p: rt6_nexthop(rt, daddr: &in6addr_any), NULL, daddr);
240 if (!daddr)
241 return;
242 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK))
243 return;
244 if (ipv6_addr_is_multicast(addr: (const struct in6_addr *)daddr))
245 return;
246 __ipv6_confirm_neigh(dev, pkey: daddr);
247}
248
249static struct dst_ops ip6_dst_ops_template = {
250 .family = AF_INET6,
251 .gc = ip6_dst_gc,
252 .gc_thresh = 1024,
253 .check = ip6_dst_check,
254 .default_advmss = ip6_default_advmss,
255 .mtu = ip6_mtu,
256 .cow_metrics = dst_cow_metrics_generic,
257 .destroy = ip6_dst_destroy,
258 .ifdown = ip6_dst_ifdown,
259 .negative_advice = ip6_negative_advice,
260 .link_failure = ip6_link_failure,
261 .update_pmtu = ip6_rt_update_pmtu,
262 .redirect = rt6_do_redirect,
263 .local_out = __ip6_local_out,
264 .neigh_lookup = ip6_dst_neigh_lookup,
265 .confirm_neigh = ip6_confirm_neigh,
266};
267
268static struct dst_ops ip6_dst_blackhole_ops = {
269 .family = AF_INET6,
270 .default_advmss = ip6_default_advmss,
271 .neigh_lookup = ip6_dst_neigh_lookup,
272 .check = ip6_dst_check,
273 .destroy = ip6_dst_destroy,
274 .cow_metrics = dst_cow_metrics_generic,
275 .update_pmtu = dst_blackhole_update_pmtu,
276 .redirect = dst_blackhole_redirect,
277 .mtu = dst_blackhole_mtu,
278};
279
280static const u32 ip6_template_metrics[RTAX_MAX] = {
281 [RTAX_HOPLIMIT - 1] = 0,
282};
283
284static const struct fib6_info fib6_null_entry_template = {
285 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP),
286 .fib6_protocol = RTPROT_KERNEL,
287 .fib6_metric = ~(u32)0,
288 .fib6_ref = REFCOUNT_INIT(1),
289 .fib6_type = RTN_UNREACHABLE,
290 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics,
291};
292
293static const struct rt6_info ip6_null_entry_template = {
294 .dst = {
295 .__rcuref = RCUREF_INIT(1),
296 .__use = 1,
297 .obsolete = DST_OBSOLETE_FORCE_CHK,
298 .error = -ENETUNREACH,
299 .input = ip6_pkt_discard,
300 .output = ip6_pkt_discard_out,
301 },
302 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
303};
304
305#ifdef CONFIG_IPV6_MULTIPLE_TABLES
306
307static const struct rt6_info ip6_prohibit_entry_template = {
308 .dst = {
309 .__rcuref = RCUREF_INIT(1),
310 .__use = 1,
311 .obsolete = DST_OBSOLETE_FORCE_CHK,
312 .error = -EACCES,
313 .input = ip6_pkt_prohibit,
314 .output = ip6_pkt_prohibit_out,
315 },
316 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
317};
318
319static const struct rt6_info ip6_blk_hole_entry_template = {
320 .dst = {
321 .__rcuref = RCUREF_INIT(1),
322 .__use = 1,
323 .obsolete = DST_OBSOLETE_FORCE_CHK,
324 .error = -EINVAL,
325 .input = dst_discard,
326 .output = dst_discard_out,
327 },
328 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
329};
330
331#endif
332
333static void rt6_info_init(struct rt6_info *rt)
334{
335 memset_after(rt, 0, dst);
336}
337
338/* allocate dst with ip6_dst_ops */
339struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
340 int flags)
341{
342 struct rt6_info *rt = dst_alloc(ops: &net->ipv6.ip6_dst_ops, dev,
343 DST_OBSOLETE_FORCE_CHK, flags);
344
345 if (rt) {
346 rt6_info_init(rt);
347 atomic_inc(v: &net->ipv6.rt6_stats->fib_rt_alloc);
348 }
349
350 return rt;
351}
352EXPORT_SYMBOL(ip6_dst_alloc);
353
354static void ip6_dst_destroy(struct dst_entry *dst)
355{
356 struct rt6_info *rt = dst_rt6_info(dst);
357 struct fib6_info *from;
358 struct inet6_dev *idev;
359
360 ip_dst_metrics_put(dst);
361 rt6_uncached_list_del(rt);
362
363 idev = rt->rt6i_idev;
364 if (idev) {
365 rt->rt6i_idev = NULL;
366 in6_dev_put(idev);
367 }
368
369 from = unrcu_pointer(xchg(&rt->from, NULL));
370 fib6_info_release(f6i: from);
371}
372
373static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
374{
375 struct rt6_info *rt = dst_rt6_info(dst);
376 struct inet6_dev *idev = rt->rt6i_idev;
377 struct fib6_info *from;
378
379 if (idev && idev->dev != blackhole_netdev) {
380 struct inet6_dev *blackhole_idev = in6_dev_get(dev: blackhole_netdev);
381
382 if (blackhole_idev) {
383 rt->rt6i_idev = blackhole_idev;
384 in6_dev_put(idev);
385 }
386 }
387 from = unrcu_pointer(xchg(&rt->from, NULL));
388 fib6_info_release(f6i: from);
389}
390
391static bool __rt6_check_expired(const struct rt6_info *rt)
392{
393 if (rt->rt6i_flags & RTF_EXPIRES)
394 return time_after(jiffies, READ_ONCE(rt->dst.expires));
395 return false;
396}
397
398static bool rt6_check_expired(const struct rt6_info *rt)
399{
400 struct fib6_info *from;
401
402 from = rcu_dereference(rt->from);
403
404 if (rt->rt6i_flags & RTF_EXPIRES) {
405 if (time_after(jiffies, READ_ONCE(rt->dst.expires)))
406 return true;
407 } else if (from) {
408 return READ_ONCE(rt->dst.obsolete) != DST_OBSOLETE_FORCE_CHK ||
409 fib6_check_expired(f6i: from);
410 }
411 return false;
412}
413
414static struct fib6_info *
415rt6_multipath_first_sibling_rcu(const struct fib6_info *rt)
416{
417 struct fib6_info *iter;
418 struct fib6_node *fn;
419
420 fn = rcu_dereference(rt->fib6_node);
421 if (!fn)
422 goto out;
423 iter = rcu_dereference(fn->leaf);
424 if (!iter)
425 goto out;
426
427 while (iter) {
428 if (iter->fib6_metric == rt->fib6_metric &&
429 rt6_qualify_for_ecmp(f6i: iter))
430 return iter;
431 iter = rcu_dereference(iter->fib6_next);
432 }
433
434out:
435 return NULL;
436}
437
438void fib6_select_path(const struct net *net, struct fib6_result *res,
439 struct flowi6 *fl6, int oif, bool have_oif_match,
440 const struct sk_buff *skb, int strict)
441{
442 struct fib6_info *first, *match = res->f6i;
443 struct fib6_info *sibling;
444 int hash;
445
446 if (!match->nh && (!match->fib6_nsiblings || have_oif_match))
447 goto out;
448
449 if (match->nh && have_oif_match && res->nh)
450 return;
451
452 if (skb)
453 IP6CB(skb)->flags |= IP6SKB_MULTIPATH;
454
455 /* We might have already computed the hash for ICMPv6 errors. In such
456 * case it will always be non-zero. Otherwise now is the time to do it.
457 */
458 if (!fl6->mp_hash &&
459 (!match->nh || nexthop_is_multipath(nh: match->nh)))
460 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL);
461
462 if (unlikely(match->nh)) {
463 nexthop_path_fib6_result(res, hash: fl6->mp_hash);
464 return;
465 }
466
467 first = rt6_multipath_first_sibling_rcu(rt: match);
468 if (!first)
469 goto out;
470
471 hash = fl6->mp_hash;
472 if (hash <= atomic_read(v: &first->fib6_nh->fib_nh_upper_bound)) {
473 if (rt6_score_route(nh: first->fib6_nh, fib6_flags: first->fib6_flags, oif,
474 strict) >= 0)
475 match = first;
476 goto out;
477 }
478
479 list_for_each_entry_rcu(sibling, &first->fib6_siblings,
480 fib6_siblings) {
481 const struct fib6_nh *nh = sibling->fib6_nh;
482 int nh_upper_bound;
483
484 nh_upper_bound = atomic_read(v: &nh->fib_nh_upper_bound);
485 if (hash > nh_upper_bound)
486 continue;
487 if (rt6_score_route(nh, fib6_flags: sibling->fib6_flags, oif, strict) < 0)
488 break;
489 match = sibling;
490 break;
491 }
492
493out:
494 res->f6i = match;
495 res->nh = match->fib6_nh;
496}
497
498/*
499 * Route lookup. rcu_read_lock() should be held.
500 */
501
502static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,
503 const struct in6_addr *saddr, int oif, int flags)
504{
505 const struct net_device *dev;
506
507 if (nh->fib_nh_flags & RTNH_F_DEAD)
508 return false;
509
510 dev = nh->fib_nh_dev;
511 if (oif) {
512 if (dev->ifindex == oif)
513 return true;
514 } else {
515 if (ipv6_chk_addr(net, addr: saddr, dev,
516 strict: flags & RT6_LOOKUP_F_IFACE))
517 return true;
518 }
519
520 return false;
521}
522
523struct fib6_nh_dm_arg {
524 struct net *net;
525 const struct in6_addr *saddr;
526 int oif;
527 int flags;
528 struct fib6_nh *nh;
529};
530
531static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)
532{
533 struct fib6_nh_dm_arg *arg = _arg;
534
535 arg->nh = nh;
536 return __rt6_device_match(net: arg->net, nh, saddr: arg->saddr, oif: arg->oif,
537 flags: arg->flags);
538}
539
540/* returns fib6_nh from nexthop or NULL */
541static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,
542 struct fib6_result *res,
543 const struct in6_addr *saddr,
544 int oif, int flags)
545{
546 struct fib6_nh_dm_arg arg = {
547 .net = net,
548 .saddr = saddr,
549 .oif = oif,
550 .flags = flags,
551 };
552
553 if (nexthop_is_blackhole(nh))
554 return NULL;
555
556 if (nexthop_for_each_fib6_nh(nh, cb: __rt6_nh_dev_match, arg: &arg))
557 return arg.nh;
558
559 return NULL;
560}
561
562static void rt6_device_match(struct net *net, struct fib6_result *res,
563 const struct in6_addr *saddr, int oif, int flags)
564{
565 struct fib6_info *f6i = res->f6i;
566 struct fib6_info *spf6i;
567 struct fib6_nh *nh;
568
569 if (!oif && ipv6_addr_any(a: saddr)) {
570 if (unlikely(f6i->nh)) {
571 nh = nexthop_fib6_nh(nh: f6i->nh);
572 if (nexthop_is_blackhole(nh: f6i->nh))
573 goto out_blackhole;
574 } else {
575 nh = f6i->fib6_nh;
576 }
577 if (!(nh->fib_nh_flags & RTNH_F_DEAD))
578 goto out;
579 }
580
581 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) {
582 bool matched = false;
583
584 if (unlikely(spf6i->nh)) {
585 nh = rt6_nh_dev_match(net, nh: spf6i->nh, res, saddr,
586 oif, flags);
587 if (nh)
588 matched = true;
589 } else {
590 nh = spf6i->fib6_nh;
591 if (__rt6_device_match(net, nh, saddr, oif, flags))
592 matched = true;
593 }
594 if (matched) {
595 res->f6i = spf6i;
596 goto out;
597 }
598 }
599
600 if (oif && flags & RT6_LOOKUP_F_IFACE) {
601 res->f6i = net->ipv6.fib6_null_entry;
602 nh = res->f6i->fib6_nh;
603 goto out;
604 }
605
606 if (unlikely(f6i->nh)) {
607 nh = nexthop_fib6_nh(nh: f6i->nh);
608 if (nexthop_is_blackhole(nh: f6i->nh))
609 goto out_blackhole;
610 } else {
611 nh = f6i->fib6_nh;
612 }
613
614 if (nh->fib_nh_flags & RTNH_F_DEAD) {
615 res->f6i = net->ipv6.fib6_null_entry;
616 nh = res->f6i->fib6_nh;
617 }
618out:
619 res->nh = nh;
620 res->fib6_type = res->f6i->fib6_type;
621 res->fib6_flags = res->f6i->fib6_flags;
622 return;
623
624out_blackhole:
625 res->fib6_flags |= RTF_REJECT;
626 res->fib6_type = RTN_BLACKHOLE;
627 res->nh = nh;
628}
629
630#ifdef CONFIG_IPV6_ROUTER_PREF
631struct __rt6_probe_work {
632 struct work_struct work;
633 struct in6_addr target;
634 struct net_device *dev;
635 netdevice_tracker dev_tracker;
636};
637
638static void rt6_probe_deferred(struct work_struct *w)
639{
640 struct in6_addr mcaddr;
641 struct __rt6_probe_work *work =
642 container_of(w, struct __rt6_probe_work, work);
643
644 addrconf_addr_solict_mult(&work->target, &mcaddr);
645 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0);
646 netdev_put(work->dev, &work->dev_tracker);
647 kfree(work);
648}
649
650static void rt6_probe(struct fib6_nh *fib6_nh)
651{
652 struct __rt6_probe_work *work = NULL;
653 const struct in6_addr *nh_gw;
654 unsigned long last_probe;
655 struct neighbour *neigh;
656 struct net_device *dev;
657 struct inet6_dev *idev;
658
659 /*
660 * Okay, this does not seem to be appropriate
661 * for now, however, we need to check if it
662 * is really so; aka Router Reachability Probing.
663 *
664 * Router Reachability Probe MUST be rate-limited
665 * to no more than one per minute.
666 */
667 if (!fib6_nh->fib_nh_gw_family)
668 return;
669
670 nh_gw = &fib6_nh->fib_nh_gw6;
671 dev = fib6_nh->fib_nh_dev;
672 rcu_read_lock();
673 last_probe = READ_ONCE(fib6_nh->last_probe);
674 idev = __in6_dev_get(dev);
675 if (!idev)
676 goto out;
677 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
678 if (neigh) {
679 if (READ_ONCE(neigh->nud_state) & NUD_VALID)
680 goto out;
681
682 write_lock_bh(&neigh->lock);
683 if (!(neigh->nud_state & NUD_VALID) &&
684 time_after(jiffies,
685 neigh->updated +
686 READ_ONCE(idev->cnf.rtr_probe_interval))) {
687 work = kmalloc(sizeof(*work), GFP_ATOMIC);
688 if (work)
689 __neigh_set_probe_once(neigh);
690 }
691 write_unlock_bh(&neigh->lock);
692 } else if (time_after(jiffies, last_probe +
693 READ_ONCE(idev->cnf.rtr_probe_interval))) {
694 work = kmalloc(sizeof(*work), GFP_ATOMIC);
695 }
696
697 if (!work || cmpxchg(&fib6_nh->last_probe,
698 last_probe, jiffies) != last_probe) {
699 kfree(work);
700 } else {
701 INIT_WORK(&work->work, rt6_probe_deferred);
702 work->target = *nh_gw;
703 netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC);
704 work->dev = dev;
705 schedule_work(&work->work);
706 }
707
708out:
709 rcu_read_unlock();
710}
711#else
712static inline void rt6_probe(struct fib6_nh *fib6_nh)
713{
714}
715#endif
716
717/*
718 * Default Router Selection (RFC 2461 6.3.6)
719 */
720static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)
721{
722 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD;
723 struct neighbour *neigh;
724
725 rcu_read_lock();
726 neigh = __ipv6_neigh_lookup_noref(dev: fib6_nh->fib_nh_dev,
727 pkey: &fib6_nh->fib_nh_gw6);
728 if (neigh) {
729 u8 nud_state = READ_ONCE(neigh->nud_state);
730
731 if (nud_state & NUD_VALID)
732 ret = RT6_NUD_SUCCEED;
733#ifdef CONFIG_IPV6_ROUTER_PREF
734 else if (!(nud_state & NUD_FAILED))
735 ret = RT6_NUD_SUCCEED;
736 else
737 ret = RT6_NUD_FAIL_PROBE;
738#endif
739 } else {
740 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ?
741 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR;
742 }
743 rcu_read_unlock();
744
745 return ret;
746}
747
748static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
749 int strict)
750{
751 int m = 0;
752
753 if (!oif || nh->fib_nh_dev->ifindex == oif)
754 m = 2;
755
756 if (!m && (strict & RT6_LOOKUP_F_IFACE))
757 return RT6_NUD_FAIL_HARD;
758#ifdef CONFIG_IPV6_ROUTER_PREF
759 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2;
760#endif
761 if ((strict & RT6_LOOKUP_F_REACHABLE) &&
762 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {
763 int n = rt6_check_neigh(fib6_nh: nh);
764 if (n < 0)
765 return n;
766 }
767 return m;
768}
769
770static bool find_match(struct fib6_nh *nh, u32 fib6_flags,
771 int oif, int strict, int *mpri, bool *do_rr)
772{
773 bool match_do_rr = false;
774 bool rc = false;
775 int m;
776
777 if (nh->fib_nh_flags & RTNH_F_DEAD)
778 goto out;
779
780 if (ip6_ignore_linkdown(dev: nh->fib_nh_dev) &&
781 nh->fib_nh_flags & RTNH_F_LINKDOWN &&
782 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE))
783 goto out;
784
785 m = rt6_score_route(nh, fib6_flags, oif, strict);
786 if (m == RT6_NUD_FAIL_DO_RR) {
787 match_do_rr = true;
788 m = 0; /* lowest valid score */
789 } else if (m == RT6_NUD_FAIL_HARD) {
790 goto out;
791 }
792
793 if (strict & RT6_LOOKUP_F_REACHABLE)
794 rt6_probe(fib6_nh: nh);
795
796 /* note that m can be RT6_NUD_FAIL_PROBE at this point */
797 if (m > *mpri) {
798 *do_rr = match_do_rr;
799 *mpri = m;
800 rc = true;
801 }
802out:
803 return rc;
804}
805
806struct fib6_nh_frl_arg {
807 u32 flags;
808 int oif;
809 int strict;
810 int *mpri;
811 bool *do_rr;
812 struct fib6_nh *nh;
813};
814
815static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)
816{
817 struct fib6_nh_frl_arg *arg = _arg;
818
819 arg->nh = nh;
820 return find_match(nh, fib6_flags: arg->flags, oif: arg->oif, strict: arg->strict,
821 mpri: arg->mpri, do_rr: arg->do_rr);
822}
823
824static void __find_rr_leaf(struct fib6_info *f6i_start,
825 struct fib6_info *nomatch, u32 metric,
826 struct fib6_result *res, struct fib6_info **cont,
827 int oif, int strict, bool *do_rr, int *mpri)
828{
829 struct fib6_info *f6i;
830
831 for (f6i = f6i_start;
832 f6i && f6i != nomatch;
833 f6i = rcu_dereference(f6i->fib6_next)) {
834 bool matched = false;
835 struct fib6_nh *nh;
836
837 if (cont && f6i->fib6_metric != metric) {
838 *cont = f6i;
839 return;
840 }
841
842 if (fib6_check_expired(f6i))
843 continue;
844
845 if (unlikely(f6i->nh)) {
846 struct fib6_nh_frl_arg arg = {
847 .flags = f6i->fib6_flags,
848 .oif = oif,
849 .strict = strict,
850 .mpri = mpri,
851 .do_rr = do_rr
852 };
853
854 if (nexthop_is_blackhole(nh: f6i->nh)) {
855 res->fib6_flags = RTF_REJECT;
856 res->fib6_type = RTN_BLACKHOLE;
857 res->f6i = f6i;
858 res->nh = nexthop_fib6_nh(nh: f6i->nh);
859 return;
860 }
861 if (nexthop_for_each_fib6_nh(nh: f6i->nh, cb: rt6_nh_find_match,
862 arg: &arg)) {
863 matched = true;
864 nh = arg.nh;
865 }
866 } else {
867 nh = f6i->fib6_nh;
868 if (find_match(nh, fib6_flags: f6i->fib6_flags, oif, strict,
869 mpri, do_rr))
870 matched = true;
871 }
872 if (matched) {
873 res->f6i = f6i;
874 res->nh = nh;
875 res->fib6_flags = f6i->fib6_flags;
876 res->fib6_type = f6i->fib6_type;
877 }
878 }
879}
880
881static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf,
882 struct fib6_info *rr_head, int oif, int strict,
883 bool *do_rr, struct fib6_result *res)
884{
885 u32 metric = rr_head->fib6_metric;
886 struct fib6_info *cont = NULL;
887 int mpri = -1;
888
889 __find_rr_leaf(f6i_start: rr_head, NULL, metric, res, cont: &cont,
890 oif, strict, do_rr, mpri: &mpri);
891
892 __find_rr_leaf(f6i_start: leaf, nomatch: rr_head, metric, res, cont: &cont,
893 oif, strict, do_rr, mpri: &mpri);
894
895 if (res->f6i || !cont)
896 return;
897
898 __find_rr_leaf(f6i_start: cont, NULL, metric, res, NULL,
899 oif, strict, do_rr, mpri: &mpri);
900}
901
902static void rt6_select(struct net *net, struct fib6_node *fn, int oif,
903 struct fib6_result *res, int strict)
904{
905 struct fib6_info *leaf = rcu_dereference(fn->leaf);
906 struct fib6_info *rt0;
907 bool do_rr = false;
908 int key_plen;
909
910 /* make sure this function or its helpers sets f6i */
911 res->f6i = NULL;
912
913 if (!leaf || leaf == net->ipv6.fib6_null_entry)
914 goto out;
915
916 rt0 = rcu_dereference(fn->rr_ptr);
917 if (!rt0)
918 rt0 = leaf;
919
920 /* Double check to make sure fn is not an intermediate node
921 * and fn->leaf does not points to its child's leaf
922 * (This might happen if all routes under fn are deleted from
923 * the tree and fib6_repair_tree() is called on the node.)
924 */
925 key_plen = rt0->fib6_dst.plen;
926#ifdef CONFIG_IPV6_SUBTREES
927 if (rt0->fib6_src.plen)
928 key_plen = rt0->fib6_src.plen;
929#endif
930 if (fn->fn_bit != key_plen)
931 goto out;
932
933 find_rr_leaf(fn, leaf, rr_head: rt0, oif, strict, do_rr: &do_rr, res);
934 if (do_rr) {
935 struct fib6_info *next = rcu_dereference(rt0->fib6_next);
936
937 /* no entries matched; do round-robin */
938 if (!next || next->fib6_metric != rt0->fib6_metric)
939 next = leaf;
940
941 if (next != rt0) {
942 spin_lock_bh(lock: &leaf->fib6_table->tb6_lock);
943 /* make sure next is not being deleted from the tree */
944 if (next->fib6_node)
945 rcu_assign_pointer(fn->rr_ptr, next);
946 spin_unlock_bh(lock: &leaf->fib6_table->tb6_lock);
947 }
948 }
949
950out:
951 if (!res->f6i) {
952 res->f6i = net->ipv6.fib6_null_entry;
953 res->nh = res->f6i->fib6_nh;
954 res->fib6_flags = res->f6i->fib6_flags;
955 res->fib6_type = res->f6i->fib6_type;
956 }
957}
958
959static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res)
960{
961 return (res->f6i->fib6_flags & RTF_NONEXTHOP) ||
962 res->nh->fib_nh_gw_family;
963}
964
965#ifdef CONFIG_IPV6_ROUTE_INFO
966int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
967 const struct in6_addr *gwaddr)
968{
969 struct net *net = dev_net(dev);
970 struct route_info *rinfo = (struct route_info *) opt;
971 struct in6_addr prefix_buf, *prefix;
972 struct fib6_table *table;
973 unsigned int pref;
974 unsigned long lifetime;
975 struct fib6_info *rt;
976
977 if (len < sizeof(struct route_info)) {
978 return -EINVAL;
979 }
980
981 /* Sanity check for prefix_len and length */
982 if (rinfo->length > 3) {
983 return -EINVAL;
984 } else if (rinfo->prefix_len > 128) {
985 return -EINVAL;
986 } else if (rinfo->prefix_len > 64) {
987 if (rinfo->length < 2) {
988 return -EINVAL;
989 }
990 } else if (rinfo->prefix_len > 0) {
991 if (rinfo->length < 1) {
992 return -EINVAL;
993 }
994 }
995
996 pref = rinfo->route_pref;
997 if (pref == ICMPV6_ROUTER_PREF_INVALID)
998 return -EINVAL;
999
1000 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ);
1001
1002 if (rinfo->length == 3)
1003 prefix = (struct in6_addr *)rinfo->prefix;
1004 else {
1005 /* this function is safe */
1006 ipv6_addr_prefix(&prefix_buf,
1007 (struct in6_addr *)rinfo->prefix,
1008 rinfo->prefix_len);
1009 prefix = &prefix_buf;
1010 }
1011
1012 if (rinfo->prefix_len == 0)
1013 rt = rt6_get_dflt_router(net, gwaddr, dev);
1014 else
1015 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len,
1016 gwaddr, dev);
1017
1018 if (rt && !lifetime) {
1019 ip6_del_rt(net, rt, false);
1020 rt = NULL;
1021 }
1022
1023 if (!rt && lifetime)
1024 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr,
1025 dev, pref);
1026 else if (rt)
1027 rt->fib6_flags = RTF_ROUTEINFO |
1028 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);
1029
1030 if (rt) {
1031 table = rt->fib6_table;
1032 spin_lock_bh(&table->tb6_lock);
1033
1034 if (!addrconf_finite_timeout(lifetime)) {
1035 fib6_clean_expires(rt);
1036 fib6_remove_gc_list(rt);
1037 } else {
1038 fib6_set_expires(rt, jiffies + HZ * lifetime);
1039 fib6_add_gc_list(rt);
1040 }
1041
1042 spin_unlock_bh(&table->tb6_lock);
1043
1044 fib6_info_release(rt);
1045 }
1046 return 0;
1047}
1048#endif
1049
1050/*
1051 * Misc support functions
1052 */
1053
1054/* called with rcu_lock held */
1055static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res)
1056{
1057 struct net_device *dev = res->nh->fib_nh_dev;
1058
1059 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) {
1060 /* for copies of local routes, dst->dev needs to be the
1061 * device if it is a master device, the master device if
1062 * device is enslaved, and the loopback as the default
1063 */
1064 if (netif_is_l3_slave(dev) &&
1065 !rt6_need_strict(daddr: &res->f6i->fib6_dst.addr))
1066 dev = l3mdev_master_dev_rcu(dev);
1067 else if (!netif_is_l3_master(dev))
1068 dev = dev_net(dev)->loopback_dev;
1069 /* last case is netif_is_l3_master(dev) is true in which
1070 * case we want dev returned to be dev
1071 */
1072 }
1073
1074 return dev;
1075}
1076
1077static const int fib6_prop[RTN_MAX + 1] = {
1078 [RTN_UNSPEC] = 0,
1079 [RTN_UNICAST] = 0,
1080 [RTN_LOCAL] = 0,
1081 [RTN_BROADCAST] = 0,
1082 [RTN_ANYCAST] = 0,
1083 [RTN_MULTICAST] = 0,
1084 [RTN_BLACKHOLE] = -EINVAL,
1085 [RTN_UNREACHABLE] = -EHOSTUNREACH,
1086 [RTN_PROHIBIT] = -EACCES,
1087 [RTN_THROW] = -EAGAIN,
1088 [RTN_NAT] = -EINVAL,
1089 [RTN_XRESOLVE] = -EINVAL,
1090};
1091
1092static int ip6_rt_type_to_error(u8 fib6_type)
1093{
1094 return fib6_prop[fib6_type];
1095}
1096
1097static unsigned short fib6_info_dst_flags(struct fib6_info *rt)
1098{
1099 unsigned short flags = 0;
1100
1101 if (rt->dst_nocount)
1102 flags |= DST_NOCOUNT;
1103 if (rt->dst_nopolicy)
1104 flags |= DST_NOPOLICY;
1105
1106 return flags;
1107}
1108
1109static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type)
1110{
1111 rt->dst.error = ip6_rt_type_to_error(fib6_type);
1112
1113 switch (fib6_type) {
1114 case RTN_BLACKHOLE:
1115 rt->dst.output = dst_discard_out;
1116 rt->dst.input = dst_discard;
1117 break;
1118 case RTN_PROHIBIT:
1119 rt->dst.output = ip6_pkt_prohibit_out;
1120 rt->dst.input = ip6_pkt_prohibit;
1121 break;
1122 case RTN_THROW:
1123 case RTN_UNREACHABLE:
1124 default:
1125 rt->dst.output = ip6_pkt_discard_out;
1126 rt->dst.input = ip6_pkt_discard;
1127 break;
1128 }
1129}
1130
1131static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res)
1132{
1133 struct fib6_info *f6i = res->f6i;
1134
1135 if (res->fib6_flags & RTF_REJECT) {
1136 ip6_rt_init_dst_reject(rt, fib6_type: res->fib6_type);
1137 return;
1138 }
1139
1140 rt->dst.error = 0;
1141 rt->dst.output = ip6_output;
1142
1143 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) {
1144 rt->dst.input = ip6_input;
1145 } else if (ipv6_addr_type(addr: &f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) {
1146 rt->dst.input = ip6_mc_input;
1147 rt->dst.output = ip6_mr_output;
1148 } else {
1149 rt->dst.input = ip6_forward;
1150 }
1151
1152 if (res->nh->fib_nh_lws) {
1153 rt->dst.lwtstate = lwtstate_get(lws: res->nh->fib_nh_lws);
1154 lwtunnel_set_redirect(dst: &rt->dst);
1155 }
1156
1157 rt->dst.lastuse = jiffies;
1158}
1159
1160/* Caller must already hold reference to @from */
1161static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
1162{
1163 rt->rt6i_flags &= ~RTF_EXPIRES;
1164 rcu_assign_pointer(rt->from, from);
1165 ip_dst_init_metrics(dst: &rt->dst, fib_metrics: from->fib6_metrics);
1166}
1167
1168/* Caller must already hold reference to f6i in result */
1169static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res)
1170{
1171 const struct fib6_nh *nh = res->nh;
1172 const struct net_device *dev = nh->fib_nh_dev;
1173 struct fib6_info *f6i = res->f6i;
1174
1175 ip6_rt_init_dst(rt, res);
1176
1177 rt->rt6i_dst = f6i->fib6_dst;
1178 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL;
1179 rt->rt6i_flags = res->fib6_flags;
1180 if (nh->fib_nh_gw_family) {
1181 rt->rt6i_gateway = nh->fib_nh_gw6;
1182 rt->rt6i_flags |= RTF_GATEWAY;
1183 }
1184 rt6_set_from(rt, from: f6i);
1185#ifdef CONFIG_IPV6_SUBTREES
1186 rt->rt6i_src = f6i->fib6_src;
1187#endif
1188}
1189
1190static struct fib6_node* fib6_backtrack(struct fib6_node *fn,
1191 struct in6_addr *saddr)
1192{
1193 struct fib6_node *pn, *sn;
1194 while (1) {
1195 if (fn->fn_flags & RTN_TL_ROOT)
1196 return NULL;
1197 pn = rcu_dereference(fn->parent);
1198 sn = FIB6_SUBTREE(pn);
1199 if (sn && sn != fn)
1200 fn = fib6_node_lookup(root: sn, NULL, saddr);
1201 else
1202 fn = pn;
1203 if (fn->fn_flags & RTN_RTINFO)
1204 return fn;
1205 }
1206}
1207
1208static bool ip6_hold_safe(struct net *net, struct rt6_info **prt)
1209{
1210 struct rt6_info *rt = *prt;
1211
1212 if (dst_hold_safe(dst: &rt->dst))
1213 return true;
1214 if (net) {
1215 rt = net->ipv6.ip6_null_entry;
1216 dst_hold(dst: &rt->dst);
1217 } else {
1218 rt = NULL;
1219 }
1220 *prt = rt;
1221 return false;
1222}
1223
1224/* called with rcu_lock held */
1225static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res)
1226{
1227 struct net_device *dev = res->nh->fib_nh_dev;
1228 struct fib6_info *f6i = res->f6i;
1229 unsigned short flags;
1230 struct rt6_info *nrt;
1231
1232 if (!fib6_info_hold_safe(f6i))
1233 goto fallback;
1234
1235 flags = fib6_info_dst_flags(rt: f6i);
1236 nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
1237 if (!nrt) {
1238 fib6_info_release(f6i);
1239 goto fallback;
1240 }
1241
1242 ip6_rt_copy_init(rt: nrt, res);
1243 return nrt;
1244
1245fallback:
1246 nrt = dev_net(dev)->ipv6.ip6_null_entry;
1247 dst_hold(dst: &nrt->dst);
1248 return nrt;
1249}
1250
1251INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net,
1252 struct fib6_table *table,
1253 struct flowi6 *fl6,
1254 const struct sk_buff *skb,
1255 int flags)
1256{
1257 struct fib6_result res = {};
1258 struct fib6_node *fn;
1259 struct rt6_info *rt;
1260
1261 rcu_read_lock();
1262 fn = fib6_node_lookup(root: &table->tb6_root, daddr: &fl6->daddr, saddr: &fl6->saddr);
1263restart:
1264 res.f6i = rcu_dereference(fn->leaf);
1265 if (!res.f6i)
1266 res.f6i = net->ipv6.fib6_null_entry;
1267 else
1268 rt6_device_match(net, res: &res, saddr: &fl6->saddr, oif: fl6->flowi6_oif,
1269 flags);
1270
1271 if (res.f6i == net->ipv6.fib6_null_entry) {
1272 fn = fib6_backtrack(fn, saddr: &fl6->saddr);
1273 if (fn)
1274 goto restart;
1275
1276 rt = net->ipv6.ip6_null_entry;
1277 dst_hold(dst: &rt->dst);
1278 goto out;
1279 } else if (res.fib6_flags & RTF_REJECT) {
1280 goto do_create;
1281 }
1282
1283 fib6_select_path(net, res: &res, fl6, oif: fl6->flowi6_oif,
1284 have_oif_match: fl6->flowi6_oif != 0, skb, strict: flags);
1285
1286 /* Search through exception table */
1287 rt = rt6_find_cached_rt(res: &res, daddr: &fl6->daddr, saddr: &fl6->saddr);
1288 if (rt) {
1289 if (ip6_hold_safe(net, prt: &rt))
1290 dst_use_noref(dst: &rt->dst, time: jiffies);
1291 } else {
1292do_create:
1293 rt = ip6_create_rt_rcu(res: &res);
1294 }
1295
1296out:
1297 trace_fib6_table_lookup(net, res: &res, table, flp: fl6);
1298
1299 rcu_read_unlock();
1300
1301 return rt;
1302}
1303
1304struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
1305 const struct sk_buff *skb, int flags)
1306{
1307 return fib6_rule_lookup(net, fl6, skb, flags, lookup: ip6_pol_route_lookup);
1308}
1309EXPORT_SYMBOL_GPL(ip6_route_lookup);
1310
1311struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
1312 const struct in6_addr *saddr, int oif,
1313 const struct sk_buff *skb, int strict)
1314{
1315 struct flowi6 fl6 = {
1316 .flowi6_oif = oif,
1317 .daddr = *daddr,
1318 };
1319 struct dst_entry *dst;
1320 int flags = strict ? RT6_LOOKUP_F_IFACE : 0;
1321
1322 if (saddr) {
1323 memcpy(to: &fl6.saddr, from: saddr, len: sizeof(*saddr));
1324 flags |= RT6_LOOKUP_F_HAS_SADDR;
1325 }
1326
1327 dst = fib6_rule_lookup(net, fl6: &fl6, skb, flags, lookup: ip6_pol_route_lookup);
1328 if (dst->error == 0)
1329 return dst_rt6_info(dst);
1330
1331 dst_release(dst);
1332
1333 return NULL;
1334}
1335EXPORT_SYMBOL(rt6_lookup);
1336
1337/* ip6_ins_rt is called with FREE table->tb6_lock.
1338 * It takes new route entry, the addition fails by any reason the
1339 * route is released.
1340 * Caller must hold dst before calling it.
1341 */
1342
1343static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info,
1344 struct netlink_ext_ack *extack)
1345{
1346 int err;
1347 struct fib6_table *table;
1348
1349 table = rt->fib6_table;
1350 spin_lock_bh(lock: &table->tb6_lock);
1351 err = fib6_add(root: &table->tb6_root, rt, info, extack);
1352 spin_unlock_bh(lock: &table->tb6_lock);
1353
1354 return err;
1355}
1356
1357int ip6_ins_rt(struct net *net, struct fib6_info *rt)
1358{
1359 struct nl_info info = { .nl_net = net, };
1360
1361 return __ip6_ins_rt(rt, info: &info, NULL);
1362}
1363
1364static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res,
1365 const struct in6_addr *daddr,
1366 const struct in6_addr *saddr)
1367{
1368 struct fib6_info *f6i = res->f6i;
1369 struct net_device *dev;
1370 struct rt6_info *rt;
1371
1372 /*
1373 * Clone the route.
1374 */
1375
1376 if (!fib6_info_hold_safe(f6i))
1377 return NULL;
1378
1379 dev = ip6_rt_get_dev_rcu(res);
1380 rt = ip6_dst_alloc(dev_net(dev), dev, 0);
1381 if (!rt) {
1382 fib6_info_release(f6i);
1383 return NULL;
1384 }
1385
1386 ip6_rt_copy_init(rt, res);
1387 rt->rt6i_flags |= RTF_CACHE;
1388 rt->rt6i_dst.addr = *daddr;
1389 rt->rt6i_dst.plen = 128;
1390
1391 if (!rt6_is_gw_or_nonexthop(res)) {
1392 if (f6i->fib6_dst.plen != 128 &&
1393 ipv6_addr_equal(a1: &f6i->fib6_dst.addr, a2: daddr))
1394 rt->rt6i_flags |= RTF_ANYCAST;
1395#ifdef CONFIG_IPV6_SUBTREES
1396 if (rt->rt6i_src.plen && saddr) {
1397 rt->rt6i_src.addr = *saddr;
1398 rt->rt6i_src.plen = 128;
1399 }
1400#endif
1401 }
1402
1403 return rt;
1404}
1405
1406static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res)
1407{
1408 struct fib6_info *f6i = res->f6i;
1409 unsigned short flags = fib6_info_dst_flags(rt: f6i);
1410 struct net_device *dev;
1411 struct rt6_info *pcpu_rt;
1412
1413 if (!fib6_info_hold_safe(f6i))
1414 return NULL;
1415
1416 rcu_read_lock();
1417 dev = ip6_rt_get_dev_rcu(res);
1418 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT);
1419 rcu_read_unlock();
1420 if (!pcpu_rt) {
1421 fib6_info_release(f6i);
1422 return NULL;
1423 }
1424 ip6_rt_copy_init(rt: pcpu_rt, res);
1425 pcpu_rt->rt6i_flags |= RTF_PCPU;
1426
1427 if (f6i->nh)
1428 pcpu_rt->sernum = rt_genid_ipv6(net: dev_net(dev));
1429
1430 return pcpu_rt;
1431}
1432
1433static bool rt6_is_valid(const struct rt6_info *rt6)
1434{
1435 return rt6->sernum == rt_genid_ipv6(net: dev_net(dev: rt6->dst.dev));
1436}
1437
1438/* It should be called with rcu_read_lock() acquired */
1439static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res)
1440{
1441 struct rt6_info *pcpu_rt;
1442
1443 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);
1444
1445 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(rt6: pcpu_rt)) {
1446 struct rt6_info *prev, **p;
1447
1448 p = this_cpu_ptr(res->nh->rt6i_pcpu);
1449 /* Paired with READ_ONCE() in __fib6_drop_pcpu_from() */
1450 prev = xchg(p, NULL);
1451 if (prev) {
1452 dst_dev_put(dst: &prev->dst);
1453 dst_release(dst: &prev->dst);
1454 }
1455
1456 pcpu_rt = NULL;
1457 }
1458
1459 return pcpu_rt;
1460}
1461
1462static struct rt6_info *rt6_make_pcpu_route(struct net *net,
1463 const struct fib6_result *res)
1464{
1465 struct rt6_info *pcpu_rt, *prev, **p;
1466
1467 pcpu_rt = ip6_rt_pcpu_alloc(res);
1468 if (!pcpu_rt)
1469 return NULL;
1470
1471 p = this_cpu_ptr(res->nh->rt6i_pcpu);
1472 prev = cmpxchg(p, NULL, pcpu_rt);
1473 BUG_ON(prev);
1474
1475 if (res->f6i->fib6_destroying) {
1476 struct fib6_info *from;
1477
1478 from = unrcu_pointer(xchg(&pcpu_rt->from, NULL));
1479 fib6_info_release(f6i: from);
1480 }
1481
1482 return pcpu_rt;
1483}
1484
1485/* exception hash table implementation
1486 */
1487static DEFINE_SPINLOCK(rt6_exception_lock);
1488
1489/* Remove rt6_ex from hash table and free the memory
1490 * Caller must hold rt6_exception_lock
1491 */
1492static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
1493 struct rt6_exception *rt6_ex)
1494{
1495 struct net *net;
1496
1497 if (!bucket || !rt6_ex)
1498 return;
1499
1500 net = dev_net(dev: rt6_ex->rt6i->dst.dev);
1501 net->ipv6.rt6_stats->fib_rt_cache--;
1502
1503 /* purge completely the exception to allow releasing the held resources:
1504 * some [sk] cache may keep the dst around for unlimited time
1505 */
1506 dst_dev_put(dst: &rt6_ex->rt6i->dst);
1507
1508 hlist_del_rcu(n: &rt6_ex->hlist);
1509 dst_release(dst: &rt6_ex->rt6i->dst);
1510 kfree_rcu(rt6_ex, rcu);
1511 WARN_ON_ONCE(!bucket->depth);
1512 bucket->depth--;
1513}
1514
1515/* Remove oldest rt6_ex in bucket and free the memory
1516 * Caller must hold rt6_exception_lock
1517 */
1518static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
1519{
1520 struct rt6_exception *rt6_ex, *oldest = NULL;
1521
1522 if (!bucket)
1523 return;
1524
1525 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
1526 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp))
1527 oldest = rt6_ex;
1528 }
1529 rt6_remove_exception(bucket, rt6_ex: oldest);
1530}
1531
1532static u32 rt6_exception_hash(const struct in6_addr *dst,
1533 const struct in6_addr *src)
1534{
1535 static siphash_aligned_key_t rt6_exception_key;
1536 struct {
1537 struct in6_addr dst;
1538 struct in6_addr src;
1539 } __aligned(SIPHASH_ALIGNMENT) combined = {
1540 .dst = *dst,
1541 };
1542 u64 val;
1543
1544 net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key));
1545
1546#ifdef CONFIG_IPV6_SUBTREES
1547 if (src)
1548 combined.src = *src;
1549#endif
1550 val = siphash(data: &combined, len: sizeof(combined), key: &rt6_exception_key);
1551
1552 return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT);
1553}
1554
1555/* Helper function to find the cached rt in the hash table
1556 * and update bucket pointer to point to the bucket for this
1557 * (daddr, saddr) pair
1558 * Caller must hold rt6_exception_lock
1559 */
1560static struct rt6_exception *
1561__rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket,
1562 const struct in6_addr *daddr,
1563 const struct in6_addr *saddr)
1564{
1565 struct rt6_exception *rt6_ex;
1566 u32 hval;
1567
1568 if (!(*bucket) || !daddr)
1569 return NULL;
1570
1571 hval = rt6_exception_hash(dst: daddr, src: saddr);
1572 *bucket += hval;
1573
1574 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) {
1575 struct rt6_info *rt6 = rt6_ex->rt6i;
1576 bool matched = ipv6_addr_equal(a1: daddr, a2: &rt6->rt6i_dst.addr);
1577
1578#ifdef CONFIG_IPV6_SUBTREES
1579 if (matched && saddr)
1580 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1581#endif
1582 if (matched)
1583 return rt6_ex;
1584 }
1585 return NULL;
1586}
1587
1588/* Helper function to find the cached rt in the hash table
1589 * and update bucket pointer to point to the bucket for this
1590 * (daddr, saddr) pair
1591 * Caller must hold rcu_read_lock()
1592 */
1593static struct rt6_exception *
1594__rt6_find_exception_rcu(struct rt6_exception_bucket **bucket,
1595 const struct in6_addr *daddr,
1596 const struct in6_addr *saddr)
1597{
1598 struct rt6_exception *rt6_ex;
1599 u32 hval;
1600
1601 WARN_ON_ONCE(!rcu_read_lock_held());
1602
1603 if (!(*bucket) || !daddr)
1604 return NULL;
1605
1606 hval = rt6_exception_hash(dst: daddr, src: saddr);
1607 *bucket += hval;
1608
1609 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) {
1610 struct rt6_info *rt6 = rt6_ex->rt6i;
1611 bool matched = ipv6_addr_equal(a1: daddr, a2: &rt6->rt6i_dst.addr);
1612
1613#ifdef CONFIG_IPV6_SUBTREES
1614 if (matched && saddr)
1615 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1616#endif
1617 if (matched)
1618 return rt6_ex;
1619 }
1620 return NULL;
1621}
1622
1623static unsigned int fib6_mtu(const struct fib6_result *res)
1624{
1625 const struct fib6_nh *nh = res->nh;
1626 unsigned int mtu;
1627
1628 if (res->f6i->fib6_pmtu) {
1629 mtu = res->f6i->fib6_pmtu;
1630 } else {
1631 struct net_device *dev = nh->fib_nh_dev;
1632 struct inet6_dev *idev;
1633
1634 rcu_read_lock();
1635 idev = __in6_dev_get(dev);
1636 mtu = READ_ONCE(idev->cnf.mtu6);
1637 rcu_read_unlock();
1638 }
1639
1640 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
1641
1642 return mtu - lwtunnel_headroom(lwtstate: nh->fib_nh_lws, mtu);
1643}
1644
1645#define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL
1646
1647/* used when the flushed bit is not relevant, only access to the bucket
1648 * (ie., all bucket users except rt6_insert_exception);
1649 *
1650 * called under rcu lock; sometimes called with rt6_exception_lock held
1651 */
1652static
1653struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,
1654 spinlock_t *lock)
1655{
1656 struct rt6_exception_bucket *bucket;
1657
1658 if (lock)
1659 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1660 lockdep_is_held(lock));
1661 else
1662 bucket = rcu_dereference(nh->rt6i_exception_bucket);
1663
1664 /* remove bucket flushed bit if set */
1665 if (bucket) {
1666 unsigned long p = (unsigned long)bucket;
1667
1668 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED;
1669 bucket = (struct rt6_exception_bucket *)p;
1670 }
1671
1672 return bucket;
1673}
1674
1675static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket)
1676{
1677 unsigned long p = (unsigned long)bucket;
1678
1679 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED);
1680}
1681
1682/* called with rt6_exception_lock held */
1683static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,
1684 spinlock_t *lock)
1685{
1686 struct rt6_exception_bucket *bucket;
1687 unsigned long p;
1688
1689 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1690 lockdep_is_held(lock));
1691
1692 p = (unsigned long)bucket;
1693 p |= FIB6_EXCEPTION_BUCKET_FLUSHED;
1694 bucket = (struct rt6_exception_bucket *)p;
1695 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
1696}
1697
1698static int rt6_insert_exception(struct rt6_info *nrt,
1699 const struct fib6_result *res)
1700{
1701 struct net *net = dev_net(dev: nrt->dst.dev);
1702 struct rt6_exception_bucket *bucket;
1703 struct fib6_info *f6i = res->f6i;
1704 struct in6_addr *src_key = NULL;
1705 struct rt6_exception *rt6_ex;
1706 struct fib6_nh *nh = res->nh;
1707 int max_depth;
1708 int err = 0;
1709
1710 spin_lock_bh(lock: &rt6_exception_lock);
1711
1712 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1713 lockdep_is_held(&rt6_exception_lock));
1714 if (!bucket) {
1715 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket),
1716 GFP_ATOMIC);
1717 if (!bucket) {
1718 err = -ENOMEM;
1719 goto out;
1720 }
1721 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
1722 } else if (fib6_nh_excptn_bucket_flushed(bucket)) {
1723 err = -EINVAL;
1724 goto out;
1725 }
1726
1727#ifdef CONFIG_IPV6_SUBTREES
1728 /* fib6_src.plen != 0 indicates f6i is in subtree
1729 * and exception table is indexed by a hash of
1730 * both fib6_dst and fib6_src.
1731 * Otherwise, the exception table is indexed by
1732 * a hash of only fib6_dst.
1733 */
1734 if (f6i->fib6_src.plen)
1735 src_key = &nrt->rt6i_src.addr;
1736#endif
1737 /* rt6_mtu_change() might lower mtu on f6i.
1738 * Only insert this exception route if its mtu
1739 * is less than f6i's mtu value.
1740 */
1741 if (dst_metric_raw(dst: &nrt->dst, RTAX_MTU) >= fib6_mtu(res)) {
1742 err = -EINVAL;
1743 goto out;
1744 }
1745
1746 rt6_ex = __rt6_find_exception_spinlock(bucket: &bucket, daddr: &nrt->rt6i_dst.addr,
1747 saddr: src_key);
1748 if (rt6_ex)
1749 rt6_remove_exception(bucket, rt6_ex);
1750
1751 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC);
1752 if (!rt6_ex) {
1753 err = -ENOMEM;
1754 goto out;
1755 }
1756 rt6_ex->rt6i = nrt;
1757 rt6_ex->stamp = jiffies;
1758 hlist_add_head_rcu(n: &rt6_ex->hlist, h: &bucket->chain);
1759 bucket->depth++;
1760 net->ipv6.rt6_stats->fib_rt_cache++;
1761
1762 /* Randomize max depth to avoid some side channels attacks. */
1763 max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH);
1764 while (bucket->depth > max_depth)
1765 rt6_exception_remove_oldest(bucket);
1766
1767out:
1768 spin_unlock_bh(lock: &rt6_exception_lock);
1769
1770 /* Update fn->fn_sernum to invalidate all cached dst */
1771 if (!err) {
1772 spin_lock_bh(lock: &f6i->fib6_table->tb6_lock);
1773 fib6_update_sernum(net, rt: f6i);
1774 fib6_add_gc_list(f6i);
1775 spin_unlock_bh(lock: &f6i->fib6_table->tb6_lock);
1776 fib6_force_start_gc(net);
1777 }
1778
1779 return err;
1780}
1781
1782static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)
1783{
1784 struct rt6_exception_bucket *bucket;
1785 struct rt6_exception *rt6_ex;
1786 struct hlist_node *tmp;
1787 int i;
1788
1789 spin_lock_bh(lock: &rt6_exception_lock);
1790
1791 bucket = fib6_nh_get_excptn_bucket(nh, lock: &rt6_exception_lock);
1792 if (!bucket)
1793 goto out;
1794
1795 /* Prevent rt6_insert_exception() to recreate the bucket list */
1796 if (!from)
1797 fib6_nh_excptn_bucket_set_flushed(nh, lock: &rt6_exception_lock);
1798
1799 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
1800 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) {
1801 if (!from ||
1802 rcu_access_pointer(rt6_ex->rt6i->from) == from)
1803 rt6_remove_exception(bucket, rt6_ex);
1804 }
1805 WARN_ON_ONCE(!from && bucket->depth);
1806 bucket++;
1807 }
1808out:
1809 spin_unlock_bh(lock: &rt6_exception_lock);
1810}
1811
1812static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)
1813{
1814 struct fib6_info *f6i = arg;
1815
1816 fib6_nh_flush_exceptions(nh, from: f6i);
1817
1818 return 0;
1819}
1820
1821void rt6_flush_exceptions(struct fib6_info *f6i)
1822{
1823 if (f6i->nh) {
1824 rcu_read_lock();
1825 nexthop_for_each_fib6_nh(nh: f6i->nh, cb: rt6_nh_flush_exceptions, arg: f6i);
1826 rcu_read_unlock();
1827 } else {
1828 fib6_nh_flush_exceptions(nh: f6i->fib6_nh, from: f6i);
1829 }
1830}
1831
1832/* Find cached rt in the hash table inside passed in rt
1833 * Caller has to hold rcu_read_lock()
1834 */
1835static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
1836 const struct in6_addr *daddr,
1837 const struct in6_addr *saddr)
1838{
1839 const struct in6_addr *src_key = NULL;
1840 struct rt6_exception_bucket *bucket;
1841 struct rt6_exception *rt6_ex;
1842 struct rt6_info *ret = NULL;
1843
1844#ifdef CONFIG_IPV6_SUBTREES
1845 /* fib6i_src.plen != 0 indicates f6i is in subtree
1846 * and exception table is indexed by a hash of
1847 * both fib6_dst and fib6_src.
1848 * However, the src addr used to create the hash
1849 * might not be exactly the passed in saddr which
1850 * is a /128 addr from the flow.
1851 * So we need to use f6i->fib6_src to redo lookup
1852 * if the passed in saddr does not find anything.
1853 * (See the logic in ip6_rt_cache_alloc() on how
1854 * rt->rt6i_src is updated.)
1855 */
1856 if (res->f6i->fib6_src.plen)
1857 src_key = saddr;
1858find_ex:
1859#endif
1860 bucket = fib6_nh_get_excptn_bucket(nh: res->nh, NULL);
1861 rt6_ex = __rt6_find_exception_rcu(bucket: &bucket, daddr, saddr: src_key);
1862
1863 if (rt6_ex && !rt6_check_expired(rt: rt6_ex->rt6i))
1864 ret = rt6_ex->rt6i;
1865
1866#ifdef CONFIG_IPV6_SUBTREES
1867 /* Use fib6_src as src_key and redo lookup */
1868 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) {
1869 src_key = &res->f6i->fib6_src.addr;
1870 goto find_ex;
1871 }
1872#endif
1873
1874 return ret;
1875}
1876
1877/* Remove the passed in cached rt from the hash table that contains it */
1878static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,
1879 const struct rt6_info *rt)
1880{
1881 const struct in6_addr *src_key = NULL;
1882 struct rt6_exception_bucket *bucket;
1883 struct rt6_exception *rt6_ex;
1884 int err;
1885
1886 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
1887 return -ENOENT;
1888
1889 spin_lock_bh(lock: &rt6_exception_lock);
1890 bucket = fib6_nh_get_excptn_bucket(nh, lock: &rt6_exception_lock);
1891
1892#ifdef CONFIG_IPV6_SUBTREES
1893 /* rt6i_src.plen != 0 indicates 'from' is in subtree
1894 * and exception table is indexed by a hash of
1895 * both rt6i_dst and rt6i_src.
1896 * Otherwise, the exception table is indexed by
1897 * a hash of only rt6i_dst.
1898 */
1899 if (plen)
1900 src_key = &rt->rt6i_src.addr;
1901#endif
1902 rt6_ex = __rt6_find_exception_spinlock(bucket: &bucket,
1903 daddr: &rt->rt6i_dst.addr,
1904 saddr: src_key);
1905 if (rt6_ex) {
1906 rt6_remove_exception(bucket, rt6_ex);
1907 err = 0;
1908 } else {
1909 err = -ENOENT;
1910 }
1911
1912 spin_unlock_bh(lock: &rt6_exception_lock);
1913 return err;
1914}
1915
1916struct fib6_nh_excptn_arg {
1917 struct rt6_info *rt;
1918 int plen;
1919};
1920
1921static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)
1922{
1923 struct fib6_nh_excptn_arg *arg = _arg;
1924 int err;
1925
1926 err = fib6_nh_remove_exception(nh, plen: arg->plen, rt: arg->rt);
1927 if (err == 0)
1928 return 1;
1929
1930 return 0;
1931}
1932
1933static int rt6_remove_exception_rt(struct rt6_info *rt)
1934{
1935 struct fib6_info *from;
1936
1937 from = rcu_dereference(rt->from);
1938 if (!from || !(rt->rt6i_flags & RTF_CACHE))
1939 return -EINVAL;
1940
1941 if (from->nh) {
1942 struct fib6_nh_excptn_arg arg = {
1943 .rt = rt,
1944 .plen = from->fib6_src.plen
1945 };
1946 int rc;
1947
1948 /* rc = 1 means an entry was found */
1949 rc = nexthop_for_each_fib6_nh(nh: from->nh,
1950 cb: rt6_nh_remove_exception_rt,
1951 arg: &arg);
1952 return rc ? 0 : -ENOENT;
1953 }
1954
1955 return fib6_nh_remove_exception(nh: from->fib6_nh,
1956 plen: from->fib6_src.plen, rt);
1957}
1958
1959/* Find rt6_ex which contains the passed in rt cache and
1960 * refresh its stamp
1961 */
1962static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,
1963 const struct rt6_info *rt)
1964{
1965 const struct in6_addr *src_key = NULL;
1966 struct rt6_exception_bucket *bucket;
1967 struct rt6_exception *rt6_ex;
1968
1969 bucket = fib6_nh_get_excptn_bucket(nh, NULL);
1970#ifdef CONFIG_IPV6_SUBTREES
1971 /* rt6i_src.plen != 0 indicates 'from' is in subtree
1972 * and exception table is indexed by a hash of
1973 * both rt6i_dst and rt6i_src.
1974 * Otherwise, the exception table is indexed by
1975 * a hash of only rt6i_dst.
1976 */
1977 if (plen)
1978 src_key = &rt->rt6i_src.addr;
1979#endif
1980 rt6_ex = __rt6_find_exception_rcu(bucket: &bucket, daddr: &rt->rt6i_dst.addr, saddr: src_key);
1981 if (rt6_ex)
1982 rt6_ex->stamp = jiffies;
1983}
1984
1985struct fib6_nh_match_arg {
1986 const struct net_device *dev;
1987 const struct in6_addr *gw;
1988 struct fib6_nh *match;
1989};
1990
1991/* determine if fib6_nh has given device and gateway */
1992static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)
1993{
1994 struct fib6_nh_match_arg *arg = _arg;
1995
1996 if (arg->dev != nh->fib_nh_dev ||
1997 (arg->gw && !nh->fib_nh_gw_family) ||
1998 (!arg->gw && nh->fib_nh_gw_family) ||
1999 (arg->gw && !ipv6_addr_equal(a1: arg->gw, a2: &nh->fib_nh_gw6)))
2000 return 0;
2001
2002 arg->match = nh;
2003
2004 /* found a match, break the loop */
2005 return 1;
2006}
2007
2008static void rt6_update_exception_stamp_rt(struct rt6_info *rt)
2009{
2010 struct fib6_info *from;
2011 struct fib6_nh *fib6_nh;
2012
2013 rcu_read_lock();
2014
2015 from = rcu_dereference(rt->from);
2016 if (!from || !(rt->rt6i_flags & RTF_CACHE))
2017 goto unlock;
2018
2019 if (from->nh) {
2020 struct fib6_nh_match_arg arg = {
2021 .dev = rt->dst.dev,
2022 .gw = &rt->rt6i_gateway,
2023 };
2024
2025 nexthop_for_each_fib6_nh(nh: from->nh, cb: fib6_nh_find_match, arg: &arg);
2026
2027 if (!arg.match)
2028 goto unlock;
2029 fib6_nh = arg.match;
2030 } else {
2031 fib6_nh = from->fib6_nh;
2032 }
2033 fib6_nh_update_exception(nh: fib6_nh, plen: from->fib6_src.plen, rt);
2034unlock:
2035 rcu_read_unlock();
2036}
2037
2038static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev,
2039 struct rt6_info *rt, int mtu)
2040{
2041 /* If the new MTU is lower than the route PMTU, this new MTU will be the
2042 * lowest MTU in the path: always allow updating the route PMTU to
2043 * reflect PMTU decreases.
2044 *
2045 * If the new MTU is higher, and the route PMTU is equal to the local
2046 * MTU, this means the old MTU is the lowest in the path, so allow
2047 * updating it: if other nodes now have lower MTUs, PMTU discovery will
2048 * handle this.
2049 */
2050
2051 if (dst_mtu(dst: &rt->dst) >= mtu)
2052 return true;
2053
2054 if (dst_mtu(dst: &rt->dst) == idev->cnf.mtu6)
2055 return true;
2056
2057 return false;
2058}
2059
2060static void rt6_exceptions_update_pmtu(struct inet6_dev *idev,
2061 const struct fib6_nh *nh, int mtu)
2062{
2063 struct rt6_exception_bucket *bucket;
2064 struct rt6_exception *rt6_ex;
2065 int i;
2066
2067 bucket = fib6_nh_get_excptn_bucket(nh, lock: &rt6_exception_lock);
2068 if (!bucket)
2069 return;
2070
2071 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2072 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
2073 struct rt6_info *entry = rt6_ex->rt6i;
2074
2075 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected
2076 * route), the metrics of its rt->from have already
2077 * been updated.
2078 */
2079 if (dst_metric_raw(dst: &entry->dst, RTAX_MTU) &&
2080 rt6_mtu_change_route_allowed(idev, rt: entry, mtu))
2081 dst_metric_set(dst: &entry->dst, RTAX_MTU, val: mtu);
2082 }
2083 bucket++;
2084 }
2085}
2086
2087#define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE)
2088
2089static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,
2090 const struct in6_addr *gateway)
2091{
2092 struct rt6_exception_bucket *bucket;
2093 struct rt6_exception *rt6_ex;
2094 struct hlist_node *tmp;
2095 int i;
2096
2097 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
2098 return;
2099
2100 spin_lock_bh(lock: &rt6_exception_lock);
2101 bucket = fib6_nh_get_excptn_bucket(nh, lock: &rt6_exception_lock);
2102 if (bucket) {
2103 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2104 hlist_for_each_entry_safe(rt6_ex, tmp,
2105 &bucket->chain, hlist) {
2106 struct rt6_info *entry = rt6_ex->rt6i;
2107
2108 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) ==
2109 RTF_CACHE_GATEWAY &&
2110 ipv6_addr_equal(a1: gateway,
2111 a2: &entry->rt6i_gateway)) {
2112 rt6_remove_exception(bucket, rt6_ex);
2113 }
2114 }
2115 bucket++;
2116 }
2117 }
2118
2119 spin_unlock_bh(lock: &rt6_exception_lock);
2120}
2121
2122static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket,
2123 struct rt6_exception *rt6_ex,
2124 struct fib6_gc_args *gc_args,
2125 unsigned long now)
2126{
2127 struct rt6_info *rt = rt6_ex->rt6i;
2128
2129 /* we are pruning and obsoleting aged-out and non gateway exceptions
2130 * even if others have still references to them, so that on next
2131 * dst_check() such references can be dropped.
2132 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when
2133 * expired, independently from their aging, as per RFC 8201 section 4
2134 */
2135 if (!(rt->rt6i_flags & RTF_EXPIRES)) {
2136 if (time_after_eq(now, READ_ONCE(rt->dst.lastuse) +
2137 gc_args->timeout)) {
2138 pr_debug("aging clone %p\n", rt);
2139 rt6_remove_exception(bucket, rt6_ex);
2140 return;
2141 }
2142 } else if (time_after(jiffies, READ_ONCE(rt->dst.expires))) {
2143 pr_debug("purging expired route %p\n", rt);
2144 rt6_remove_exception(bucket, rt6_ex);
2145 return;
2146 }
2147
2148 if (rt->rt6i_flags & RTF_GATEWAY) {
2149 struct neighbour *neigh;
2150
2151 neigh = __ipv6_neigh_lookup_noref(dev: rt->dst.dev, pkey: &rt->rt6i_gateway);
2152
2153 if (!(neigh && (neigh->flags & NTF_ROUTER))) {
2154 pr_debug("purging route %p via non-router but gateway\n",
2155 rt);
2156 rt6_remove_exception(bucket, rt6_ex);
2157 return;
2158 }
2159 }
2160
2161 gc_args->more++;
2162}
2163
2164static void fib6_nh_age_exceptions(const struct fib6_nh *nh,
2165 struct fib6_gc_args *gc_args,
2166 unsigned long now)
2167{
2168 struct rt6_exception_bucket *bucket;
2169 struct rt6_exception *rt6_ex;
2170 struct hlist_node *tmp;
2171 int i;
2172
2173 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
2174 return;
2175
2176 rcu_read_lock_bh();
2177 spin_lock(lock: &rt6_exception_lock);
2178 bucket = fib6_nh_get_excptn_bucket(nh, lock: &rt6_exception_lock);
2179 if (bucket) {
2180 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2181 hlist_for_each_entry_safe(rt6_ex, tmp,
2182 &bucket->chain, hlist) {
2183 rt6_age_examine_exception(bucket, rt6_ex,
2184 gc_args, now);
2185 }
2186 bucket++;
2187 }
2188 }
2189 spin_unlock(lock: &rt6_exception_lock);
2190 rcu_read_unlock_bh();
2191}
2192
2193struct fib6_nh_age_excptn_arg {
2194 struct fib6_gc_args *gc_args;
2195 unsigned long now;
2196};
2197
2198static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)
2199{
2200 struct fib6_nh_age_excptn_arg *arg = _arg;
2201
2202 fib6_nh_age_exceptions(nh, gc_args: arg->gc_args, now: arg->now);
2203 return 0;
2204}
2205
2206void rt6_age_exceptions(struct fib6_info *f6i,
2207 struct fib6_gc_args *gc_args,
2208 unsigned long now)
2209{
2210 if (f6i->nh) {
2211 struct fib6_nh_age_excptn_arg arg = {
2212 .gc_args = gc_args,
2213 .now = now
2214 };
2215
2216 nexthop_for_each_fib6_nh(nh: f6i->nh, cb: rt6_nh_age_exceptions,
2217 arg: &arg);
2218 } else {
2219 fib6_nh_age_exceptions(nh: f6i->fib6_nh, gc_args, now);
2220 }
2221}
2222
2223/* must be called with rcu lock held */
2224int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif,
2225 struct flowi6 *fl6, struct fib6_result *res, int strict)
2226{
2227 struct fib6_node *fn, *saved_fn;
2228
2229 fn = fib6_node_lookup(root: &table->tb6_root, daddr: &fl6->daddr, saddr: &fl6->saddr);
2230 saved_fn = fn;
2231
2232redo_rt6_select:
2233 rt6_select(net, fn, oif, res, strict);
2234 if (res->f6i == net->ipv6.fib6_null_entry) {
2235 fn = fib6_backtrack(fn, saddr: &fl6->saddr);
2236 if (fn)
2237 goto redo_rt6_select;
2238 else if (strict & RT6_LOOKUP_F_REACHABLE) {
2239 /* also consider unreachable route */
2240 strict &= ~RT6_LOOKUP_F_REACHABLE;
2241 fn = saved_fn;
2242 goto redo_rt6_select;
2243 }
2244 }
2245
2246 trace_fib6_table_lookup(net, res, table, flp: fl6);
2247
2248 return 0;
2249}
2250
2251struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
2252 int oif, struct flowi6 *fl6,
2253 const struct sk_buff *skb, int flags)
2254{
2255 struct fib6_result res = {};
2256 struct rt6_info *rt = NULL;
2257 int strict = 0;
2258
2259 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) &&
2260 !rcu_read_lock_held());
2261
2262 strict |= flags & RT6_LOOKUP_F_IFACE;
2263 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE;
2264 if (READ_ONCE(net->ipv6.devconf_all->forwarding) == 0)
2265 strict |= RT6_LOOKUP_F_REACHABLE;
2266
2267 rcu_read_lock();
2268
2269 fib6_table_lookup(net, table, oif, fl6, res: &res, strict);
2270 if (res.f6i == net->ipv6.fib6_null_entry)
2271 goto out;
2272
2273 fib6_select_path(net, res: &res, fl6, oif, have_oif_match: false, skb, strict);
2274
2275 /*Search through exception table */
2276 rt = rt6_find_cached_rt(res: &res, daddr: &fl6->daddr, saddr: &fl6->saddr);
2277 if (rt) {
2278 goto out;
2279 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) &&
2280 !res.nh->fib_nh_gw_family)) {
2281 /* Create a RTF_CACHE clone which will not be
2282 * owned by the fib6 tree. It is for the special case where
2283 * the daddr in the skb during the neighbor look-up is different
2284 * from the fl6->daddr used to look-up route here.
2285 */
2286 rt = ip6_rt_cache_alloc(res: &res, daddr: &fl6->daddr, NULL);
2287
2288 if (rt) {
2289 /* 1 refcnt is taken during ip6_rt_cache_alloc().
2290 * As rt6_uncached_list_add() does not consume refcnt,
2291 * this refcnt is always returned to the caller even
2292 * if caller sets RT6_LOOKUP_F_DST_NOREF flag.
2293 */
2294 rt6_uncached_list_add(rt);
2295 rcu_read_unlock();
2296
2297 return rt;
2298 }
2299 } else {
2300 /* Get a percpu copy */
2301 local_bh_disable();
2302 rt = rt6_get_pcpu_route(res: &res);
2303
2304 if (!rt)
2305 rt = rt6_make_pcpu_route(net, res: &res);
2306
2307 local_bh_enable();
2308 }
2309out:
2310 if (!rt)
2311 rt = net->ipv6.ip6_null_entry;
2312 if (!(flags & RT6_LOOKUP_F_DST_NOREF))
2313 ip6_hold_safe(net, prt: &rt);
2314 rcu_read_unlock();
2315
2316 return rt;
2317}
2318EXPORT_SYMBOL_GPL(ip6_pol_route);
2319
2320INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net,
2321 struct fib6_table *table,
2322 struct flowi6 *fl6,
2323 const struct sk_buff *skb,
2324 int flags)
2325{
2326 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags);
2327}
2328
2329struct dst_entry *ip6_route_input_lookup(struct net *net,
2330 struct net_device *dev,
2331 struct flowi6 *fl6,
2332 const struct sk_buff *skb,
2333 int flags)
2334{
2335 if (rt6_need_strict(daddr: &fl6->daddr) && dev->type != ARPHRD_PIMREG)
2336 flags |= RT6_LOOKUP_F_IFACE;
2337
2338 return fib6_rule_lookup(net, fl6, skb, flags, lookup: ip6_pol_route_input);
2339}
2340EXPORT_SYMBOL_GPL(ip6_route_input_lookup);
2341
2342static void ip6_multipath_l3_keys(const struct sk_buff *skb,
2343 struct flow_keys *keys,
2344 struct flow_keys *flkeys)
2345{
2346 const struct ipv6hdr *outer_iph = ipv6_hdr(skb);
2347 const struct ipv6hdr *key_iph = outer_iph;
2348 struct flow_keys *_flkeys = flkeys;
2349 const struct ipv6hdr *inner_iph;
2350 const struct icmp6hdr *icmph;
2351 struct ipv6hdr _inner_iph;
2352 struct icmp6hdr _icmph;
2353
2354 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6))
2355 goto out;
2356
2357 icmph = skb_header_pointer(skb, offset: skb_transport_offset(skb),
2358 len: sizeof(_icmph), buffer: &_icmph);
2359 if (!icmph)
2360 goto out;
2361
2362 if (!icmpv6_is_err(type: icmph->icmp6_type))
2363 goto out;
2364
2365 inner_iph = skb_header_pointer(skb,
2366 offset: skb_transport_offset(skb) + sizeof(*icmph),
2367 len: sizeof(_inner_iph), buffer: &_inner_iph);
2368 if (!inner_iph)
2369 goto out;
2370
2371 key_iph = inner_iph;
2372 _flkeys = NULL;
2373out:
2374 if (_flkeys) {
2375 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src;
2376 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst;
2377 keys->tags.flow_label = _flkeys->tags.flow_label;
2378 keys->basic.ip_proto = _flkeys->basic.ip_proto;
2379 } else {
2380 keys->addrs.v6addrs.src = key_iph->saddr;
2381 keys->addrs.v6addrs.dst = key_iph->daddr;
2382 keys->tags.flow_label = ip6_flowlabel(hdr: key_iph);
2383 keys->basic.ip_proto = key_iph->nexthdr;
2384 }
2385}
2386
2387static u32 rt6_multipath_custom_hash_outer(const struct net *net,
2388 const struct sk_buff *skb,
2389 bool *p_has_inner)
2390{
2391 u32 hash_fields = ip6_multipath_hash_fields(net);
2392 struct flow_keys keys, hash_keys;
2393
2394 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
2395 return 0;
2396
2397 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2398 skb_flow_dissect_flow_keys(skb, flow: &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
2399
2400 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2401 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
2402 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
2403 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
2404 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
2405 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
2406 hash_keys.basic.ip_proto = keys.basic.ip_proto;
2407 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
2408 hash_keys.tags.flow_label = keys.tags.flow_label;
2409 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
2410 hash_keys.ports.src = keys.ports.src;
2411 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
2412 hash_keys.ports.dst = keys.ports.dst;
2413
2414 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION);
2415 return fib_multipath_hash_from_keys(net, keys: &hash_keys);
2416}
2417
2418static u32 rt6_multipath_custom_hash_inner(const struct net *net,
2419 const struct sk_buff *skb,
2420 bool has_inner)
2421{
2422 u32 hash_fields = ip6_multipath_hash_fields(net);
2423 struct flow_keys keys, hash_keys;
2424
2425 /* We assume the packet carries an encapsulation, but if none was
2426 * encountered during dissection of the outer flow, then there is no
2427 * point in calling the flow dissector again.
2428 */
2429 if (!has_inner)
2430 return 0;
2431
2432 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK))
2433 return 0;
2434
2435 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2436 skb_flow_dissect_flow_keys(skb, flow: &keys, flags: 0);
2437
2438 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION))
2439 return 0;
2440
2441 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2442 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
2443 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
2444 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
2445 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
2446 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
2447 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2448 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2449 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
2450 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
2451 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
2452 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
2453 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
2454 hash_keys.tags.flow_label = keys.tags.flow_label;
2455 }
2456
2457 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
2458 hash_keys.basic.ip_proto = keys.basic.ip_proto;
2459 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
2460 hash_keys.ports.src = keys.ports.src;
2461 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
2462 hash_keys.ports.dst = keys.ports.dst;
2463
2464 return fib_multipath_hash_from_keys(net, keys: &hash_keys);
2465}
2466
2467static u32 rt6_multipath_custom_hash_skb(const struct net *net,
2468 const struct sk_buff *skb)
2469{
2470 u32 mhash, mhash_inner;
2471 bool has_inner = true;
2472
2473 mhash = rt6_multipath_custom_hash_outer(net, skb, p_has_inner: &has_inner);
2474 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner);
2475
2476 return jhash_2words(a: mhash, b: mhash_inner, initval: 0);
2477}
2478
2479static u32 rt6_multipath_custom_hash_fl6(const struct net *net,
2480 const struct flowi6 *fl6)
2481{
2482 u32 hash_fields = ip6_multipath_hash_fields(net);
2483 struct flow_keys hash_keys;
2484
2485 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
2486 return 0;
2487
2488 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2489 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2490 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
2491 hash_keys.addrs.v6addrs.src = fl6->saddr;
2492 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
2493 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2494 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
2495 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2496 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
2497 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2498 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) {
2499 if (fl6->flowi6_flags & FLOWI_FLAG_ANY_SPORT)
2500 hash_keys.ports.src = (__force __be16)get_random_u16();
2501 else
2502 hash_keys.ports.src = fl6->fl6_sport;
2503 }
2504 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
2505 hash_keys.ports.dst = fl6->fl6_dport;
2506
2507 return fib_multipath_hash_from_keys(net, keys: &hash_keys);
2508}
2509
2510/* if skb is set it will be used and fl6 can be NULL */
2511u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6,
2512 const struct sk_buff *skb, struct flow_keys *flkeys)
2513{
2514 struct flow_keys hash_keys;
2515 u32 mhash = 0;
2516
2517 switch (ip6_multipath_hash_policy(net)) {
2518 case 0:
2519 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2520 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2521 if (skb) {
2522 ip6_multipath_l3_keys(skb, keys: &hash_keys, flkeys);
2523 } else {
2524 hash_keys.addrs.v6addrs.src = fl6->saddr;
2525 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2526 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2527 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2528 }
2529 mhash = fib_multipath_hash_from_keys(net, keys: &hash_keys);
2530 break;
2531 case 1:
2532 if (skb) {
2533 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
2534 struct flow_keys keys;
2535
2536 /* short-circuit if we already have L4 hash present */
2537 if (skb->l4_hash)
2538 return skb_get_hash_raw(skb) >> 1;
2539
2540 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2541
2542 if (!flkeys) {
2543 skb_flow_dissect_flow_keys(skb, flow: &keys, flags: flag);
2544 flkeys = &keys;
2545 }
2546 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2547 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2548 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2549 hash_keys.ports.src = flkeys->ports.src;
2550 hash_keys.ports.dst = flkeys->ports.dst;
2551 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2552 } else {
2553 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2554 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2555 hash_keys.addrs.v6addrs.src = fl6->saddr;
2556 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2557 if (fl6->flowi6_flags & FLOWI_FLAG_ANY_SPORT)
2558 hash_keys.ports.src = (__force __be16)get_random_u16();
2559 else
2560 hash_keys.ports.src = fl6->fl6_sport;
2561 hash_keys.ports.dst = fl6->fl6_dport;
2562 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2563 }
2564 mhash = fib_multipath_hash_from_keys(net, keys: &hash_keys);
2565 break;
2566 case 2:
2567 memset(s: &hash_keys, c: 0, n: sizeof(hash_keys));
2568 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2569 if (skb) {
2570 struct flow_keys keys;
2571
2572 if (!flkeys) {
2573 skb_flow_dissect_flow_keys(skb, flow: &keys, flags: 0);
2574 flkeys = &keys;
2575 }
2576
2577 /* Inner can be v4 or v6 */
2578 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2579 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
2580 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
2581 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
2582 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2583 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2584 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2585 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2586 hash_keys.tags.flow_label = flkeys->tags.flow_label;
2587 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2588 } else {
2589 /* Same as case 0 */
2590 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2591 ip6_multipath_l3_keys(skb, keys: &hash_keys, flkeys);
2592 }
2593 } else {
2594 /* Same as case 0 */
2595 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2596 hash_keys.addrs.v6addrs.src = fl6->saddr;
2597 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2598 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2599 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2600 }
2601 mhash = fib_multipath_hash_from_keys(net, keys: &hash_keys);
2602 break;
2603 case 3:
2604 if (skb)
2605 mhash = rt6_multipath_custom_hash_skb(net, skb);
2606 else
2607 mhash = rt6_multipath_custom_hash_fl6(net, fl6);
2608 break;
2609 }
2610
2611 return mhash >> 1;
2612}
2613
2614/* Called with rcu held */
2615void ip6_route_input(struct sk_buff *skb)
2616{
2617 const struct ipv6hdr *iph = ipv6_hdr(skb);
2618 struct net *net = dev_net(dev: skb->dev);
2619 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF;
2620 struct ip_tunnel_info *tun_info;
2621 struct flowi6 fl6 = {
2622 .flowi6_iif = skb->dev->ifindex,
2623 .daddr = iph->daddr,
2624 .saddr = iph->saddr,
2625 .flowlabel = ip6_flowinfo(hdr: iph),
2626 .flowi6_mark = skb->mark,
2627 .flowi6_proto = iph->nexthdr,
2628 };
2629 struct flow_keys *flkeys = NULL, _flkeys;
2630
2631 tun_info = skb_tunnel_info(skb);
2632 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
2633 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id;
2634
2635 if (fib6_rules_early_flow_dissect(net, skb, fl6: &fl6, flkeys: &_flkeys))
2636 flkeys = &_flkeys;
2637
2638 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6))
2639 fl6.mp_hash = rt6_multipath_hash(net, fl6: &fl6, skb, flkeys);
2640 skb_dst_drop(skb);
2641 skb_dst_set_noref(skb, dst: ip6_route_input_lookup(net, skb->dev,
2642 &fl6, skb, flags));
2643}
2644
2645INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net,
2646 struct fib6_table *table,
2647 struct flowi6 *fl6,
2648 const struct sk_buff *skb,
2649 int flags)
2650{
2651 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags);
2652}
2653
2654static struct dst_entry *ip6_route_output_flags_noref(struct net *net,
2655 const struct sock *sk,
2656 struct flowi6 *fl6,
2657 int flags)
2658{
2659 bool any_src;
2660
2661 if (ipv6_addr_type(addr: &fl6->daddr) &
2662 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) {
2663 struct dst_entry *dst;
2664
2665 /* This function does not take refcnt on the dst */
2666 dst = l3mdev_link_scope_lookup(net, fl6);
2667 if (dst)
2668 return dst;
2669 }
2670
2671 fl6->flowi6_iif = LOOPBACK_IFINDEX;
2672
2673 flags |= RT6_LOOKUP_F_DST_NOREF;
2674 any_src = ipv6_addr_any(a: &fl6->saddr);
2675 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(daddr: &fl6->daddr) ||
2676 (fl6->flowi6_oif && any_src))
2677 flags |= RT6_LOOKUP_F_IFACE;
2678
2679 if (!any_src)
2680 flags |= RT6_LOOKUP_F_HAS_SADDR;
2681 else if (sk)
2682 flags |= rt6_srcprefs2flags(READ_ONCE(inet6_sk(sk)->srcprefs));
2683
2684 return fib6_rule_lookup(net, fl6, NULL, flags, lookup: ip6_pol_route_output);
2685}
2686
2687struct dst_entry *ip6_route_output_flags(struct net *net,
2688 const struct sock *sk,
2689 struct flowi6 *fl6,
2690 int flags)
2691{
2692 struct dst_entry *dst;
2693 struct rt6_info *rt6;
2694
2695 rcu_read_lock();
2696 dst = ip6_route_output_flags_noref(net, sk, fl6, flags);
2697 rt6 = dst_rt6_info(dst);
2698 /* For dst cached in uncached_list, refcnt is already taken. */
2699 if (list_empty(head: &rt6->dst.rt_uncached) && !dst_hold_safe(dst)) {
2700 dst = &net->ipv6.ip6_null_entry->dst;
2701 dst_hold(dst);
2702 }
2703 rcu_read_unlock();
2704
2705 return dst;
2706}
2707EXPORT_SYMBOL_GPL(ip6_route_output_flags);
2708
2709struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2710{
2711 struct rt6_info *rt, *ort = dst_rt6_info(dst_orig);
2712 struct net_device *loopback_dev = net->loopback_dev;
2713 struct dst_entry *new = NULL;
2714
2715 rt = dst_alloc(ops: &ip6_dst_blackhole_ops, dev: loopback_dev,
2716 DST_OBSOLETE_DEAD, flags: 0);
2717 if (rt) {
2718 rt6_info_init(rt);
2719 atomic_inc(v: &net->ipv6.rt6_stats->fib_rt_alloc);
2720
2721 new = &rt->dst;
2722 new->__use = 1;
2723 new->input = dst_discard;
2724 new->output = dst_discard_out;
2725
2726 dst_copy_metrics(dest: new, src: &ort->dst);
2727
2728 rt->rt6i_idev = in6_dev_get(dev: loopback_dev);
2729 rt->rt6i_gateway = ort->rt6i_gateway;
2730 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU;
2731
2732 memcpy(to: &rt->rt6i_dst, from: &ort->rt6i_dst, len: sizeof(struct rt6key));
2733#ifdef CONFIG_IPV6_SUBTREES
2734 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key));
2735#endif
2736 }
2737
2738 dst_release(dst: dst_orig);
2739 return new ? new : ERR_PTR(error: -ENOMEM);
2740}
2741
2742/*
2743 * Destination cache support functions
2744 */
2745
2746static bool fib6_check(struct fib6_info *f6i, u32 cookie)
2747{
2748 u32 rt_cookie = 0;
2749
2750 if (!fib6_get_cookie_safe(f6i, cookie: &rt_cookie) || rt_cookie != cookie)
2751 return false;
2752
2753 if (fib6_check_expired(f6i))
2754 return false;
2755
2756 return true;
2757}
2758
2759static struct dst_entry *rt6_check(struct rt6_info *rt,
2760 struct fib6_info *from,
2761 u32 cookie)
2762{
2763 u32 rt_cookie = 0;
2764
2765 if (!from || !fib6_get_cookie_safe(f6i: from, cookie: &rt_cookie) ||
2766 rt_cookie != cookie)
2767 return NULL;
2768
2769 if (rt6_check_expired(rt))
2770 return NULL;
2771
2772 return &rt->dst;
2773}
2774
2775static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
2776 struct fib6_info *from,
2777 u32 cookie)
2778{
2779 if (!__rt6_check_expired(rt) &&
2780 READ_ONCE(rt->dst.obsolete) == DST_OBSOLETE_FORCE_CHK &&
2781 fib6_check(f6i: from, cookie))
2782 return &rt->dst;
2783 return NULL;
2784}
2785
2786INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst,
2787 u32 cookie)
2788{
2789 struct dst_entry *dst_ret;
2790 struct fib6_info *from;
2791 struct rt6_info *rt;
2792
2793 rt = dst_rt6_info(dst);
2794
2795 if (rt->sernum)
2796 return rt6_is_valid(rt6: rt) ? dst : NULL;
2797
2798 rcu_read_lock();
2799
2800 /* All IPV6 dsts are created with ->obsolete set to the value
2801 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
2802 * into this function always.
2803 */
2804
2805 from = rcu_dereference(rt->from);
2806
2807 if (from && (rt->rt6i_flags & RTF_PCPU ||
2808 unlikely(!list_empty(&rt->dst.rt_uncached))))
2809 dst_ret = rt6_dst_from_check(rt, from, cookie);
2810 else
2811 dst_ret = rt6_check(rt, from, cookie);
2812
2813 rcu_read_unlock();
2814
2815 return dst_ret;
2816}
2817EXPORT_INDIRECT_CALLABLE(ip6_dst_check);
2818
2819static void ip6_negative_advice(struct sock *sk,
2820 struct dst_entry *dst)
2821{
2822 struct rt6_info *rt = dst_rt6_info(dst);
2823
2824 if (rt->rt6i_flags & RTF_CACHE) {
2825 rcu_read_lock();
2826 if (rt6_check_expired(rt)) {
2827 /* rt/dst can not be destroyed yet,
2828 * because of rcu_read_lock()
2829 */
2830 sk_dst_reset(sk);
2831 rt6_remove_exception_rt(rt);
2832 }
2833 rcu_read_unlock();
2834 return;
2835 }
2836 sk_dst_reset(sk);
2837}
2838
2839static void ip6_link_failure(struct sk_buff *skb)
2840{
2841 struct rt6_info *rt;
2842
2843 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, info: 0);
2844
2845 rt = dst_rt6_info(skb_dst(skb));
2846 if (rt) {
2847 rcu_read_lock();
2848 if (rt->rt6i_flags & RTF_CACHE) {
2849 rt6_remove_exception_rt(rt);
2850 } else {
2851 struct fib6_info *from;
2852 struct fib6_node *fn;
2853
2854 from = rcu_dereference(rt->from);
2855 if (from) {
2856 fn = rcu_dereference(from->fib6_node);
2857 if (fn && (rt->rt6i_flags & RTF_DEFAULT))
2858 WRITE_ONCE(fn->fn_sernum, -1);
2859 }
2860 }
2861 rcu_read_unlock();
2862 }
2863}
2864
2865static void rt6_update_expires(struct rt6_info *rt0, int timeout)
2866{
2867 if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
2868 struct fib6_info *from;
2869
2870 rcu_read_lock();
2871 from = rcu_dereference(rt0->from);
2872 if (from)
2873 WRITE_ONCE(rt0->dst.expires, from->expires);
2874 rcu_read_unlock();
2875 }
2876
2877 dst_set_expires(dst: &rt0->dst, timeout);
2878 rt0->rt6i_flags |= RTF_EXPIRES;
2879}
2880
2881static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
2882{
2883 struct net *net = dev_net(dev: rt->dst.dev);
2884
2885 dst_metric_set(dst: &rt->dst, RTAX_MTU, val: mtu);
2886 rt->rt6i_flags |= RTF_MODIFIED;
2887 rt6_update_expires(rt0: rt, timeout: net->ipv6.sysctl.ip6_rt_mtu_expires);
2888}
2889
2890static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
2891{
2892 return !(rt->rt6i_flags & RTF_CACHE) &&
2893 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from));
2894}
2895
2896static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
2897 const struct ipv6hdr *iph, u32 mtu,
2898 bool confirm_neigh)
2899{
2900 const struct in6_addr *daddr, *saddr;
2901 struct rt6_info *rt6 = dst_rt6_info(dst);
2902
2903 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU)
2904 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it.
2905 * [see also comment in rt6_mtu_change_route()]
2906 */
2907
2908 if (iph) {
2909 daddr = &iph->daddr;
2910 saddr = &iph->saddr;
2911 } else if (sk) {
2912 daddr = &sk->sk_v6_daddr;
2913 saddr = &inet6_sk(sk: sk)->saddr;
2914 } else {
2915 daddr = NULL;
2916 saddr = NULL;
2917 }
2918
2919 if (confirm_neigh)
2920 dst_confirm_neigh(dst, daddr);
2921
2922 if (mtu < IPV6_MIN_MTU)
2923 return;
2924 if (mtu >= dst_mtu(dst))
2925 return;
2926
2927 if (!rt6_cache_allowed_for_pmtu(rt: rt6)) {
2928 rt6_do_update_pmtu(rt: rt6, mtu);
2929 /* update rt6_ex->stamp for cache */
2930 if (rt6->rt6i_flags & RTF_CACHE)
2931 rt6_update_exception_stamp_rt(rt: rt6);
2932 } else if (daddr) {
2933 struct fib6_result res = {};
2934 struct rt6_info *nrt6;
2935
2936 rcu_read_lock();
2937 res.f6i = rcu_dereference(rt6->from);
2938 if (!res.f6i)
2939 goto out_unlock;
2940
2941 res.fib6_flags = res.f6i->fib6_flags;
2942 res.fib6_type = res.f6i->fib6_type;
2943
2944 if (res.f6i->nh) {
2945 struct fib6_nh_match_arg arg = {
2946 .dev = dst_dev_rcu(dst),
2947 .gw = &rt6->rt6i_gateway,
2948 };
2949
2950 nexthop_for_each_fib6_nh(nh: res.f6i->nh,
2951 cb: fib6_nh_find_match, arg: &arg);
2952
2953 /* fib6_info uses a nexthop that does not have fib6_nh
2954 * using the dst->dev + gw. Should be impossible.
2955 */
2956 if (!arg.match)
2957 goto out_unlock;
2958
2959 res.nh = arg.match;
2960 } else {
2961 res.nh = res.f6i->fib6_nh;
2962 }
2963
2964 nrt6 = ip6_rt_cache_alloc(res: &res, daddr, saddr);
2965 if (nrt6) {
2966 rt6_do_update_pmtu(rt: nrt6, mtu);
2967 if (rt6_insert_exception(nrt: nrt6, res: &res))
2968 dst_release_immediate(dst: &nrt6->dst);
2969 }
2970out_unlock:
2971 rcu_read_unlock();
2972 }
2973}
2974
2975static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
2976 struct sk_buff *skb, u32 mtu,
2977 bool confirm_neigh)
2978{
2979 __ip6_rt_update_pmtu(dst, sk, iph: skb ? ipv6_hdr(skb) : NULL, mtu,
2980 confirm_neigh);
2981}
2982
2983void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu,
2984 int oif, u32 mark, kuid_t uid)
2985{
2986 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
2987 struct dst_entry *dst;
2988 struct flowi6 fl6 = {
2989 .flowi6_oif = oif,
2990 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark),
2991 .daddr = iph->daddr,
2992 .saddr = iph->saddr,
2993 .flowlabel = ip6_flowinfo(hdr: iph),
2994 .flowi6_uid = uid,
2995 };
2996
2997 dst = ip6_route_output(net, NULL, fl6: &fl6);
2998 if (!dst->error)
2999 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), confirm_neigh: true);
3000 dst_release(dst);
3001}
3002EXPORT_SYMBOL_GPL(ip6_update_pmtu);
3003
3004void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu)
3005{
3006 int oif = sk->sk_bound_dev_if;
3007 struct dst_entry *dst;
3008
3009 if (!oif && skb->dev)
3010 oif = l3mdev_master_ifindex(dev: skb->dev);
3011
3012 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, READ_ONCE(sk->sk_mark),
3013 sk_uid(sk));
3014
3015 dst = __sk_dst_get(sk);
3016 if (!dst || !READ_ONCE(dst->obsolete) ||
3017 dst->ops->check(dst, inet6_sk(sk: sk)->dst_cookie))
3018 return;
3019
3020 bh_lock_sock(sk);
3021 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(a: &sk->sk_v6_daddr))
3022 ip6_datagram_dst_update(sk, fix_sk_saddr: false);
3023 bh_unlock_sock(sk);
3024}
3025EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu);
3026
3027void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst,
3028 const struct flowi6 *fl6)
3029{
3030#ifdef CONFIG_IPV6_SUBTREES
3031 struct ipv6_pinfo *np = inet6_sk(sk);
3032#endif
3033
3034 ip6_dst_store(sk, dst,
3035 daddr_set: ipv6_addr_equal(a1: &fl6->daddr, a2: &sk->sk_v6_daddr),
3036#ifdef CONFIG_IPV6_SUBTREES
3037 ipv6_addr_equal(&fl6->saddr, &np->saddr) ?
3038 true :
3039#endif
3040 saddr_set: false);
3041}
3042
3043static bool ip6_redirect_nh_match(const struct fib6_result *res,
3044 struct flowi6 *fl6,
3045 const struct in6_addr *gw,
3046 struct rt6_info **ret)
3047{
3048 const struct fib6_nh *nh = res->nh;
3049
3050 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||
3051 fl6->flowi6_oif != nh->fib_nh_dev->ifindex)
3052 return false;
3053
3054 /* rt_cache's gateway might be different from its 'parent'
3055 * in the case of an ip redirect.
3056 * So we keep searching in the exception table if the gateway
3057 * is different.
3058 */
3059 if (!ipv6_addr_equal(a1: gw, a2: &nh->fib_nh_gw6)) {
3060 struct rt6_info *rt_cache;
3061
3062 rt_cache = rt6_find_cached_rt(res, daddr: &fl6->daddr, saddr: &fl6->saddr);
3063 if (rt_cache &&
3064 ipv6_addr_equal(a1: gw, a2: &rt_cache->rt6i_gateway)) {
3065 *ret = rt_cache;
3066 return true;
3067 }
3068 return false;
3069 }
3070 return true;
3071}
3072
3073struct fib6_nh_rd_arg {
3074 struct fib6_result *res;
3075 struct flowi6 *fl6;
3076 const struct in6_addr *gw;
3077 struct rt6_info **ret;
3078};
3079
3080static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)
3081{
3082 struct fib6_nh_rd_arg *arg = _arg;
3083
3084 arg->res->nh = nh;
3085 return ip6_redirect_nh_match(res: arg->res, fl6: arg->fl6, gw: arg->gw, ret: arg->ret);
3086}
3087
3088/* Handle redirects */
3089struct ip6rd_flowi {
3090 struct flowi6 fl6;
3091 struct in6_addr gateway;
3092};
3093
3094INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net,
3095 struct fib6_table *table,
3096 struct flowi6 *fl6,
3097 const struct sk_buff *skb,
3098 int flags)
3099{
3100 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6;
3101 struct rt6_info *ret = NULL;
3102 struct fib6_result res = {};
3103 struct fib6_nh_rd_arg arg = {
3104 .res = &res,
3105 .fl6 = fl6,
3106 .gw = &rdfl->gateway,
3107 .ret = &ret
3108 };
3109 struct fib6_info *rt;
3110 struct fib6_node *fn;
3111
3112 /* Get the "current" route for this destination and
3113 * check if the redirect has come from appropriate router.
3114 *
3115 * RFC 4861 specifies that redirects should only be
3116 * accepted if they come from the nexthop to the target.
3117 * Due to the way the routes are chosen, this notion
3118 * is a bit fuzzy and one might need to check all possible
3119 * routes.
3120 */
3121
3122 rcu_read_lock();
3123 fn = fib6_node_lookup(root: &table->tb6_root, daddr: &fl6->daddr, saddr: &fl6->saddr);
3124restart:
3125 for_each_fib6_node_rt_rcu(fn) {
3126 res.f6i = rt;
3127 if (fib6_check_expired(f6i: rt))
3128 continue;
3129 if (rt->fib6_flags & RTF_REJECT)
3130 break;
3131 if (unlikely(rt->nh)) {
3132 if (nexthop_is_blackhole(nh: rt->nh))
3133 continue;
3134 /* on match, res->nh is filled in and potentially ret */
3135 if (nexthop_for_each_fib6_nh(nh: rt->nh,
3136 cb: fib6_nh_redirect_match,
3137 arg: &arg))
3138 goto out;
3139 } else {
3140 res.nh = rt->fib6_nh;
3141 if (ip6_redirect_nh_match(res: &res, fl6, gw: &rdfl->gateway,
3142 ret: &ret))
3143 goto out;
3144 }
3145 }
3146
3147 if (!rt)
3148 rt = net->ipv6.fib6_null_entry;
3149 else if (rt->fib6_flags & RTF_REJECT) {
3150 ret = net->ipv6.ip6_null_entry;
3151 goto out;
3152 }
3153
3154 if (rt == net->ipv6.fib6_null_entry) {
3155 fn = fib6_backtrack(fn, saddr: &fl6->saddr);
3156 if (fn)
3157 goto restart;
3158 }
3159
3160 res.f6i = rt;
3161 res.nh = rt->fib6_nh;
3162out:
3163 if (ret) {
3164 ip6_hold_safe(net, prt: &ret);
3165 } else {
3166 res.fib6_flags = res.f6i->fib6_flags;
3167 res.fib6_type = res.f6i->fib6_type;
3168 ret = ip6_create_rt_rcu(res: &res);
3169 }
3170
3171 rcu_read_unlock();
3172
3173 trace_fib6_table_lookup(net, res: &res, table, flp: fl6);
3174 return ret;
3175};
3176
3177static struct dst_entry *ip6_route_redirect(struct net *net,
3178 const struct flowi6 *fl6,
3179 const struct sk_buff *skb,
3180 const struct in6_addr *gateway)
3181{
3182 int flags = RT6_LOOKUP_F_HAS_SADDR;
3183 struct ip6rd_flowi rdfl;
3184
3185 rdfl.fl6 = *fl6;
3186 rdfl.gateway = *gateway;
3187
3188 return fib6_rule_lookup(net, fl6: &rdfl.fl6, skb,
3189 flags, lookup: __ip6_route_redirect);
3190}
3191
3192void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark,
3193 kuid_t uid)
3194{
3195 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
3196 struct dst_entry *dst;
3197 struct flowi6 fl6 = {
3198 .flowi6_iif = LOOPBACK_IFINDEX,
3199 .flowi6_oif = oif,
3200 .flowi6_mark = mark,
3201 .daddr = iph->daddr,
3202 .saddr = iph->saddr,
3203 .flowlabel = ip6_flowinfo(hdr: iph),
3204 .flowi6_uid = uid,
3205 };
3206
3207 dst = ip6_route_redirect(net, fl6: &fl6, skb, gateway: &ipv6_hdr(skb)->saddr);
3208 rt6_do_redirect(dst, NULL, skb);
3209 dst_release(dst);
3210}
3211EXPORT_SYMBOL_GPL(ip6_redirect);
3212
3213void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif)
3214{
3215 const struct ipv6hdr *iph = ipv6_hdr(skb);
3216 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb);
3217 struct dst_entry *dst;
3218 struct flowi6 fl6 = {
3219 .flowi6_iif = LOOPBACK_IFINDEX,
3220 .flowi6_oif = oif,
3221 .daddr = msg->dest,
3222 .saddr = iph->daddr,
3223 .flowi6_uid = sock_net_uid(net, NULL),
3224 };
3225
3226 dst = ip6_route_redirect(net, fl6: &fl6, skb, gateway: &iph->saddr);
3227 rt6_do_redirect(dst, NULL, skb);
3228 dst_release(dst);
3229}
3230
3231void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk)
3232{
3233 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if,
3234 READ_ONCE(sk->sk_mark), sk_uid(sk));
3235}
3236EXPORT_SYMBOL_GPL(ip6_sk_redirect);
3237
3238static unsigned int ip6_default_advmss(const struct dst_entry *dst)
3239{
3240 unsigned int mtu = dst_mtu(dst);
3241 struct net *net;
3242
3243 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
3244
3245 rcu_read_lock();
3246
3247 net = dst_dev_net_rcu(dst);
3248 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss)
3249 mtu = net->ipv6.sysctl.ip6_rt_min_advmss;
3250
3251 rcu_read_unlock();
3252
3253 /*
3254 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and
3255 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size.
3256 * IPV6_MAXPLEN is also valid and means: "any MSS,
3257 * rely only on pmtu discovery"
3258 */
3259 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr))
3260 mtu = IPV6_MAXPLEN;
3261 return mtu;
3262}
3263
3264INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst)
3265{
3266 return ip6_dst_mtu_maybe_forward(dst, forwarding: false);
3267}
3268EXPORT_INDIRECT_CALLABLE(ip6_mtu);
3269
3270/* MTU selection:
3271 * 1. mtu on route is locked - use it
3272 * 2. mtu from nexthop exception
3273 * 3. mtu from egress device
3274 *
3275 * based on ip6_dst_mtu_forward and exception logic of
3276 * rt6_find_cached_rt; called with rcu_read_lock
3277 */
3278u32 ip6_mtu_from_fib6(const struct fib6_result *res,
3279 const struct in6_addr *daddr,
3280 const struct in6_addr *saddr)
3281{
3282 const struct fib6_nh *nh = res->nh;
3283 struct fib6_info *f6i = res->f6i;
3284 struct inet6_dev *idev;
3285 struct rt6_info *rt;
3286 u32 mtu = 0;
3287
3288 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) {
3289 mtu = f6i->fib6_pmtu;
3290 if (mtu)
3291 goto out;
3292 }
3293
3294 rt = rt6_find_cached_rt(res, daddr, saddr);
3295 if (unlikely(rt)) {
3296 mtu = dst_metric_raw(dst: &rt->dst, RTAX_MTU);
3297 } else {
3298 struct net_device *dev = nh->fib_nh_dev;
3299
3300 mtu = IPV6_MIN_MTU;
3301 idev = __in6_dev_get(dev);
3302 if (idev)
3303 mtu = max_t(u32, mtu, READ_ONCE(idev->cnf.mtu6));
3304 }
3305
3306 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
3307out:
3308 return mtu - lwtunnel_headroom(lwtstate: nh->fib_nh_lws, mtu);
3309}
3310
3311struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
3312 struct flowi6 *fl6)
3313{
3314 struct dst_entry *dst;
3315 struct rt6_info *rt;
3316 struct inet6_dev *idev = in6_dev_get(dev);
3317 struct net *net = dev_net(dev);
3318
3319 if (unlikely(!idev))
3320 return ERR_PTR(error: -ENODEV);
3321
3322 rt = ip6_dst_alloc(net, dev, 0);
3323 if (unlikely(!rt)) {
3324 in6_dev_put(idev);
3325 dst = ERR_PTR(error: -ENOMEM);
3326 goto out;
3327 }
3328
3329 rt->dst.input = ip6_input;
3330 rt->dst.output = ip6_output;
3331 rt->rt6i_gateway = fl6->daddr;
3332 rt->rt6i_dst.addr = fl6->daddr;
3333 rt->rt6i_dst.plen = 128;
3334 rt->rt6i_idev = idev;
3335 dst_metric_set(dst: &rt->dst, RTAX_HOPLIMIT, val: 0);
3336
3337 /* Add this dst into uncached_list so that rt6_disable_ip() can
3338 * do proper release of the net_device
3339 */
3340 rt6_uncached_list_add(rt);
3341
3342 dst = xfrm_lookup(net, dst_orig: &rt->dst, fl: flowi6_to_flowi(fl6), NULL, flags: 0);
3343
3344out:
3345 return dst;
3346}
3347
3348static void ip6_dst_gc(struct dst_ops *ops)
3349{
3350 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
3351 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
3352 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
3353 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
3354 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
3355 unsigned int val;
3356 int entries;
3357
3358 if (time_after(rt_last_gc + rt_min_interval, jiffies))
3359 goto out;
3360
3361 fib6_run_gc(expires: atomic_inc_return(v: &net->ipv6.ip6_rt_gc_expire), net, force: true);
3362 entries = dst_entries_get_slow(dst: ops);
3363 if (entries < ops->gc_thresh)
3364 atomic_set(v: &net->ipv6.ip6_rt_gc_expire, i: rt_gc_timeout >> 1);
3365out:
3366 val = atomic_read(v: &net->ipv6.ip6_rt_gc_expire);
3367 atomic_set(v: &net->ipv6.ip6_rt_gc_expire, i: val - (val >> rt_elasticity));
3368}
3369
3370static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
3371 const struct in6_addr *gw_addr, u32 tbid,
3372 int flags, struct fib6_result *res)
3373{
3374 struct flowi6 fl6 = {
3375 .flowi6_oif = cfg->fc_ifindex,
3376 .daddr = *gw_addr,
3377 .saddr = cfg->fc_prefsrc,
3378 };
3379 struct fib6_table *table;
3380 int err;
3381
3382 table = fib6_get_table(net, id: tbid);
3383 if (!table)
3384 return -EINVAL;
3385
3386 if (!ipv6_addr_any(a: &cfg->fc_prefsrc))
3387 flags |= RT6_LOOKUP_F_HAS_SADDR;
3388
3389 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE;
3390
3391 err = fib6_table_lookup(net, table, oif: cfg->fc_ifindex, fl6: &fl6, res, strict: flags);
3392 if (!err && res->f6i != net->ipv6.fib6_null_entry)
3393 fib6_select_path(net, res, fl6: &fl6, oif: cfg->fc_ifindex,
3394 have_oif_match: cfg->fc_ifindex != 0, NULL, strict: flags);
3395
3396 return err;
3397}
3398
3399static int ip6_route_check_nh_onlink(struct net *net,
3400 struct fib6_config *cfg,
3401 const struct net_device *dev,
3402 struct netlink_ext_ack *extack)
3403{
3404 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
3405 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3406 struct fib6_result res = {};
3407 int err;
3408
3409 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, flags: 0, res: &res);
3410 if (!err && !(res.fib6_flags & RTF_REJECT) &&
3411 /* ignore match if it is the default route */
3412 !ipv6_addr_any(a: &res.f6i->fib6_dst.addr) &&
3413 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {
3414 NL_SET_ERR_MSG(extack,
3415 "Nexthop has invalid gateway or device mismatch");
3416 err = -EINVAL;
3417 }
3418
3419 return err;
3420}
3421
3422static int ip6_route_check_nh(struct net *net,
3423 struct fib6_config *cfg,
3424 struct net_device **_dev,
3425 netdevice_tracker *dev_tracker,
3426 struct inet6_dev **idev)
3427{
3428 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3429 struct net_device *dev = _dev ? *_dev : NULL;
3430 int flags = RT6_LOOKUP_F_IFACE;
3431 struct fib6_result res = {};
3432 int err = -EHOSTUNREACH;
3433
3434 if (cfg->fc_table) {
3435 err = ip6_nh_lookup_table(net, cfg, gw_addr,
3436 tbid: cfg->fc_table, flags, res: &res);
3437 /* gw_addr can not require a gateway or resolve to a reject
3438 * route. If a device is given, it must match the result.
3439 */
3440 if (err || res.fib6_flags & RTF_REJECT ||
3441 res.nh->fib_nh_gw_family ||
3442 (dev && dev != res.nh->fib_nh_dev))
3443 err = -EHOSTUNREACH;
3444 }
3445
3446 if (err < 0) {
3447 struct flowi6 fl6 = {
3448 .flowi6_oif = cfg->fc_ifindex,
3449 .daddr = *gw_addr,
3450 };
3451
3452 err = fib6_lookup(net, oif: cfg->fc_ifindex, fl6: &fl6, res: &res, flags);
3453 if (err || res.fib6_flags & RTF_REJECT ||
3454 res.nh->fib_nh_gw_family)
3455 err = -EHOSTUNREACH;
3456
3457 if (err)
3458 return err;
3459
3460 fib6_select_path(net, res: &res, fl6: &fl6, oif: cfg->fc_ifindex,
3461 have_oif_match: cfg->fc_ifindex != 0, NULL, strict: flags);
3462 }
3463
3464 err = 0;
3465 if (dev) {
3466 if (dev != res.nh->fib_nh_dev)
3467 err = -EHOSTUNREACH;
3468 } else {
3469 *_dev = dev = res.nh->fib_nh_dev;
3470 netdev_hold(dev, tracker: dev_tracker, GFP_ATOMIC);
3471 *idev = in6_dev_get(dev);
3472 }
3473
3474 return err;
3475}
3476
3477static int ip6_validate_gw(struct net *net, struct fib6_config *cfg,
3478 struct net_device **_dev,
3479 netdevice_tracker *dev_tracker,
3480 struct inet6_dev **idev,
3481 struct netlink_ext_ack *extack)
3482{
3483 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3484 int gwa_type = ipv6_addr_type(addr: gw_addr);
3485 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true;
3486 const struct net_device *dev = *_dev;
3487 bool need_addr_check = !dev;
3488 int err = -EINVAL;
3489
3490 /* if gw_addr is local we will fail to detect this in case
3491 * address is still TENTATIVE (DAD in progress). rt6_lookup()
3492 * will return already-added prefix route via interface that
3493 * prefix route was assigned to, which might be non-loopback.
3494 */
3495 if (dev &&
3496 ipv6_chk_addr_and_flags(net, addr: gw_addr, dev, skip_dev_check: skip_dev, strict: 0, banned_flags: 0)) {
3497 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
3498 goto out;
3499 }
3500
3501 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) {
3502 /* IPv6 strictly inhibits using not link-local
3503 * addresses as nexthop address.
3504 * Otherwise, router will not able to send redirects.
3505 * It is very good, but in some (rare!) circumstances
3506 * (SIT, PtP, NBMA NOARP links) it is handy to allow
3507 * some exceptions. --ANK
3508 * We allow IPv4-mapped nexthops to support RFC4798-type
3509 * addressing
3510 */
3511 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) {
3512 NL_SET_ERR_MSG(extack, "Invalid gateway address");
3513 goto out;
3514 }
3515
3516 rcu_read_lock();
3517
3518 if (cfg->fc_flags & RTNH_F_ONLINK)
3519 err = ip6_route_check_nh_onlink(net, cfg, dev, extack);
3520 else
3521 err = ip6_route_check_nh(net, cfg, _dev, dev_tracker,
3522 idev);
3523
3524 rcu_read_unlock();
3525
3526 if (err)
3527 goto out;
3528 }
3529
3530 /* reload in case device was changed */
3531 dev = *_dev;
3532
3533 err = -EINVAL;
3534 if (!dev) {
3535 NL_SET_ERR_MSG(extack, "Egress device not specified");
3536 goto out;
3537 } else if (dev->flags & IFF_LOOPBACK) {
3538 NL_SET_ERR_MSG(extack,
3539 "Egress device can not be loopback device for this route");
3540 goto out;
3541 }
3542
3543 /* if we did not check gw_addr above, do so now that the
3544 * egress device has been resolved.
3545 */
3546 if (need_addr_check &&
3547 ipv6_chk_addr_and_flags(net, addr: gw_addr, dev, skip_dev_check: skip_dev, strict: 0, banned_flags: 0)) {
3548 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
3549 goto out;
3550 }
3551
3552 err = 0;
3553out:
3554 return err;
3555}
3556
3557static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type)
3558{
3559 if ((flags & RTF_REJECT) ||
3560 (dev && (dev->flags & IFF_LOOPBACK) &&
3561 !(addr_type & IPV6_ADDR_LOOPBACK) &&
3562 !(flags & (RTF_ANYCAST | RTF_LOCAL))))
3563 return true;
3564
3565 return false;
3566}
3567
3568int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
3569 struct fib6_config *cfg, gfp_t gfp_flags,
3570 struct netlink_ext_ack *extack)
3571{
3572 netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker;
3573 struct net_device *dev = NULL;
3574 struct inet6_dev *idev = NULL;
3575 int addr_type;
3576 int err;
3577
3578 fib6_nh->fib_nh_family = AF_INET6;
3579#ifdef CONFIG_IPV6_ROUTER_PREF
3580 fib6_nh->last_probe = jiffies;
3581#endif
3582 if (cfg->fc_is_fdb) {
3583 fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
3584 fib6_nh->fib_nh_gw_family = AF_INET6;
3585 return 0;
3586 }
3587
3588 err = -ENODEV;
3589 if (cfg->fc_ifindex) {
3590 dev = netdev_get_by_index(net, ifindex: cfg->fc_ifindex,
3591 tracker: dev_tracker, gfp: gfp_flags);
3592 if (!dev)
3593 goto out;
3594 idev = in6_dev_get(dev);
3595 if (!idev)
3596 goto out;
3597 }
3598
3599 if (cfg->fc_flags & RTNH_F_ONLINK) {
3600 if (!dev) {
3601 NL_SET_ERR_MSG(extack,
3602 "Nexthop device required for onlink");
3603 goto out;
3604 }
3605
3606 if (!(dev->flags & IFF_UP)) {
3607 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3608 err = -ENETDOWN;
3609 goto out;
3610 }
3611
3612 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;
3613 }
3614
3615 fib6_nh->fib_nh_weight = 1;
3616
3617 /* We cannot add true routes via loopback here,
3618 * they would result in kernel looping; promote them to reject routes
3619 */
3620 addr_type = ipv6_addr_type(addr: &cfg->fc_dst);
3621 if (fib6_is_reject(flags: cfg->fc_flags, dev, addr_type)) {
3622 /* hold loopback dev/idev if we haven't done so. */
3623 if (dev != net->loopback_dev) {
3624 if (dev) {
3625 netdev_put(dev, tracker: dev_tracker);
3626 in6_dev_put(idev);
3627 }
3628 dev = net->loopback_dev;
3629 netdev_hold(dev, tracker: dev_tracker, gfp: gfp_flags);
3630 idev = in6_dev_get(dev);
3631 if (!idev) {
3632 err = -ENODEV;
3633 goto out;
3634 }
3635 }
3636 goto pcpu_alloc;
3637 }
3638
3639 if (cfg->fc_flags & RTF_GATEWAY) {
3640 err = ip6_validate_gw(net, cfg, dev: &dev, dev_tracker,
3641 idev: &idev, extack);
3642 if (err)
3643 goto out;
3644
3645 fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
3646 fib6_nh->fib_nh_gw_family = AF_INET6;
3647 }
3648
3649 err = -ENODEV;
3650 if (!dev)
3651 goto out;
3652
3653 if (!idev || idev->cnf.disable_ipv6) {
3654 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device");
3655 err = -EACCES;
3656 goto out;
3657 }
3658
3659 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) {
3660 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3661 err = -ENETDOWN;
3662 goto out;
3663 }
3664
3665 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) &&
3666 !netif_carrier_ok(dev))
3667 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
3668
3669 err = fib_nh_common_init(net, nhc: &fib6_nh->nh_common, fc_encap: cfg->fc_encap,
3670 fc_encap_type: cfg->fc_encap_type, cfg, gfp_flags, extack);
3671 if (err)
3672 goto out;
3673
3674pcpu_alloc:
3675 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
3676 if (!fib6_nh->rt6i_pcpu) {
3677 err = -ENOMEM;
3678 goto out;
3679 }
3680
3681 fib6_nh->fib_nh_dev = dev;
3682 fib6_nh->fib_nh_oif = dev->ifindex;
3683 err = 0;
3684out:
3685 if (idev)
3686 in6_dev_put(idev);
3687
3688 if (err) {
3689 fib_nh_common_release(nhc: &fib6_nh->nh_common);
3690 fib6_nh->nh_common.nhc_pcpu_rth_output = NULL;
3691 fib6_nh->fib_nh_lws = NULL;
3692 netdev_put(dev, tracker: dev_tracker);
3693 }
3694
3695 return err;
3696}
3697
3698void fib6_nh_release(struct fib6_nh *fib6_nh)
3699{
3700 struct rt6_exception_bucket *bucket;
3701
3702 rcu_read_lock();
3703
3704 fib6_nh_flush_exceptions(nh: fib6_nh, NULL);
3705 bucket = fib6_nh_get_excptn_bucket(nh: fib6_nh, NULL);
3706 if (bucket) {
3707 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);
3708 kfree(objp: bucket);
3709 }
3710
3711 rcu_read_unlock();
3712
3713 fib6_nh_release_dsts(fib6_nh);
3714 free_percpu(pdata: fib6_nh->rt6i_pcpu);
3715
3716 fib_nh_common_release(nhc: &fib6_nh->nh_common);
3717}
3718
3719void fib6_nh_release_dsts(struct fib6_nh *fib6_nh)
3720{
3721 int cpu;
3722
3723 if (!fib6_nh->rt6i_pcpu)
3724 return;
3725
3726 for_each_possible_cpu(cpu) {
3727 struct rt6_info *pcpu_rt, **ppcpu_rt;
3728
3729 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
3730 pcpu_rt = xchg(ppcpu_rt, NULL);
3731 if (pcpu_rt) {
3732 dst_dev_put(dst: &pcpu_rt->dst);
3733 dst_release(dst: &pcpu_rt->dst);
3734 }
3735 }
3736}
3737
3738static int fib6_config_validate(struct fib6_config *cfg,
3739 struct netlink_ext_ack *extack)
3740{
3741 /* RTF_PCPU is an internal flag; can not be set by userspace */
3742 if (cfg->fc_flags & RTF_PCPU) {
3743 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU");
3744 goto errout;
3745 }
3746
3747 /* RTF_CACHE is an internal flag; can not be set by userspace */
3748 if (cfg->fc_flags & RTF_CACHE) {
3749 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE");
3750 goto errout;
3751 }
3752
3753 if (cfg->fc_type > RTN_MAX) {
3754 NL_SET_ERR_MSG(extack, "Invalid route type");
3755 goto errout;
3756 }
3757
3758 if (cfg->fc_dst_len > 128) {
3759 NL_SET_ERR_MSG(extack, "Invalid prefix length");
3760 goto errout;
3761 }
3762
3763#ifdef CONFIG_IPV6_SUBTREES
3764 if (cfg->fc_src_len > 128) {
3765 NL_SET_ERR_MSG(extack, "Invalid source address length");
3766 goto errout;
3767 }
3768
3769 if (cfg->fc_nh_id && cfg->fc_src_len) {
3770 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing");
3771 goto errout;
3772 }
3773#else
3774 if (cfg->fc_src_len) {
3775 NL_SET_ERR_MSG(extack,
3776 "Specifying source address requires IPV6_SUBTREES to be enabled");
3777 goto errout;
3778 }
3779#endif
3780 return 0;
3781errout:
3782 return -EINVAL;
3783}
3784
3785static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
3786 gfp_t gfp_flags,
3787 struct netlink_ext_ack *extack)
3788{
3789 struct net *net = cfg->fc_nlinfo.nl_net;
3790 struct fib6_table *table;
3791 struct fib6_info *rt;
3792 int err;
3793
3794 if (cfg->fc_nlinfo.nlh &&
3795 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) {
3796 table = fib6_get_table(net, id: cfg->fc_table);
3797 if (!table) {
3798 pr_warn("NLM_F_CREATE should be specified when creating new route\n");
3799 table = fib6_new_table(net, id: cfg->fc_table);
3800 }
3801 } else {
3802 table = fib6_new_table(net, id: cfg->fc_table);
3803 }
3804 if (!table) {
3805 err = -ENOBUFS;
3806 goto err;
3807 }
3808
3809 rt = fib6_info_alloc(gfp_flags, with_fib6_nh: !cfg->fc_nh_id);
3810 if (!rt) {
3811 err = -ENOMEM;
3812 goto err;
3813 }
3814
3815 rt->fib6_metrics = ip_fib_metrics_init(fc_mx: cfg->fc_mx, fc_mx_len: cfg->fc_mx_len,
3816 extack);
3817 if (IS_ERR(ptr: rt->fib6_metrics)) {
3818 err = PTR_ERR(ptr: rt->fib6_metrics);
3819 goto free;
3820 }
3821
3822 if (cfg->fc_flags & RTF_ADDRCONF)
3823 rt->dst_nocount = true;
3824
3825 if (cfg->fc_flags & RTF_EXPIRES)
3826 fib6_set_expires(f6i: rt, expires: jiffies +
3827 clock_t_to_jiffies(x: cfg->fc_expires));
3828
3829 if (cfg->fc_protocol == RTPROT_UNSPEC)
3830 cfg->fc_protocol = RTPROT_BOOT;
3831
3832 rt->fib6_protocol = cfg->fc_protocol;
3833 rt->fib6_table = table;
3834 rt->fib6_metric = cfg->fc_metric;
3835 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST;
3836 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY;
3837
3838 ipv6_addr_prefix(pfx: &rt->fib6_dst.addr, addr: &cfg->fc_dst, plen: cfg->fc_dst_len);
3839 rt->fib6_dst.plen = cfg->fc_dst_len;
3840
3841#ifdef CONFIG_IPV6_SUBTREES
3842 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len);
3843 rt->fib6_src.plen = cfg->fc_src_len;
3844#endif
3845 return rt;
3846free:
3847 kfree(objp: rt);
3848err:
3849 return ERR_PTR(error: err);
3850}
3851
3852static int ip6_route_info_create_nh(struct fib6_info *rt,
3853 struct fib6_config *cfg,
3854 gfp_t gfp_flags,
3855 struct netlink_ext_ack *extack)
3856{
3857 struct net *net = cfg->fc_nlinfo.nl_net;
3858 struct fib6_nh *fib6_nh;
3859 int err;
3860
3861 if (cfg->fc_nh_id) {
3862 struct nexthop *nh;
3863
3864 rcu_read_lock();
3865
3866 nh = nexthop_find_by_id(net, id: cfg->fc_nh_id);
3867 if (!nh) {
3868 err = -EINVAL;
3869 NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
3870 goto out_free;
3871 }
3872
3873 err = fib6_check_nexthop(nh, cfg, extack);
3874 if (err)
3875 goto out_free;
3876
3877 if (!nexthop_get(nh)) {
3878 NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
3879 err = -ENOENT;
3880 goto out_free;
3881 }
3882
3883 rt->nh = nh;
3884 fib6_nh = nexthop_fib6_nh(nh: rt->nh);
3885
3886 rcu_read_unlock();
3887 } else {
3888 int addr_type;
3889
3890 err = fib6_nh_init(net, fib6_nh: rt->fib6_nh, cfg, gfp_flags, extack);
3891 if (err)
3892 goto out_release;
3893
3894 fib6_nh = rt->fib6_nh;
3895
3896 /* We cannot add true routes via loopback here, they would
3897 * result in kernel looping; promote them to reject routes
3898 */
3899 addr_type = ipv6_addr_type(addr: &cfg->fc_dst);
3900 if (fib6_is_reject(flags: cfg->fc_flags, dev: rt->fib6_nh->fib_nh_dev,
3901 addr_type))
3902 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP;
3903 }
3904
3905 if (!ipv6_addr_any(a: &cfg->fc_prefsrc)) {
3906 struct net_device *dev = fib6_nh->fib_nh_dev;
3907
3908 if (!ipv6_chk_addr(net, addr: &cfg->fc_prefsrc, dev, strict: 0)) {
3909 NL_SET_ERR_MSG(extack, "Invalid source address");
3910 err = -EINVAL;
3911 goto out_release;
3912 }
3913 rt->fib6_prefsrc.addr = cfg->fc_prefsrc;
3914 rt->fib6_prefsrc.plen = 128;
3915 }
3916
3917 return 0;
3918out_release:
3919 fib6_info_release(f6i: rt);
3920 return err;
3921out_free:
3922 rcu_read_unlock();
3923 ip_fib_metrics_put(fib_metrics: rt->fib6_metrics);
3924 kfree(objp: rt);
3925 return err;
3926}
3927
3928int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
3929 struct netlink_ext_ack *extack)
3930{
3931 struct fib6_info *rt;
3932 int err;
3933
3934 err = fib6_config_validate(cfg, extack);
3935 if (err)
3936 return err;
3937
3938 rt = ip6_route_info_create(cfg, gfp_flags, extack);
3939 if (IS_ERR(ptr: rt))
3940 return PTR_ERR(ptr: rt);
3941
3942 err = ip6_route_info_create_nh(rt, cfg, gfp_flags, extack);
3943 if (err)
3944 return err;
3945
3946 err = __ip6_ins_rt(rt, info: &cfg->fc_nlinfo, extack);
3947 fib6_info_release(f6i: rt);
3948
3949 return err;
3950}
3951
3952static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info)
3953{
3954 struct net *net = info->nl_net;
3955 struct fib6_table *table;
3956 int err;
3957
3958 if (rt == net->ipv6.fib6_null_entry) {
3959 err = -ENOENT;
3960 goto out;
3961 }
3962
3963 table = rt->fib6_table;
3964 spin_lock_bh(lock: &table->tb6_lock);
3965 err = fib6_del(rt, info);
3966 spin_unlock_bh(lock: &table->tb6_lock);
3967
3968out:
3969 fib6_info_release(f6i: rt);
3970 return err;
3971}
3972
3973int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify)
3974{
3975 struct nl_info info = {
3976 .nl_net = net,
3977 .skip_notify = skip_notify
3978 };
3979
3980 return __ip6_del_rt(rt, info: &info);
3981}
3982
3983static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg)
3984{
3985 struct nl_info *info = &cfg->fc_nlinfo;
3986 struct net *net = info->nl_net;
3987 struct sk_buff *skb = NULL;
3988 struct fib6_table *table;
3989 int err = -ENOENT;
3990
3991 if (rt == net->ipv6.fib6_null_entry)
3992 goto out_put;
3993 table = rt->fib6_table;
3994 spin_lock_bh(lock: &table->tb6_lock);
3995
3996 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) {
3997 struct fib6_info *sibling, *next_sibling;
3998 struct fib6_node *fn;
3999
4000 /* prefer to send a single notification with all hops */
4001 skb = nlmsg_new(payload: rt6_nlmsg_size(f6i: rt), flags: gfp_any());
4002 if (skb) {
4003 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
4004
4005 if (rt6_fill_node(net, skb, rt, NULL,
4006 NULL, NULL, iif: 0, RTM_DELROUTE,
4007 portid: info->portid, seq, flags: 0) < 0) {
4008 kfree_skb(skb);
4009 skb = NULL;
4010 } else
4011 info->skip_notify = 1;
4012 }
4013
4014 /* 'rt' points to the first sibling route. If it is not the
4015 * leaf, then we do not need to send a notification. Otherwise,
4016 * we need to check if the last sibling has a next route or not
4017 * and emit a replace or delete notification, respectively.
4018 */
4019 info->skip_notify_kernel = 1;
4020 fn = rcu_dereference_protected(rt->fib6_node,
4021 lockdep_is_held(&table->tb6_lock));
4022 if (rcu_access_pointer(fn->leaf) == rt) {
4023 struct fib6_info *last_sibling, *replace_rt;
4024
4025 last_sibling = list_last_entry(&rt->fib6_siblings,
4026 struct fib6_info,
4027 fib6_siblings);
4028 replace_rt = rcu_dereference_protected(
4029 last_sibling->fib6_next,
4030 lockdep_is_held(&table->tb6_lock));
4031 if (replace_rt)
4032 call_fib6_entry_notifiers_replace(net,
4033 rt: replace_rt);
4034 else
4035 call_fib6_multipath_entry_notifiers(net,
4036 event_type: FIB_EVENT_ENTRY_DEL,
4037 rt, nsiblings: rt->fib6_nsiblings,
4038 NULL);
4039 }
4040 list_for_each_entry_safe(sibling, next_sibling,
4041 &rt->fib6_siblings,
4042 fib6_siblings) {
4043 err = fib6_del(rt: sibling, info);
4044 if (err)
4045 goto out_unlock;
4046 }
4047 }
4048
4049 err = fib6_del(rt, info);
4050out_unlock:
4051 spin_unlock_bh(lock: &table->tb6_lock);
4052out_put:
4053 fib6_info_release(f6i: rt);
4054
4055 if (skb) {
4056 rtnl_notify(skb, net, pid: info->portid, RTNLGRP_IPV6_ROUTE,
4057 nlh: info->nlh, flags: gfp_any());
4058 }
4059 return err;
4060}
4061
4062static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg)
4063{
4064 int rc = -ESRCH;
4065
4066 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex)
4067 goto out;
4068
4069 if (cfg->fc_flags & RTF_GATEWAY &&
4070 !ipv6_addr_equal(a1: &cfg->fc_gateway, a2: &rt->rt6i_gateway))
4071 goto out;
4072
4073 rc = rt6_remove_exception_rt(rt);
4074out:
4075 return rc;
4076}
4077
4078static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt,
4079 struct fib6_nh *nh)
4080{
4081 struct fib6_result res = {
4082 .f6i = rt,
4083 .nh = nh,
4084 };
4085 struct rt6_info *rt_cache;
4086
4087 rt_cache = rt6_find_cached_rt(res: &res, daddr: &cfg->fc_dst, saddr: &cfg->fc_src);
4088 if (rt_cache)
4089 return __ip6_del_cached_rt(rt: rt_cache, cfg);
4090
4091 return 0;
4092}
4093
4094struct fib6_nh_del_cached_rt_arg {
4095 struct fib6_config *cfg;
4096 struct fib6_info *f6i;
4097};
4098
4099static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)
4100{
4101 struct fib6_nh_del_cached_rt_arg *arg = _arg;
4102 int rc;
4103
4104 rc = ip6_del_cached_rt(cfg: arg->cfg, rt: arg->f6i, nh);
4105 return rc != -ESRCH ? rc : 0;
4106}
4107
4108static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i)
4109{
4110 struct fib6_nh_del_cached_rt_arg arg = {
4111 .cfg = cfg,
4112 .f6i = f6i
4113 };
4114
4115 return nexthop_for_each_fib6_nh(nh: f6i->nh, cb: fib6_nh_del_cached_rt, arg: &arg);
4116}
4117
4118static int ip6_route_del(struct fib6_config *cfg,
4119 struct netlink_ext_ack *extack)
4120{
4121 struct fib6_table *table;
4122 struct fib6_info *rt;
4123 struct fib6_node *fn;
4124 int err = -ESRCH;
4125
4126 table = fib6_get_table(net: cfg->fc_nlinfo.nl_net, id: cfg->fc_table);
4127 if (!table) {
4128 NL_SET_ERR_MSG(extack, "FIB table does not exist");
4129 return err;
4130 }
4131
4132 rcu_read_lock();
4133
4134 fn = fib6_locate(root: &table->tb6_root,
4135 daddr: &cfg->fc_dst, dst_len: cfg->fc_dst_len,
4136 saddr: &cfg->fc_src, src_len: cfg->fc_src_len,
4137 exact_match: !(cfg->fc_flags & RTF_CACHE));
4138
4139 if (fn) {
4140 for_each_fib6_node_rt_rcu(fn) {
4141 struct fib6_nh *nh;
4142
4143 if (rt->nh && cfg->fc_nh_id &&
4144 rt->nh->id != cfg->fc_nh_id)
4145 continue;
4146
4147 if (cfg->fc_flags & RTF_CACHE) {
4148 int rc = 0;
4149
4150 if (rt->nh) {
4151 rc = ip6_del_cached_rt_nh(cfg, f6i: rt);
4152 } else if (cfg->fc_nh_id) {
4153 continue;
4154 } else {
4155 nh = rt->fib6_nh;
4156 rc = ip6_del_cached_rt(cfg, rt, nh);
4157 }
4158 if (rc != -ESRCH) {
4159 rcu_read_unlock();
4160 return rc;
4161 }
4162 continue;
4163 }
4164
4165 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric)
4166 continue;
4167 if (cfg->fc_protocol &&
4168 cfg->fc_protocol != rt->fib6_protocol)
4169 continue;
4170
4171 if (rt->nh) {
4172 if (!fib6_info_hold_safe(f6i: rt))
4173 continue;
4174
4175 err = __ip6_del_rt(rt, info: &cfg->fc_nlinfo);
4176 break;
4177 }
4178 if (cfg->fc_nh_id)
4179 continue;
4180
4181 nh = rt->fib6_nh;
4182 if (cfg->fc_ifindex &&
4183 (!nh->fib_nh_dev ||
4184 nh->fib_nh_dev->ifindex != cfg->fc_ifindex))
4185 continue;
4186 if (cfg->fc_flags & RTF_GATEWAY &&
4187 !ipv6_addr_equal(a1: &cfg->fc_gateway, a2: &nh->fib_nh_gw6))
4188 continue;
4189 if (!fib6_info_hold_safe(f6i: rt))
4190 continue;
4191
4192 /* if gateway was specified only delete the one hop */
4193 if (cfg->fc_flags & RTF_GATEWAY)
4194 err = __ip6_del_rt(rt, info: &cfg->fc_nlinfo);
4195 else
4196 err = __ip6_del_rt_siblings(rt, cfg);
4197 break;
4198 }
4199 }
4200 rcu_read_unlock();
4201
4202 return err;
4203}
4204
4205static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
4206{
4207 struct netevent_redirect netevent;
4208 struct rt6_info *rt, *nrt = NULL;
4209 struct fib6_result res = {};
4210 struct ndisc_options ndopts;
4211 struct inet6_dev *in6_dev;
4212 struct neighbour *neigh;
4213 struct rd_msg *msg;
4214 int optlen, on_link;
4215 u8 *lladdr;
4216
4217 optlen = skb_tail_pointer(skb) - skb_transport_header(skb);
4218 optlen -= sizeof(*msg);
4219
4220 if (optlen < 0) {
4221 net_dbg_ratelimited("rt6_do_redirect: packet too short\n");
4222 return;
4223 }
4224
4225 msg = (struct rd_msg *)icmp6_hdr(skb);
4226
4227 if (ipv6_addr_is_multicast(addr: &msg->dest)) {
4228 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n");
4229 return;
4230 }
4231
4232 on_link = 0;
4233 if (ipv6_addr_equal(a1: &msg->dest, a2: &msg->target)) {
4234 on_link = 1;
4235 } else if (ipv6_addr_type(addr: &msg->target) !=
4236 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
4237 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n");
4238 return;
4239 }
4240
4241 in6_dev = __in6_dev_get(dev: skb->dev);
4242 if (!in6_dev)
4243 return;
4244 if (READ_ONCE(in6_dev->cnf.forwarding) ||
4245 !READ_ONCE(in6_dev->cnf.accept_redirects))
4246 return;
4247
4248 /* RFC2461 8.1:
4249 * The IP source address of the Redirect MUST be the same as the current
4250 * first-hop router for the specified ICMP Destination Address.
4251 */
4252
4253 if (!ndisc_parse_options(dev: skb->dev, opt: msg->opt, opt_len: optlen, ndopts: &ndopts)) {
4254 net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
4255 return;
4256 }
4257
4258 lladdr = NULL;
4259 if (ndopts.nd_opts_tgt_lladdr) {
4260 lladdr = ndisc_opt_addr_data(p: ndopts.nd_opts_tgt_lladdr,
4261 dev: skb->dev);
4262 if (!lladdr) {
4263 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
4264 return;
4265 }
4266 }
4267
4268 rt = dst_rt6_info(dst);
4269 if (rt->rt6i_flags & RTF_REJECT) {
4270 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n");
4271 return;
4272 }
4273
4274 /* Redirect received -> path was valid.
4275 * Look, redirects are sent only in response to data packets,
4276 * so that this nexthop apparently is reachable. --ANK
4277 */
4278 dst_confirm_neigh(dst: &rt->dst, daddr: &ipv6_hdr(skb)->saddr);
4279
4280 neigh = __neigh_lookup(tbl: &nd_tbl, pkey: &msg->target, dev: skb->dev, creat: 1);
4281 if (!neigh)
4282 return;
4283
4284 /*
4285 * We have finally decided to accept it.
4286 */
4287
4288 ndisc_update(dev: skb->dev, neigh, lladdr, NUD_STALE,
4289 NEIGH_UPDATE_F_WEAK_OVERRIDE|
4290 NEIGH_UPDATE_F_OVERRIDE|
4291 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
4292 NEIGH_UPDATE_F_ISROUTER)),
4293 NDISC_REDIRECT, ndopts: &ndopts);
4294
4295 rcu_read_lock();
4296 res.f6i = rcu_dereference(rt->from);
4297 if (!res.f6i)
4298 goto out;
4299
4300 if (res.f6i->nh) {
4301 struct fib6_nh_match_arg arg = {
4302 .dev = dst_dev_rcu(dst),
4303 .gw = &rt->rt6i_gateway,
4304 };
4305
4306 nexthop_for_each_fib6_nh(nh: res.f6i->nh,
4307 cb: fib6_nh_find_match, arg: &arg);
4308
4309 /* fib6_info uses a nexthop that does not have fib6_nh
4310 * using the dst->dev. Should be impossible
4311 */
4312 if (!arg.match)
4313 goto out;
4314 res.nh = arg.match;
4315 } else {
4316 res.nh = res.f6i->fib6_nh;
4317 }
4318
4319 res.fib6_flags = res.f6i->fib6_flags;
4320 res.fib6_type = res.f6i->fib6_type;
4321 nrt = ip6_rt_cache_alloc(res: &res, daddr: &msg->dest, NULL);
4322 if (!nrt)
4323 goto out;
4324
4325 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE;
4326 if (on_link)
4327 nrt->rt6i_flags &= ~RTF_GATEWAY;
4328
4329 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key;
4330
4331 /* rt6_insert_exception() will take care of duplicated exceptions */
4332 if (rt6_insert_exception(nrt, res: &res)) {
4333 dst_release_immediate(dst: &nrt->dst);
4334 goto out;
4335 }
4336
4337 netevent.old = &rt->dst;
4338 netevent.new = &nrt->dst;
4339 netevent.daddr = &msg->dest;
4340 netevent.neigh = neigh;
4341 call_netevent_notifiers(val: NETEVENT_REDIRECT, v: &netevent);
4342
4343out:
4344 rcu_read_unlock();
4345 neigh_release(neigh);
4346}
4347
4348#ifdef CONFIG_IPV6_ROUTE_INFO
4349static struct fib6_info *rt6_get_route_info(struct net *net,
4350 const struct in6_addr *prefix, int prefixlen,
4351 const struct in6_addr *gwaddr,
4352 struct net_device *dev)
4353{
4354 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
4355 int ifindex = dev->ifindex;
4356 struct fib6_node *fn;
4357 struct fib6_info *rt = NULL;
4358 struct fib6_table *table;
4359
4360 table = fib6_get_table(net, tb_id);
4361 if (!table)
4362 return NULL;
4363
4364 rcu_read_lock();
4365 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true);
4366 if (!fn)
4367 goto out;
4368
4369 for_each_fib6_node_rt_rcu(fn) {
4370 /* these routes do not use nexthops */
4371 if (rt->nh)
4372 continue;
4373 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)
4374 continue;
4375 if (!(rt->fib6_flags & RTF_ROUTEINFO) ||
4376 !rt->fib6_nh->fib_nh_gw_family)
4377 continue;
4378 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))
4379 continue;
4380 if (!fib6_info_hold_safe(rt))
4381 continue;
4382 break;
4383 }
4384out:
4385 rcu_read_unlock();
4386 return rt;
4387}
4388
4389static struct fib6_info *rt6_add_route_info(struct net *net,
4390 const struct in6_addr *prefix, int prefixlen,
4391 const struct in6_addr *gwaddr,
4392 struct net_device *dev,
4393 unsigned int pref)
4394{
4395 struct fib6_config cfg = {
4396 .fc_metric = IP6_RT_PRIO_USER,
4397 .fc_ifindex = dev->ifindex,
4398 .fc_dst_len = prefixlen,
4399 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO |
4400 RTF_UP | RTF_PREF(pref),
4401 .fc_protocol = RTPROT_RA,
4402 .fc_type = RTN_UNICAST,
4403 .fc_nlinfo.portid = 0,
4404 .fc_nlinfo.nlh = NULL,
4405 .fc_nlinfo.nl_net = net,
4406 };
4407
4408 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
4409 cfg.fc_dst = *prefix;
4410 cfg.fc_gateway = *gwaddr;
4411
4412 /* We should treat it as a default route if prefix length is 0. */
4413 if (!prefixlen)
4414 cfg.fc_flags |= RTF_DEFAULT;
4415
4416 ip6_route_add(&cfg, GFP_ATOMIC, NULL);
4417
4418 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev);
4419}
4420#endif
4421
4422struct fib6_info *rt6_get_dflt_router(struct net *net,
4423 const struct in6_addr *addr,
4424 struct net_device *dev)
4425{
4426 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
4427 struct fib6_info *rt;
4428 struct fib6_table *table;
4429
4430 table = fib6_get_table(net, id: tb_id);
4431 if (!table)
4432 return NULL;
4433
4434 rcu_read_lock();
4435 for_each_fib6_node_rt_rcu(&table->tb6_root) {
4436 struct fib6_nh *nh;
4437
4438 /* RA routes do not use nexthops */
4439 if (rt->nh)
4440 continue;
4441
4442 nh = rt->fib6_nh;
4443 if (dev == nh->fib_nh_dev &&
4444 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) &&
4445 ipv6_addr_equal(a1: &nh->fib_nh_gw6, a2: addr))
4446 break;
4447 }
4448 if (rt && !fib6_info_hold_safe(f6i: rt))
4449 rt = NULL;
4450 rcu_read_unlock();
4451 return rt;
4452}
4453
4454struct fib6_info *rt6_add_dflt_router(struct net *net,
4455 const struct in6_addr *gwaddr,
4456 struct net_device *dev,
4457 unsigned int pref,
4458 u32 defrtr_usr_metric,
4459 int lifetime)
4460{
4461 struct fib6_config cfg = {
4462 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
4463 .fc_metric = defrtr_usr_metric,
4464 .fc_ifindex = dev->ifindex,
4465 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
4466 RTF_UP | RTF_EXPIRES | RTF_PREF(pref),
4467 .fc_protocol = RTPROT_RA,
4468 .fc_type = RTN_UNICAST,
4469 .fc_nlinfo.portid = 0,
4470 .fc_nlinfo.nlh = NULL,
4471 .fc_nlinfo.nl_net = net,
4472 .fc_expires = jiffies_to_clock_t(x: lifetime * HZ),
4473 };
4474
4475 cfg.fc_gateway = *gwaddr;
4476
4477 if (!ip6_route_add(cfg: &cfg, GFP_ATOMIC, NULL)) {
4478 struct fib6_table *table;
4479
4480 table = fib6_get_table(net: dev_net(dev), id: cfg.fc_table);
4481 if (table)
4482 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER;
4483 }
4484
4485 return rt6_get_dflt_router(net, addr: gwaddr, dev);
4486}
4487
4488static void __rt6_purge_dflt_routers(struct net *net,
4489 struct fib6_table *table)
4490{
4491 struct fib6_info *rt;
4492
4493restart:
4494 rcu_read_lock();
4495 for_each_fib6_node_rt_rcu(&table->tb6_root) {
4496 struct net_device *dev = fib6_info_nh_dev(f6i: rt);
4497 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;
4498
4499 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
4500 (!idev || idev->cnf.accept_ra != 2) &&
4501 fib6_info_hold_safe(f6i: rt)) {
4502 rcu_read_unlock();
4503 ip6_del_rt(net, rt, skip_notify: false);
4504 goto restart;
4505 }
4506 }
4507 rcu_read_unlock();
4508
4509 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
4510}
4511
4512void rt6_purge_dflt_routers(struct net *net)
4513{
4514 struct fib6_table *table;
4515 struct hlist_head *head;
4516 unsigned int h;
4517
4518 rcu_read_lock();
4519
4520 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
4521 head = &net->ipv6.fib_table_hash[h];
4522 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
4523 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
4524 __rt6_purge_dflt_routers(net, table);
4525 }
4526 }
4527
4528 rcu_read_unlock();
4529}
4530
4531static void rtmsg_to_fib6_config(struct net *net,
4532 struct in6_rtmsg *rtmsg,
4533 struct fib6_config *cfg)
4534{
4535 *cfg = (struct fib6_config){
4536 .fc_table = l3mdev_fib_table_by_index(net, ifindex: rtmsg->rtmsg_ifindex) ?
4537 : RT6_TABLE_MAIN,
4538 .fc_ifindex = rtmsg->rtmsg_ifindex,
4539 .fc_metric = rtmsg->rtmsg_metric,
4540 .fc_expires = rtmsg->rtmsg_info,
4541 .fc_dst_len = rtmsg->rtmsg_dst_len,
4542 .fc_src_len = rtmsg->rtmsg_src_len,
4543 .fc_flags = rtmsg->rtmsg_flags,
4544 .fc_type = rtmsg->rtmsg_type,
4545
4546 .fc_nlinfo.nl_net = net,
4547
4548 .fc_dst = rtmsg->rtmsg_dst,
4549 .fc_src = rtmsg->rtmsg_src,
4550 .fc_gateway = rtmsg->rtmsg_gateway,
4551 };
4552}
4553
4554int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg)
4555{
4556 struct fib6_config cfg;
4557 int err;
4558
4559 if (cmd != SIOCADDRT && cmd != SIOCDELRT)
4560 return -EINVAL;
4561 if (!ns_capable(ns: net->user_ns, CAP_NET_ADMIN))
4562 return -EPERM;
4563
4564 rtmsg_to_fib6_config(net, rtmsg, cfg: &cfg);
4565
4566 switch (cmd) {
4567 case SIOCADDRT:
4568 /* Only do the default setting of fc_metric in route adding */
4569 if (cfg.fc_metric == 0)
4570 cfg.fc_metric = IP6_RT_PRIO_USER;
4571 err = ip6_route_add(cfg: &cfg, GFP_KERNEL, NULL);
4572 break;
4573 case SIOCDELRT:
4574 err = ip6_route_del(cfg: &cfg, NULL);
4575 break;
4576 }
4577
4578 return err;
4579}
4580
4581/*
4582 * Drop the packet on the floor
4583 */
4584
4585static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
4586{
4587 struct dst_entry *dst = skb_dst(skb);
4588 struct net_device *dev = dst_dev(dst);
4589 struct net *net = dev_net(dev);
4590 struct inet6_dev *idev;
4591 SKB_DR(reason);
4592 int type;
4593
4594 if (netif_is_l3_master(dev: skb->dev) ||
4595 dev == net->loopback_dev)
4596 idev = __in6_dev_get_safely(dev: dev_get_by_index_rcu(net, IP6CB(skb)->iif));
4597 else
4598 idev = ip6_dst_idev(dst);
4599
4600 switch (ipstats_mib_noroutes) {
4601 case IPSTATS_MIB_INNOROUTES:
4602 type = ipv6_addr_type(addr: &ipv6_hdr(skb)->daddr);
4603 if (type == IPV6_ADDR_ANY) {
4604 SKB_DR_SET(reason, IP_INADDRERRORS);
4605 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
4606 break;
4607 }
4608 SKB_DR_SET(reason, IP_INNOROUTES);
4609 fallthrough;
4610 case IPSTATS_MIB_OUTNOROUTES:
4611 SKB_DR_OR(reason, IP_OUTNOROUTES);
4612 IP6_INC_STATS(net, idev, ipstats_mib_noroutes);
4613 break;
4614 }
4615
4616 /* Start over by dropping the dst for l3mdev case */
4617 if (netif_is_l3_master(dev: skb->dev))
4618 skb_dst_drop(skb);
4619
4620 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, info: 0);
4621 kfree_skb_reason(skb, reason);
4622 return 0;
4623}
4624
4625static int ip6_pkt_discard(struct sk_buff *skb)
4626{
4627 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, ipstats_mib_noroutes: IPSTATS_MIB_INNOROUTES);
4628}
4629
4630static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
4631{
4632 skb->dev = skb_dst_dev(skb);
4633 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, ipstats_mib_noroutes: IPSTATS_MIB_OUTNOROUTES);
4634}
4635
4636static int ip6_pkt_prohibit(struct sk_buff *skb)
4637{
4638 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, ipstats_mib_noroutes: IPSTATS_MIB_INNOROUTES);
4639}
4640
4641static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb)
4642{
4643 skb->dev = skb_dst_dev(skb);
4644 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, ipstats_mib_noroutes: IPSTATS_MIB_OUTNOROUTES);
4645}
4646
4647/*
4648 * Allocate a dst for local (unicast / anycast) address.
4649 */
4650
4651struct fib6_info *addrconf_f6i_alloc(struct net *net,
4652 struct inet6_dev *idev,
4653 const struct in6_addr *addr,
4654 bool anycast, gfp_t gfp_flags,
4655 struct netlink_ext_ack *extack)
4656{
4657 struct fib6_config cfg = {
4658 .fc_table = l3mdev_fib_table(dev: idev->dev) ? : RT6_TABLE_LOCAL,
4659 .fc_ifindex = idev->dev->ifindex,
4660 .fc_flags = RTF_UP | RTF_NONEXTHOP,
4661 .fc_dst = *addr,
4662 .fc_dst_len = 128,
4663 .fc_protocol = RTPROT_KERNEL,
4664 .fc_nlinfo.nl_net = net,
4665 .fc_ignore_dev_down = true,
4666 };
4667 struct fib6_info *f6i;
4668 int err;
4669
4670 if (anycast) {
4671 cfg.fc_type = RTN_ANYCAST;
4672 cfg.fc_flags |= RTF_ANYCAST;
4673 } else {
4674 cfg.fc_type = RTN_LOCAL;
4675 cfg.fc_flags |= RTF_LOCAL;
4676 }
4677
4678 f6i = ip6_route_info_create(cfg: &cfg, gfp_flags, extack);
4679 if (IS_ERR(ptr: f6i))
4680 return f6i;
4681
4682 err = ip6_route_info_create_nh(rt: f6i, cfg: &cfg, gfp_flags, extack);
4683 if (err)
4684 return ERR_PTR(error: err);
4685
4686 f6i->dst_nocount = true;
4687
4688 if (!anycast &&
4689 (READ_ONCE(net->ipv6.devconf_all->disable_policy) ||
4690 READ_ONCE(idev->cnf.disable_policy)))
4691 f6i->dst_nopolicy = true;
4692
4693 return f6i;
4694}
4695
4696/* remove deleted ip from prefsrc entries */
4697struct arg_dev_net_ip {
4698 struct net *net;
4699 struct in6_addr *addr;
4700};
4701
4702static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg)
4703{
4704 struct net *net = ((struct arg_dev_net_ip *)arg)->net;
4705 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr;
4706
4707 if (!rt->nh &&
4708 rt != net->ipv6.fib6_null_entry &&
4709 ipv6_addr_equal(a1: addr, a2: &rt->fib6_prefsrc.addr) &&
4710 !ipv6_chk_addr(net, addr, dev: rt->fib6_nh->fib_nh_dev, strict: 0)) {
4711 spin_lock_bh(lock: &rt6_exception_lock);
4712 /* remove prefsrc entry */
4713 rt->fib6_prefsrc.plen = 0;
4714 spin_unlock_bh(lock: &rt6_exception_lock);
4715 }
4716 return 0;
4717}
4718
4719void rt6_remove_prefsrc(struct inet6_ifaddr *ifp)
4720{
4721 struct net *net = dev_net(dev: ifp->idev->dev);
4722 struct arg_dev_net_ip adni = {
4723 .net = net,
4724 .addr = &ifp->addr,
4725 };
4726 fib6_clean_all(net, func: fib6_remove_prefsrc, arg: &adni);
4727}
4728
4729#define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
4730
4731/* Remove routers and update dst entries when gateway turn into host. */
4732static int fib6_clean_tohost(struct fib6_info *rt, void *arg)
4733{
4734 struct in6_addr *gateway = (struct in6_addr *)arg;
4735 struct fib6_nh *nh;
4736
4737 /* RA routes do not use nexthops */
4738 if (rt->nh)
4739 return 0;
4740
4741 nh = rt->fib6_nh;
4742 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) &&
4743 nh->fib_nh_gw_family && ipv6_addr_equal(a1: gateway, a2: &nh->fib_nh_gw6))
4744 return -1;
4745
4746 /* Further clean up cached routes in exception table.
4747 * This is needed because cached route may have a different
4748 * gateway than its 'parent' in the case of an ip redirect.
4749 */
4750 fib6_nh_exceptions_clean_tohost(nh, gateway);
4751
4752 return 0;
4753}
4754
4755void rt6_clean_tohost(struct net *net, struct in6_addr *gateway)
4756{
4757 fib6_clean_all(net, func: fib6_clean_tohost, arg: gateway);
4758}
4759
4760struct arg_netdev_event {
4761 const struct net_device *dev;
4762 union {
4763 unsigned char nh_flags;
4764 unsigned long event;
4765 };
4766};
4767
4768static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt)
4769{
4770 struct fib6_info *iter;
4771 struct fib6_node *fn;
4772
4773 fn = rcu_dereference_protected(rt->fib6_node,
4774 lockdep_is_held(&rt->fib6_table->tb6_lock));
4775 iter = rcu_dereference_protected(fn->leaf,
4776 lockdep_is_held(&rt->fib6_table->tb6_lock));
4777 while (iter) {
4778 if (iter->fib6_metric == rt->fib6_metric &&
4779 rt6_qualify_for_ecmp(f6i: iter))
4780 return iter;
4781 iter = rcu_dereference_protected(iter->fib6_next,
4782 lockdep_is_held(&rt->fib6_table->tb6_lock));
4783 }
4784
4785 return NULL;
4786}
4787
4788/* only called for fib entries with builtin fib6_nh */
4789static bool rt6_is_dead(const struct fib6_info *rt)
4790{
4791 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||
4792 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&
4793 ip6_ignore_linkdown(dev: rt->fib6_nh->fib_nh_dev)))
4794 return true;
4795
4796 return false;
4797}
4798
4799static int rt6_multipath_total_weight(const struct fib6_info *rt)
4800{
4801 struct fib6_info *iter;
4802 int total = 0;
4803
4804 if (!rt6_is_dead(rt))
4805 total += rt->fib6_nh->fib_nh_weight;
4806
4807 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
4808 if (!rt6_is_dead(rt: iter))
4809 total += iter->fib6_nh->fib_nh_weight;
4810 }
4811
4812 return total;
4813}
4814
4815static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total)
4816{
4817 int upper_bound = -1;
4818
4819 if (!rt6_is_dead(rt)) {
4820 *weight += rt->fib6_nh->fib_nh_weight;
4821 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31,
4822 total) - 1;
4823 }
4824 atomic_set(v: &rt->fib6_nh->fib_nh_upper_bound, i: upper_bound);
4825}
4826
4827static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total)
4828{
4829 struct fib6_info *iter;
4830 int weight = 0;
4831
4832 rt6_upper_bound_set(rt, weight: &weight, total);
4833
4834 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4835 rt6_upper_bound_set(rt: iter, weight: &weight, total);
4836}
4837
4838void rt6_multipath_rebalance(struct fib6_info *rt)
4839{
4840 struct fib6_info *first;
4841 int total;
4842
4843 /* In case the entire multipath route was marked for flushing,
4844 * then there is no need to rebalance upon the removal of every
4845 * sibling route.
4846 */
4847 if (!rt->fib6_nsiblings || rt->should_flush)
4848 return;
4849
4850 /* During lookup routes are evaluated in order, so we need to
4851 * make sure upper bounds are assigned from the first sibling
4852 * onwards.
4853 */
4854 first = rt6_multipath_first_sibling(rt);
4855 if (WARN_ON_ONCE(!first))
4856 return;
4857
4858 total = rt6_multipath_total_weight(rt: first);
4859 rt6_multipath_upper_bound_set(rt: first, total);
4860}
4861
4862static int fib6_ifup(struct fib6_info *rt, void *p_arg)
4863{
4864 const struct arg_netdev_event *arg = p_arg;
4865 struct net *net = dev_net(dev: arg->dev);
4866
4867 if (rt != net->ipv6.fib6_null_entry && !rt->nh &&
4868 rt->fib6_nh->fib_nh_dev == arg->dev) {
4869 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;
4870 fib6_update_sernum_upto_root(net, rt);
4871 rt6_multipath_rebalance(rt);
4872 }
4873
4874 return 0;
4875}
4876
4877void rt6_sync_up(struct net_device *dev, unsigned char nh_flags)
4878{
4879 struct arg_netdev_event arg = {
4880 .dev = dev,
4881 {
4882 .nh_flags = nh_flags,
4883 },
4884 };
4885
4886 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev))
4887 arg.nh_flags |= RTNH_F_LINKDOWN;
4888
4889 fib6_clean_all(net: dev_net(dev), func: fib6_ifup, arg: &arg);
4890}
4891
4892/* only called for fib entries with inline fib6_nh */
4893static bool rt6_multipath_uses_dev(const struct fib6_info *rt,
4894 const struct net_device *dev)
4895{
4896 struct fib6_info *iter;
4897
4898 if (rt->fib6_nh->fib_nh_dev == dev)
4899 return true;
4900 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4901 if (iter->fib6_nh->fib_nh_dev == dev)
4902 return true;
4903
4904 return false;
4905}
4906
4907static void rt6_multipath_flush(struct fib6_info *rt)
4908{
4909 struct fib6_info *iter;
4910
4911 rt->should_flush = 1;
4912 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4913 iter->should_flush = 1;
4914}
4915
4916static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt,
4917 const struct net_device *down_dev)
4918{
4919 struct fib6_info *iter;
4920 unsigned int dead = 0;
4921
4922 if (rt->fib6_nh->fib_nh_dev == down_dev ||
4923 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
4924 dead++;
4925 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4926 if (iter->fib6_nh->fib_nh_dev == down_dev ||
4927 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
4928 dead++;
4929
4930 return dead;
4931}
4932
4933static void rt6_multipath_nh_flags_set(struct fib6_info *rt,
4934 const struct net_device *dev,
4935 unsigned char nh_flags)
4936{
4937 struct fib6_info *iter;
4938
4939 if (rt->fib6_nh->fib_nh_dev == dev)
4940 rt->fib6_nh->fib_nh_flags |= nh_flags;
4941 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4942 if (iter->fib6_nh->fib_nh_dev == dev)
4943 iter->fib6_nh->fib_nh_flags |= nh_flags;
4944}
4945
4946/* called with write lock held for table with rt */
4947static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
4948{
4949 const struct arg_netdev_event *arg = p_arg;
4950 const struct net_device *dev = arg->dev;
4951 struct net *net = dev_net(dev);
4952
4953 if (rt == net->ipv6.fib6_null_entry || rt->nh)
4954 return 0;
4955
4956 switch (arg->event) {
4957 case NETDEV_UNREGISTER:
4958 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
4959 case NETDEV_DOWN:
4960 if (rt->should_flush)
4961 return -1;
4962 if (!rt->fib6_nsiblings)
4963 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
4964 if (rt6_multipath_uses_dev(rt, dev)) {
4965 unsigned int count;
4966
4967 count = rt6_multipath_dead_count(rt, down_dev: dev);
4968 if (rt->fib6_nsiblings + 1 == count) {
4969 rt6_multipath_flush(rt);
4970 return -1;
4971 }
4972 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD |
4973 RTNH_F_LINKDOWN);
4974 fib6_update_sernum(net, rt);
4975 rt6_multipath_rebalance(rt);
4976 }
4977 return -2;
4978 case NETDEV_CHANGE:
4979 if (rt->fib6_nh->fib_nh_dev != dev ||
4980 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
4981 break;
4982 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
4983 rt6_multipath_rebalance(rt);
4984 break;
4985 }
4986
4987 return 0;
4988}
4989
4990void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
4991{
4992 struct arg_netdev_event arg = {
4993 .dev = dev,
4994 {
4995 .event = event,
4996 },
4997 };
4998 struct net *net = dev_net(dev);
4999
5000 if (net->ipv6.sysctl.skip_notify_on_dev_down)
5001 fib6_clean_all_skip_notify(net, func: fib6_ifdown, arg: &arg);
5002 else
5003 fib6_clean_all(net, func: fib6_ifdown, arg: &arg);
5004}
5005
5006void rt6_disable_ip(struct net_device *dev, unsigned long event)
5007{
5008 rt6_sync_down_dev(dev, event);
5009 rt6_uncached_list_flush_dev(dev);
5010 neigh_ifdown(tbl: &nd_tbl, dev);
5011}
5012
5013struct rt6_mtu_change_arg {
5014 struct net_device *dev;
5015 unsigned int mtu;
5016 struct fib6_info *f6i;
5017};
5018
5019static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)
5020{
5021 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg;
5022 struct fib6_info *f6i = arg->f6i;
5023
5024 /* For administrative MTU increase, there is no way to discover
5025 * IPv6 PMTU increase, so PMTU increase should be updated here.
5026 * Since RFC 1981 doesn't include administrative MTU increase
5027 * update PMTU increase is a MUST. (i.e. jumbo frame)
5028 */
5029 if (nh->fib_nh_dev == arg->dev) {
5030 struct inet6_dev *idev = __in6_dev_get(dev: arg->dev);
5031 u32 mtu = f6i->fib6_pmtu;
5032
5033 if (mtu >= arg->mtu ||
5034 (mtu < arg->mtu && mtu == idev->cnf.mtu6))
5035 fib6_metric_set(f6i, RTAX_MTU, val: arg->mtu);
5036
5037 spin_lock_bh(lock: &rt6_exception_lock);
5038 rt6_exceptions_update_pmtu(idev, nh, mtu: arg->mtu);
5039 spin_unlock_bh(lock: &rt6_exception_lock);
5040 }
5041
5042 return 0;
5043}
5044
5045static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg)
5046{
5047 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg;
5048 struct inet6_dev *idev;
5049
5050 /* In IPv6 pmtu discovery is not optional,
5051 so that RTAX_MTU lock cannot disable it.
5052 We still use this lock to block changes
5053 caused by addrconf/ndisc.
5054 */
5055
5056 idev = __in6_dev_get(dev: arg->dev);
5057 if (!idev)
5058 return 0;
5059
5060 if (fib6_metric_locked(f6i, RTAX_MTU))
5061 return 0;
5062
5063 arg->f6i = f6i;
5064 if (f6i->nh) {
5065 /* fib6_nh_mtu_change only returns 0, so this is safe */
5066 return nexthop_for_each_fib6_nh(nh: f6i->nh, cb: fib6_nh_mtu_change,
5067 arg);
5068 }
5069
5070 return fib6_nh_mtu_change(nh: f6i->fib6_nh, arg: arg);
5071}
5072
5073void rt6_mtu_change(struct net_device *dev, unsigned int mtu)
5074{
5075 struct rt6_mtu_change_arg arg = {
5076 .dev = dev,
5077 .mtu = mtu,
5078 };
5079
5080 fib6_clean_all(net: dev_net(dev), func: rt6_mtu_change_route, arg: &arg);
5081}
5082
5083static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = {
5084 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 },
5085 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) },
5086 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) },
5087 [RTA_OIF] = { .type = NLA_U32 },
5088 [RTA_IIF] = { .type = NLA_U32 },
5089 [RTA_PRIORITY] = { .type = NLA_U32 },
5090 [RTA_METRICS] = { .type = NLA_NESTED },
5091 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) },
5092 [RTA_PREF] = { .type = NLA_U8 },
5093 [RTA_ENCAP_TYPE] = { .type = NLA_U16 },
5094 [RTA_ENCAP] = { .type = NLA_NESTED },
5095 [RTA_EXPIRES] = { .type = NLA_U32 },
5096 [RTA_UID] = { .type = NLA_U32 },
5097 [RTA_MARK] = { .type = NLA_U32 },
5098 [RTA_TABLE] = { .type = NLA_U32 },
5099 [RTA_IP_PROTO] = { .type = NLA_U8 },
5100 [RTA_SPORT] = { .type = NLA_U16 },
5101 [RTA_DPORT] = { .type = NLA_U16 },
5102 [RTA_NH_ID] = { .type = NLA_U32 },
5103 [RTA_FLOWLABEL] = { .type = NLA_BE32 },
5104};
5105
5106static int rtm_to_fib6_multipath_config(struct fib6_config *cfg,
5107 struct netlink_ext_ack *extack,
5108 bool newroute)
5109{
5110 struct rtnexthop *rtnh;
5111 int remaining;
5112
5113 remaining = cfg->fc_mp_len;
5114 rtnh = (struct rtnexthop *)cfg->fc_mp;
5115
5116 if (!rtnh_ok(rtnh, remaining)) {
5117 NL_SET_ERR_MSG(extack, "Invalid nexthop configuration - no valid nexthops");
5118 return -EINVAL;
5119 }
5120
5121 do {
5122 bool has_gateway = cfg->fc_flags & RTF_GATEWAY;
5123 int attrlen = rtnh_attrlen(rtnh);
5124
5125 if (attrlen > 0) {
5126 struct nlattr *nla, *attrs;
5127
5128 attrs = rtnh_attrs(rtnh);
5129 nla = nla_find(head: attrs, len: attrlen, attrtype: RTA_GATEWAY);
5130 if (nla) {
5131 if (nla_len(nla) < sizeof(cfg->fc_gateway)) {
5132 NL_SET_ERR_MSG(extack,
5133 "Invalid IPv6 address in RTA_GATEWAY");
5134 return -EINVAL;
5135 }
5136
5137 has_gateway = true;
5138 }
5139 }
5140
5141 if (newroute && (cfg->fc_nh_id || !has_gateway)) {
5142 NL_SET_ERR_MSG(extack,
5143 "Device only routes can not be added for IPv6 using the multipath API.");
5144 return -EINVAL;
5145 }
5146
5147 rtnh = rtnh_next(rtnh, remaining: &remaining);
5148 } while (rtnh_ok(rtnh, remaining));
5149
5150 return lwtunnel_valid_encap_type_attr(attr: cfg->fc_mp, len: cfg->fc_mp_len, extack);
5151}
5152
5153static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
5154 struct fib6_config *cfg,
5155 struct netlink_ext_ack *extack)
5156{
5157 bool newroute = nlh->nlmsg_type == RTM_NEWROUTE;
5158 struct nlattr *tb[RTA_MAX+1];
5159 struct rtmsg *rtm;
5160 unsigned int pref;
5161 int err;
5162
5163 err = nlmsg_parse_deprecated(nlh, hdrlen: sizeof(*rtm), tb, RTA_MAX,
5164 policy: rtm_ipv6_policy, extack);
5165 if (err < 0)
5166 goto errout;
5167
5168 err = -EINVAL;
5169 rtm = nlmsg_data(nlh);
5170
5171 if (rtm->rtm_tos) {
5172 NL_SET_ERR_MSG(extack,
5173 "Invalid dsfield (tos): option not available for IPv6");
5174 goto errout;
5175 }
5176
5177 if (tb[RTA_FLOWLABEL]) {
5178 NL_SET_ERR_MSG_ATTR(extack, tb[RTA_FLOWLABEL],
5179 "Flow label cannot be specified for this operation");
5180 goto errout;
5181 }
5182
5183 *cfg = (struct fib6_config){
5184 .fc_table = rtm->rtm_table,
5185 .fc_dst_len = rtm->rtm_dst_len,
5186 .fc_src_len = rtm->rtm_src_len,
5187 .fc_flags = RTF_UP,
5188 .fc_protocol = rtm->rtm_protocol,
5189 .fc_type = rtm->rtm_type,
5190
5191 .fc_nlinfo.portid = NETLINK_CB(skb).portid,
5192 .fc_nlinfo.nlh = nlh,
5193 .fc_nlinfo.nl_net = sock_net(sk: skb->sk),
5194 };
5195
5196 if (rtm->rtm_type == RTN_UNREACHABLE ||
5197 rtm->rtm_type == RTN_BLACKHOLE ||
5198 rtm->rtm_type == RTN_PROHIBIT ||
5199 rtm->rtm_type == RTN_THROW)
5200 cfg->fc_flags |= RTF_REJECT;
5201
5202 if (rtm->rtm_type == RTN_LOCAL)
5203 cfg->fc_flags |= RTF_LOCAL;
5204
5205 if (rtm->rtm_flags & RTM_F_CLONED)
5206 cfg->fc_flags |= RTF_CACHE;
5207
5208 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);
5209
5210 if (tb[RTA_NH_ID]) {
5211 if (tb[RTA_GATEWAY] || tb[RTA_OIF] ||
5212 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) {
5213 NL_SET_ERR_MSG(extack,
5214 "Nexthop specification and nexthop id are mutually exclusive");
5215 goto errout;
5216 }
5217 cfg->fc_nh_id = nla_get_u32(nla: tb[RTA_NH_ID]);
5218 }
5219
5220 if (tb[RTA_GATEWAY]) {
5221 cfg->fc_gateway = nla_get_in6_addr(nla: tb[RTA_GATEWAY]);
5222 cfg->fc_flags |= RTF_GATEWAY;
5223 }
5224 if (tb[RTA_VIA]) {
5225 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute");
5226 goto errout;
5227 }
5228
5229 if (tb[RTA_DST]) {
5230 int plen = (rtm->rtm_dst_len + 7) >> 3;
5231
5232 if (nla_len(nla: tb[RTA_DST]) < plen)
5233 goto errout;
5234
5235 nla_memcpy(dest: &cfg->fc_dst, src: tb[RTA_DST], count: plen);
5236 }
5237
5238 if (tb[RTA_SRC]) {
5239 int plen = (rtm->rtm_src_len + 7) >> 3;
5240
5241 if (nla_len(nla: tb[RTA_SRC]) < plen)
5242 goto errout;
5243
5244 nla_memcpy(dest: &cfg->fc_src, src: tb[RTA_SRC], count: plen);
5245 }
5246
5247 if (tb[RTA_PREFSRC])
5248 cfg->fc_prefsrc = nla_get_in6_addr(nla: tb[RTA_PREFSRC]);
5249
5250 if (tb[RTA_OIF])
5251 cfg->fc_ifindex = nla_get_u32(nla: tb[RTA_OIF]);
5252
5253 if (tb[RTA_PRIORITY])
5254 cfg->fc_metric = nla_get_u32(nla: tb[RTA_PRIORITY]);
5255
5256 if (tb[RTA_METRICS]) {
5257 cfg->fc_mx = nla_data(nla: tb[RTA_METRICS]);
5258 cfg->fc_mx_len = nla_len(nla: tb[RTA_METRICS]);
5259 }
5260
5261 if (tb[RTA_TABLE])
5262 cfg->fc_table = nla_get_u32(nla: tb[RTA_TABLE]);
5263
5264 if (tb[RTA_MULTIPATH]) {
5265 cfg->fc_mp = nla_data(nla: tb[RTA_MULTIPATH]);
5266 cfg->fc_mp_len = nla_len(nla: tb[RTA_MULTIPATH]);
5267
5268 err = rtm_to_fib6_multipath_config(cfg, extack, newroute);
5269 if (err < 0)
5270 goto errout;
5271 }
5272
5273 if (tb[RTA_PREF]) {
5274 pref = nla_get_u8(nla: tb[RTA_PREF]);
5275 if (pref != ICMPV6_ROUTER_PREF_LOW &&
5276 pref != ICMPV6_ROUTER_PREF_HIGH)
5277 pref = ICMPV6_ROUTER_PREF_MEDIUM;
5278 cfg->fc_flags |= RTF_PREF(pref);
5279 }
5280
5281 if (tb[RTA_ENCAP])
5282 cfg->fc_encap = tb[RTA_ENCAP];
5283
5284 if (tb[RTA_ENCAP_TYPE]) {
5285 cfg->fc_encap_type = nla_get_u16(nla: tb[RTA_ENCAP_TYPE]);
5286
5287 err = lwtunnel_valid_encap_type(encap_type: cfg->fc_encap_type, extack);
5288 if (err < 0)
5289 goto errout;
5290 }
5291
5292 if (tb[RTA_EXPIRES]) {
5293 unsigned long timeout = addrconf_timeout_fixup(timeout: nla_get_u32(nla: tb[RTA_EXPIRES]), HZ);
5294
5295 if (addrconf_finite_timeout(timeout)) {
5296 cfg->fc_expires = jiffies_to_clock_t(x: timeout * HZ);
5297 cfg->fc_flags |= RTF_EXPIRES;
5298 }
5299 }
5300
5301 err = 0;
5302errout:
5303 return err;
5304}
5305
5306struct rt6_nh {
5307 struct fib6_info *fib6_info;
5308 struct fib6_config r_cfg;
5309 struct list_head list;
5310};
5311
5312static int ip6_route_info_append(struct list_head *rt6_nh_list,
5313 struct fib6_info *rt,
5314 struct fib6_config *r_cfg)
5315{
5316 struct rt6_nh *nh;
5317
5318 list_for_each_entry(nh, rt6_nh_list, list) {
5319 /* check if fib6_info already exists */
5320 if (rt6_duplicate_nexthop(a: nh->fib6_info, b: rt))
5321 return -EEXIST;
5322 }
5323
5324 nh = kzalloc(sizeof(*nh), GFP_KERNEL);
5325 if (!nh)
5326 return -ENOMEM;
5327
5328 nh->fib6_info = rt;
5329 memcpy(to: &nh->r_cfg, from: r_cfg, len: sizeof(*r_cfg));
5330 list_add_tail(new: &nh->list, head: rt6_nh_list);
5331
5332 return 0;
5333}
5334
5335static void ip6_route_mpath_notify(struct fib6_info *rt,
5336 struct fib6_info *rt_last,
5337 struct nl_info *info,
5338 __u16 nlflags)
5339{
5340 /* if this is an APPEND route, then rt points to the first route
5341 * inserted and rt_last points to last route inserted. Userspace
5342 * wants a consistent dump of the route which starts at the first
5343 * nexthop. Since sibling routes are always added at the end of
5344 * the list, find the first sibling of the last route appended
5345 */
5346 rcu_read_lock();
5347
5348 if ((nlflags & NLM_F_APPEND) && rt_last &&
5349 READ_ONCE(rt_last->fib6_nsiblings)) {
5350 rt = list_first_or_null_rcu(&rt_last->fib6_siblings,
5351 struct fib6_info,
5352 fib6_siblings);
5353 }
5354
5355 if (rt)
5356 inet6_rt_notify(RTM_NEWROUTE, rt, info, flags: nlflags);
5357
5358 rcu_read_unlock();
5359}
5360
5361static bool ip6_route_mpath_should_notify(const struct fib6_info *rt)
5362{
5363 bool rt_can_ecmp = rt6_qualify_for_ecmp(f6i: rt);
5364 bool should_notify = false;
5365 struct fib6_info *leaf;
5366 struct fib6_node *fn;
5367
5368 rcu_read_lock();
5369 fn = rcu_dereference(rt->fib6_node);
5370 if (!fn)
5371 goto out;
5372
5373 leaf = rcu_dereference(fn->leaf);
5374 if (!leaf)
5375 goto out;
5376
5377 if (rt == leaf ||
5378 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric &&
5379 rt6_qualify_for_ecmp(f6i: leaf)))
5380 should_notify = true;
5381out:
5382 rcu_read_unlock();
5383
5384 return should_notify;
5385}
5386
5387static int ip6_route_multipath_add(struct fib6_config *cfg,
5388 struct netlink_ext_ack *extack)
5389{
5390 struct fib6_info *rt_notif = NULL, *rt_last = NULL;
5391 struct nl_info *info = &cfg->fc_nlinfo;
5392 struct rt6_nh *nh, *nh_safe;
5393 struct fib6_config r_cfg;
5394 struct rtnexthop *rtnh;
5395 LIST_HEAD(rt6_nh_list);
5396 struct rt6_nh *err_nh;
5397 struct fib6_info *rt;
5398 __u16 nlflags;
5399 int remaining;
5400 int attrlen;
5401 int replace;
5402 int nhn = 0;
5403 int err;
5404
5405 err = fib6_config_validate(cfg, extack);
5406 if (err)
5407 return err;
5408
5409 replace = (cfg->fc_nlinfo.nlh &&
5410 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE));
5411
5412 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE;
5413 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND)
5414 nlflags |= NLM_F_APPEND;
5415
5416 remaining = cfg->fc_mp_len;
5417 rtnh = (struct rtnexthop *)cfg->fc_mp;
5418
5419 /* Parse a Multipath Entry and build a list (rt6_nh_list) of
5420 * fib6_info structs per nexthop
5421 */
5422 while (rtnh_ok(rtnh, remaining)) {
5423 memcpy(to: &r_cfg, from: cfg, len: sizeof(*cfg));
5424 if (rtnh->rtnh_ifindex)
5425 r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
5426
5427 attrlen = rtnh_attrlen(rtnh);
5428 if (attrlen > 0) {
5429 struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
5430
5431 nla = nla_find(head: attrs, len: attrlen, attrtype: RTA_GATEWAY);
5432 if (nla) {
5433 r_cfg.fc_gateway = nla_get_in6_addr(nla);
5434 r_cfg.fc_flags |= RTF_GATEWAY;
5435 }
5436
5437 r_cfg.fc_encap = nla_find(head: attrs, len: attrlen, attrtype: RTA_ENCAP);
5438 nla = nla_find(head: attrs, len: attrlen, attrtype: RTA_ENCAP_TYPE);
5439 if (nla)
5440 r_cfg.fc_encap_type = nla_get_u16(nla);
5441 }
5442
5443 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK);
5444 rt = ip6_route_info_create(cfg: &r_cfg, GFP_KERNEL, extack);
5445 if (IS_ERR(ptr: rt)) {
5446 err = PTR_ERR(ptr: rt);
5447 rt = NULL;
5448 goto cleanup;
5449 }
5450
5451 err = ip6_route_info_create_nh(rt, cfg: &r_cfg, GFP_KERNEL, extack);
5452 if (err) {
5453 rt = NULL;
5454 goto cleanup;
5455 }
5456
5457 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;
5458
5459 err = ip6_route_info_append(rt6_nh_list: &rt6_nh_list, rt, r_cfg: &r_cfg);
5460 if (err) {
5461 fib6_info_release(f6i: rt);
5462 goto cleanup;
5463 }
5464
5465 rtnh = rtnh_next(rtnh, remaining: &remaining);
5466 }
5467
5468 /* for add and replace send one notification with all nexthops.
5469 * Skip the notification in fib6_add_rt2node and send one with
5470 * the full route when done
5471 */
5472 info->skip_notify = 1;
5473
5474 /* For add and replace, send one notification with all nexthops. For
5475 * append, send one notification with all appended nexthops.
5476 */
5477 info->skip_notify_kernel = 1;
5478
5479 err_nh = NULL;
5480 list_for_each_entry(nh, &rt6_nh_list, list) {
5481 err = __ip6_ins_rt(rt: nh->fib6_info, info, extack);
5482
5483 if (err) {
5484 if (replace && nhn)
5485 NL_SET_ERR_MSG_MOD(extack,
5486 "multipath route replace failed (check consistency of installed routes)");
5487 err_nh = nh;
5488 goto add_errout;
5489 }
5490 /* save reference to last route successfully inserted */
5491 rt_last = nh->fib6_info;
5492
5493 /* save reference to first route for notification */
5494 if (!rt_notif)
5495 rt_notif = nh->fib6_info;
5496
5497 /* Because each route is added like a single route we remove
5498 * these flags after the first nexthop: if there is a collision,
5499 * we have already failed to add the first nexthop:
5500 * fib6_add_rt2node() has rejected it; when replacing, old
5501 * nexthops have been replaced by first new, the rest should
5502 * be added to it.
5503 */
5504 if (cfg->fc_nlinfo.nlh) {
5505 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL |
5506 NLM_F_REPLACE);
5507 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE;
5508 }
5509 nhn++;
5510 }
5511
5512 /* An in-kernel notification should only be sent in case the new
5513 * multipath route is added as the first route in the node, or if
5514 * it was appended to it. We pass 'rt_notif' since it is the first
5515 * sibling and might allow us to skip some checks in the replace case.
5516 */
5517 if (ip6_route_mpath_should_notify(rt: rt_notif)) {
5518 enum fib_event_type fib_event;
5519
5520 if (rt_notif->fib6_nsiblings != nhn - 1)
5521 fib_event = FIB_EVENT_ENTRY_APPEND;
5522 else
5523 fib_event = FIB_EVENT_ENTRY_REPLACE;
5524
5525 err = call_fib6_multipath_entry_notifiers(net: info->nl_net,
5526 event_type: fib_event, rt: rt_notif,
5527 nsiblings: nhn - 1, extack);
5528 if (err) {
5529 /* Delete all the siblings that were just added */
5530 err_nh = NULL;
5531 goto add_errout;
5532 }
5533 }
5534
5535 /* success ... tell user about new route */
5536 ip6_route_mpath_notify(rt: rt_notif, rt_last, info, nlflags);
5537 goto cleanup;
5538
5539add_errout:
5540 /* send notification for routes that were added so that
5541 * the delete notifications sent by ip6_route_del are
5542 * coherent
5543 */
5544 if (rt_notif)
5545 ip6_route_mpath_notify(rt: rt_notif, rt_last, info, nlflags);
5546
5547 /* Delete routes that were already added */
5548 list_for_each_entry(nh, &rt6_nh_list, list) {
5549 if (err_nh == nh)
5550 break;
5551 ip6_route_del(cfg: &nh->r_cfg, extack);
5552 }
5553
5554cleanup:
5555 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, list) {
5556 fib6_info_release(f6i: nh->fib6_info);
5557 list_del(entry: &nh->list);
5558 kfree(objp: nh);
5559 }
5560
5561 return err;
5562}
5563
5564static int ip6_route_multipath_del(struct fib6_config *cfg,
5565 struct netlink_ext_ack *extack)
5566{
5567 struct fib6_config r_cfg;
5568 struct rtnexthop *rtnh;
5569 int last_err = 0;
5570 int remaining;
5571 int attrlen;
5572 int err;
5573
5574 remaining = cfg->fc_mp_len;
5575 rtnh = (struct rtnexthop *)cfg->fc_mp;
5576
5577 /* Parse a Multipath Entry */
5578 while (rtnh_ok(rtnh, remaining)) {
5579 memcpy(to: &r_cfg, from: cfg, len: sizeof(*cfg));
5580 if (rtnh->rtnh_ifindex)
5581 r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
5582
5583 attrlen = rtnh_attrlen(rtnh);
5584 if (attrlen > 0) {
5585 struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
5586
5587 nla = nla_find(head: attrs, len: attrlen, attrtype: RTA_GATEWAY);
5588 if (nla) {
5589 r_cfg.fc_gateway = nla_get_in6_addr(nla);
5590 r_cfg.fc_flags |= RTF_GATEWAY;
5591 }
5592 }
5593
5594 err = ip6_route_del(cfg: &r_cfg, extack);
5595 if (err)
5596 last_err = err;
5597
5598 rtnh = rtnh_next(rtnh, remaining: &remaining);
5599 }
5600
5601 return last_err;
5602}
5603
5604static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
5605 struct netlink_ext_ack *extack)
5606{
5607 struct fib6_config cfg;
5608 int err;
5609
5610 err = rtm_to_fib6_config(skb, nlh, cfg: &cfg, extack);
5611 if (err < 0)
5612 return err;
5613
5614 if (cfg.fc_nh_id) {
5615 rcu_read_lock();
5616 err = !nexthop_find_by_id(net: sock_net(sk: skb->sk), id: cfg.fc_nh_id);
5617 rcu_read_unlock();
5618
5619 if (err) {
5620 NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
5621 return -EINVAL;
5622 }
5623 }
5624
5625 if (cfg.fc_mp) {
5626 return ip6_route_multipath_del(cfg: &cfg, extack);
5627 } else {
5628 cfg.fc_delete_all_nh = 1;
5629 return ip6_route_del(cfg: &cfg, extack);
5630 }
5631}
5632
5633static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
5634 struct netlink_ext_ack *extack)
5635{
5636 struct fib6_config cfg;
5637 int err;
5638
5639 err = rtm_to_fib6_config(skb, nlh, cfg: &cfg, extack);
5640 if (err < 0)
5641 return err;
5642
5643 if (cfg.fc_metric == 0)
5644 cfg.fc_metric = IP6_RT_PRIO_USER;
5645
5646 if (cfg.fc_mp)
5647 return ip6_route_multipath_add(cfg: &cfg, extack);
5648 else
5649 return ip6_route_add(cfg: &cfg, GFP_KERNEL, extack);
5650}
5651
5652/* add the overhead of this fib6_nh to nexthop_len */
5653static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)
5654{
5655 int *nexthop_len = arg;
5656
5657 *nexthop_len += nla_total_size(payload: 0) /* RTA_MULTIPATH */
5658 + NLA_ALIGN(sizeof(struct rtnexthop))
5659 + nla_total_size(payload: 16); /* RTA_GATEWAY */
5660
5661 if (nh->fib_nh_lws) {
5662 /* RTA_ENCAP_TYPE */
5663 *nexthop_len += lwtunnel_get_encap_size(lwtstate: nh->fib_nh_lws);
5664 /* RTA_ENCAP */
5665 *nexthop_len += nla_total_size(payload: 2);
5666 }
5667
5668 return 0;
5669}
5670
5671static size_t rt6_nlmsg_size(struct fib6_info *f6i)
5672{
5673 struct fib6_info *sibling;
5674 struct fib6_nh *nh;
5675 int nexthop_len;
5676
5677 if (f6i->nh) {
5678 nexthop_len = nla_total_size(payload: 4); /* RTA_NH_ID */
5679 nexthop_for_each_fib6_nh(nh: f6i->nh, cb: rt6_nh_nlmsg_size,
5680 arg: &nexthop_len);
5681 goto common;
5682 }
5683
5684 rcu_read_lock();
5685retry:
5686 nh = f6i->fib6_nh;
5687 nexthop_len = 0;
5688 if (READ_ONCE(f6i->fib6_nsiblings)) {
5689 rt6_nh_nlmsg_size(nh, arg: &nexthop_len);
5690
5691 list_for_each_entry_rcu(sibling, &f6i->fib6_siblings,
5692 fib6_siblings) {
5693 rt6_nh_nlmsg_size(nh: sibling->fib6_nh, arg: &nexthop_len);
5694 if (!READ_ONCE(f6i->fib6_nsiblings))
5695 goto retry;
5696 }
5697 }
5698 rcu_read_unlock();
5699 nexthop_len += lwtunnel_get_encap_size(lwtstate: nh->fib_nh_lws);
5700common:
5701 return NLMSG_ALIGN(sizeof(struct rtmsg))
5702 + nla_total_size(payload: 16) /* RTA_SRC */
5703 + nla_total_size(payload: 16) /* RTA_DST */
5704 + nla_total_size(payload: 16) /* RTA_GATEWAY */
5705 + nla_total_size(payload: 16) /* RTA_PREFSRC */
5706 + nla_total_size(payload: 4) /* RTA_TABLE */
5707 + nla_total_size(payload: 4) /* RTA_IIF */
5708 + nla_total_size(payload: 4) /* RTA_OIF */
5709 + nla_total_size(payload: 4) /* RTA_PRIORITY */
5710 + RTAX_MAX * nla_total_size(payload: 4) /* RTA_METRICS */
5711 + nla_total_size(payload: sizeof(struct rta_cacheinfo))
5712 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */
5713 + nla_total_size(payload: 1) /* RTA_PREF */
5714 + nexthop_len;
5715}
5716
5717static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,
5718 unsigned char *flags)
5719{
5720 if (nexthop_is_multipath(nh)) {
5721 struct nlattr *mp;
5722
5723 mp = nla_nest_start_noflag(skb, attrtype: RTA_MULTIPATH);
5724 if (!mp)
5725 goto nla_put_failure;
5726
5727 if (nexthop_mpath_fill_node(skb, nh, AF_INET6))
5728 goto nla_put_failure;
5729
5730 nla_nest_end(skb, start: mp);
5731 } else {
5732 struct fib6_nh *fib6_nh;
5733
5734 fib6_nh = nexthop_fib6_nh(nh);
5735 if (fib_nexthop_info(skb, nh: &fib6_nh->nh_common, AF_INET6,
5736 flags, skip_oif: false) < 0)
5737 goto nla_put_failure;
5738 }
5739
5740 return 0;
5741
5742nla_put_failure:
5743 return -EMSGSIZE;
5744}
5745
5746static int rt6_fill_node(struct net *net, struct sk_buff *skb,
5747 struct fib6_info *rt, struct dst_entry *dst,
5748 struct in6_addr *dest, struct in6_addr *src,
5749 int iif, int type, u32 portid, u32 seq,
5750 unsigned int flags)
5751{
5752 struct rt6_info *rt6 = dst_rt6_info(dst);
5753 struct rt6key *rt6_dst, *rt6_src;
5754 u32 *pmetrics, table, rt6_flags;
5755 unsigned char nh_flags = 0;
5756 struct nlmsghdr *nlh;
5757 struct rtmsg *rtm;
5758 long expires = 0;
5759
5760 nlh = nlmsg_put(skb, portid, seq, type, payload: sizeof(*rtm), flags);
5761 if (!nlh)
5762 return -EMSGSIZE;
5763
5764 if (rt6) {
5765 rt6_dst = &rt6->rt6i_dst;
5766 rt6_src = &rt6->rt6i_src;
5767 rt6_flags = rt6->rt6i_flags;
5768 } else {
5769 rt6_dst = &rt->fib6_dst;
5770 rt6_src = &rt->fib6_src;
5771 rt6_flags = rt->fib6_flags;
5772 }
5773
5774 rtm = nlmsg_data(nlh);
5775 rtm->rtm_family = AF_INET6;
5776 rtm->rtm_dst_len = rt6_dst->plen;
5777 rtm->rtm_src_len = rt6_src->plen;
5778 rtm->rtm_tos = 0;
5779 if (rt->fib6_table)
5780 table = rt->fib6_table->tb6_id;
5781 else
5782 table = RT6_TABLE_UNSPEC;
5783 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT;
5784 if (nla_put_u32(skb, attrtype: RTA_TABLE, value: table))
5785 goto nla_put_failure;
5786
5787 rtm->rtm_type = rt->fib6_type;
5788 rtm->rtm_flags = 0;
5789 rtm->rtm_scope = RT_SCOPE_UNIVERSE;
5790 rtm->rtm_protocol = rt->fib6_protocol;
5791
5792 if (rt6_flags & RTF_CACHE)
5793 rtm->rtm_flags |= RTM_F_CLONED;
5794
5795 if (dest) {
5796 if (nla_put_in6_addr(skb, attrtype: RTA_DST, addr: dest))
5797 goto nla_put_failure;
5798 rtm->rtm_dst_len = 128;
5799 } else if (rtm->rtm_dst_len)
5800 if (nla_put_in6_addr(skb, attrtype: RTA_DST, addr: &rt6_dst->addr))
5801 goto nla_put_failure;
5802#ifdef CONFIG_IPV6_SUBTREES
5803 if (src) {
5804 if (nla_put_in6_addr(skb, RTA_SRC, src))
5805 goto nla_put_failure;
5806 rtm->rtm_src_len = 128;
5807 } else if (rtm->rtm_src_len &&
5808 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr))
5809 goto nla_put_failure;
5810#endif
5811 if (iif) {
5812#ifdef CONFIG_IPV6_MROUTE
5813 if (ipv6_addr_is_multicast(&rt6_dst->addr)) {
5814 int err = ip6mr_get_route(net, skb, rtm, portid);
5815
5816 if (err == 0)
5817 return 0;
5818 if (err < 0)
5819 goto nla_put_failure;
5820 } else
5821#endif
5822 if (nla_put_u32(skb, attrtype: RTA_IIF, value: iif))
5823 goto nla_put_failure;
5824 } else if (dest) {
5825 struct in6_addr saddr_buf;
5826 if (ip6_route_get_saddr(net, f6i: rt, daddr: dest, prefs: 0, l3mdev_index: 0, saddr: &saddr_buf) == 0 &&
5827 nla_put_in6_addr(skb, attrtype: RTA_PREFSRC, addr: &saddr_buf))
5828 goto nla_put_failure;
5829 }
5830
5831 if (rt->fib6_prefsrc.plen) {
5832 struct in6_addr saddr_buf;
5833 saddr_buf = rt->fib6_prefsrc.addr;
5834 if (nla_put_in6_addr(skb, attrtype: RTA_PREFSRC, addr: &saddr_buf))
5835 goto nla_put_failure;
5836 }
5837
5838 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics;
5839 if (rtnetlink_put_metrics(skb, metrics: pmetrics) < 0)
5840 goto nla_put_failure;
5841
5842 if (nla_put_u32(skb, attrtype: RTA_PRIORITY, value: rt->fib6_metric))
5843 goto nla_put_failure;
5844
5845 /* For multipath routes, walk the siblings list and add
5846 * each as a nexthop within RTA_MULTIPATH.
5847 */
5848 if (rt6) {
5849 struct net_device *dev;
5850
5851 if (rt6_flags & RTF_GATEWAY &&
5852 nla_put_in6_addr(skb, attrtype: RTA_GATEWAY, addr: &rt6->rt6i_gateway))
5853 goto nla_put_failure;
5854
5855 dev = dst_dev(dst);
5856 if (dev && nla_put_u32(skb, attrtype: RTA_OIF, value: dev->ifindex))
5857 goto nla_put_failure;
5858
5859 if (lwtunnel_fill_encap(skb, lwtstate: dst->lwtstate, encap_attr: RTA_ENCAP, encap_type_attr: RTA_ENCAP_TYPE) < 0)
5860 goto nla_put_failure;
5861 } else if (READ_ONCE(rt->fib6_nsiblings)) {
5862 struct fib6_info *sibling;
5863 struct nlattr *mp;
5864
5865 mp = nla_nest_start_noflag(skb, attrtype: RTA_MULTIPATH);
5866 if (!mp)
5867 goto nla_put_failure;
5868
5869 if (fib_add_nexthop(skb, nh: &rt->fib6_nh->nh_common,
5870 nh_weight: rt->fib6_nh->fib_nh_weight, AF_INET6,
5871 nh_tclassid: 0) < 0)
5872 goto nla_put_failure;
5873
5874 rcu_read_lock();
5875
5876 list_for_each_entry_rcu(sibling, &rt->fib6_siblings,
5877 fib6_siblings) {
5878 if (fib_add_nexthop(skb, nh: &sibling->fib6_nh->nh_common,
5879 nh_weight: sibling->fib6_nh->fib_nh_weight,
5880 AF_INET6, nh_tclassid: 0) < 0) {
5881 rcu_read_unlock();
5882
5883 goto nla_put_failure;
5884 }
5885 }
5886
5887 rcu_read_unlock();
5888
5889 nla_nest_end(skb, start: mp);
5890 } else if (rt->nh) {
5891 if (nla_put_u32(skb, attrtype: RTA_NH_ID, value: rt->nh->id))
5892 goto nla_put_failure;
5893
5894 if (nexthop_is_blackhole(nh: rt->nh))
5895 rtm->rtm_type = RTN_BLACKHOLE;
5896
5897 if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) &&
5898 rt6_fill_node_nexthop(skb, nh: rt->nh, flags: &nh_flags) < 0)
5899 goto nla_put_failure;
5900
5901 rtm->rtm_flags |= nh_flags;
5902 } else {
5903 if (fib_nexthop_info(skb, nh: &rt->fib6_nh->nh_common, AF_INET6,
5904 flags: &nh_flags, skip_oif: false) < 0)
5905 goto nla_put_failure;
5906
5907 rtm->rtm_flags |= nh_flags;
5908 }
5909
5910 if (rt6_flags & RTF_EXPIRES) {
5911 expires = dst ? READ_ONCE(dst->expires) : rt->expires;
5912 expires -= jiffies;
5913 }
5914
5915 if (!dst) {
5916 if (READ_ONCE(rt->offload))
5917 rtm->rtm_flags |= RTM_F_OFFLOAD;
5918 if (READ_ONCE(rt->trap))
5919 rtm->rtm_flags |= RTM_F_TRAP;
5920 if (READ_ONCE(rt->offload_failed))
5921 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED;
5922 }
5923
5924 if (rtnl_put_cacheinfo(skb, dst, id: 0, expires, error: dst ? dst->error : 0) < 0)
5925 goto nla_put_failure;
5926
5927 if (nla_put_u8(skb, attrtype: RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags)))
5928 goto nla_put_failure;
5929
5930
5931 nlmsg_end(skb, nlh);
5932 return 0;
5933
5934nla_put_failure:
5935 nlmsg_cancel(skb, nlh);
5936 return -EMSGSIZE;
5937}
5938
5939static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)
5940{
5941 const struct net_device *dev = arg;
5942
5943 if (nh->fib_nh_dev == dev)
5944 return 1;
5945
5946 return 0;
5947}
5948
5949static bool fib6_info_uses_dev(const struct fib6_info *f6i,
5950 const struct net_device *dev)
5951{
5952 if (f6i->nh) {
5953 struct net_device *_dev = (struct net_device *)dev;
5954
5955 return !!nexthop_for_each_fib6_nh(nh: f6i->nh,
5956 cb: fib6_info_nh_uses_dev,
5957 arg: _dev);
5958 }
5959
5960 if (f6i->fib6_nh->fib_nh_dev == dev)
5961 return true;
5962
5963 if (READ_ONCE(f6i->fib6_nsiblings)) {
5964 const struct fib6_info *sibling;
5965
5966 rcu_read_lock();
5967 list_for_each_entry_rcu(sibling, &f6i->fib6_siblings,
5968 fib6_siblings) {
5969 if (sibling->fib6_nh->fib_nh_dev == dev) {
5970 rcu_read_unlock();
5971 return true;
5972 }
5973 if (!READ_ONCE(f6i->fib6_nsiblings))
5974 break;
5975 }
5976 rcu_read_unlock();
5977 }
5978 return false;
5979}
5980
5981struct fib6_nh_exception_dump_walker {
5982 struct rt6_rtnl_dump_arg *dump;
5983 struct fib6_info *rt;
5984 unsigned int flags;
5985 unsigned int skip;
5986 unsigned int count;
5987};
5988
5989static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)
5990{
5991 struct fib6_nh_exception_dump_walker *w = arg;
5992 struct rt6_rtnl_dump_arg *dump = w->dump;
5993 struct rt6_exception_bucket *bucket;
5994 struct rt6_exception *rt6_ex;
5995 int i, err;
5996
5997 bucket = fib6_nh_get_excptn_bucket(nh, NULL);
5998 if (!bucket)
5999 return 0;
6000
6001 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
6002 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
6003 if (w->skip) {
6004 w->skip--;
6005 continue;
6006 }
6007
6008 /* Expiration of entries doesn't bump sernum, insertion
6009 * does. Removal is triggered by insertion, so we can
6010 * rely on the fact that if entries change between two
6011 * partial dumps, this node is scanned again completely,
6012 * see rt6_insert_exception() and fib6_dump_table().
6013 *
6014 * Count expired entries we go through as handled
6015 * entries that we'll skip next time, in case of partial
6016 * node dump. Otherwise, if entries expire meanwhile,
6017 * we'll skip the wrong amount.
6018 */
6019 if (rt6_check_expired(rt: rt6_ex->rt6i)) {
6020 w->count++;
6021 continue;
6022 }
6023
6024 err = rt6_fill_node(net: dump->net, skb: dump->skb, rt: w->rt,
6025 dst: &rt6_ex->rt6i->dst, NULL, NULL, iif: 0,
6026 RTM_NEWROUTE,
6027 NETLINK_CB(dump->cb->skb).portid,
6028 seq: dump->cb->nlh->nlmsg_seq, flags: w->flags);
6029 if (err)
6030 return err;
6031
6032 w->count++;
6033 }
6034 bucket++;
6035 }
6036
6037 return 0;
6038}
6039
6040/* Return -1 if done with node, number of handled routes on partial dump */
6041int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip)
6042{
6043 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg;
6044 struct fib_dump_filter *filter = &arg->filter;
6045 unsigned int flags = NLM_F_MULTI;
6046 struct net *net = arg->net;
6047 int count = 0;
6048
6049 if (rt == net->ipv6.fib6_null_entry)
6050 return -1;
6051
6052 if ((filter->flags & RTM_F_PREFIX) &&
6053 !(rt->fib6_flags & RTF_PREFIX_RT)) {
6054 /* success since this is not a prefix route */
6055 return -1;
6056 }
6057 if (filter->filter_set &&
6058 ((filter->rt_type && rt->fib6_type != filter->rt_type) ||
6059 (filter->dev && !fib6_info_uses_dev(f6i: rt, dev: filter->dev)) ||
6060 (filter->protocol && rt->fib6_protocol != filter->protocol))) {
6061 return -1;
6062 }
6063
6064 if (filter->filter_set ||
6065 !filter->dump_routes || !filter->dump_exceptions) {
6066 flags |= NLM_F_DUMP_FILTERED;
6067 }
6068
6069 if (filter->dump_routes) {
6070 if (skip) {
6071 skip--;
6072 } else {
6073 if (rt6_fill_node(net, skb: arg->skb, rt, NULL, NULL, NULL,
6074 iif: 0, RTM_NEWROUTE,
6075 NETLINK_CB(arg->cb->skb).portid,
6076 seq: arg->cb->nlh->nlmsg_seq, flags)) {
6077 return 0;
6078 }
6079 count++;
6080 }
6081 }
6082
6083 if (filter->dump_exceptions) {
6084 struct fib6_nh_exception_dump_walker w = { .dump = arg,
6085 .rt = rt,
6086 .flags = flags,
6087 .skip = skip,
6088 .count = 0 };
6089 int err;
6090
6091 rcu_read_lock();
6092 if (rt->nh) {
6093 err = nexthop_for_each_fib6_nh(nh: rt->nh,
6094 cb: rt6_nh_dump_exceptions,
6095 arg: &w);
6096 } else {
6097 err = rt6_nh_dump_exceptions(nh: rt->fib6_nh, arg: &w);
6098 }
6099 rcu_read_unlock();
6100
6101 if (err)
6102 return count + w.count;
6103 }
6104
6105 return -1;
6106}
6107
6108static int inet6_rtm_valid_getroute_req(struct sk_buff *skb,
6109 const struct nlmsghdr *nlh,
6110 struct nlattr **tb,
6111 struct netlink_ext_ack *extack)
6112{
6113 struct rtmsg *rtm;
6114 int i, err;
6115
6116 rtm = nlmsg_payload(nlh, len: sizeof(*rtm));
6117 if (!rtm) {
6118 NL_SET_ERR_MSG_MOD(extack,
6119 "Invalid header for get route request");
6120 return -EINVAL;
6121 }
6122
6123 if (!netlink_strict_get_check(skb))
6124 return nlmsg_parse_deprecated(nlh, hdrlen: sizeof(*rtm), tb, RTA_MAX,
6125 policy: rtm_ipv6_policy, extack);
6126
6127 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) ||
6128 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) ||
6129 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope ||
6130 rtm->rtm_type) {
6131 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
6132 return -EINVAL;
6133 }
6134 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
6135 NL_SET_ERR_MSG_MOD(extack,
6136 "Invalid flags for get route request");
6137 return -EINVAL;
6138 }
6139
6140 err = nlmsg_parse_deprecated_strict(nlh, hdrlen: sizeof(*rtm), tb, RTA_MAX,
6141 policy: rtm_ipv6_policy, extack);
6142 if (err)
6143 return err;
6144
6145 if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
6146 (tb[RTA_DST] && !rtm->rtm_dst_len)) {
6147 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6");
6148 return -EINVAL;
6149 }
6150
6151 if (tb[RTA_FLOWLABEL] &&
6152 (nla_get_be32(nla: tb[RTA_FLOWLABEL]) & ~IPV6_FLOWLABEL_MASK)) {
6153 NL_SET_ERR_MSG_ATTR(extack, tb[RTA_FLOWLABEL],
6154 "Invalid flow label");
6155 return -EINVAL;
6156 }
6157
6158 for (i = 0; i <= RTA_MAX; i++) {
6159 if (!tb[i])
6160 continue;
6161
6162 switch (i) {
6163 case RTA_SRC:
6164 case RTA_DST:
6165 case RTA_IIF:
6166 case RTA_OIF:
6167 case RTA_MARK:
6168 case RTA_UID:
6169 case RTA_SPORT:
6170 case RTA_DPORT:
6171 case RTA_IP_PROTO:
6172 case RTA_FLOWLABEL:
6173 break;
6174 default:
6175 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
6176 return -EINVAL;
6177 }
6178 }
6179
6180 return 0;
6181}
6182
6183static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
6184 struct netlink_ext_ack *extack)
6185{
6186 struct net *net = sock_net(sk: in_skb->sk);
6187 struct nlattr *tb[RTA_MAX+1];
6188 int err, iif = 0, oif = 0;
6189 struct fib6_info *from;
6190 struct dst_entry *dst;
6191 struct rt6_info *rt;
6192 struct sk_buff *skb;
6193 struct rtmsg *rtm;
6194 struct flowi6 fl6 = {};
6195 __be32 flowlabel;
6196 bool fibmatch;
6197
6198 err = inet6_rtm_valid_getroute_req(skb: in_skb, nlh, tb, extack);
6199 if (err < 0)
6200 goto errout;
6201
6202 err = -EINVAL;
6203 rtm = nlmsg_data(nlh);
6204 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH);
6205
6206 if (tb[RTA_SRC]) {
6207 if (nla_len(nla: tb[RTA_SRC]) < sizeof(struct in6_addr))
6208 goto errout;
6209
6210 fl6.saddr = *(struct in6_addr *)nla_data(nla: tb[RTA_SRC]);
6211 }
6212
6213 if (tb[RTA_DST]) {
6214 if (nla_len(nla: tb[RTA_DST]) < sizeof(struct in6_addr))
6215 goto errout;
6216
6217 fl6.daddr = *(struct in6_addr *)nla_data(nla: tb[RTA_DST]);
6218 }
6219
6220 if (tb[RTA_IIF])
6221 iif = nla_get_u32(nla: tb[RTA_IIF]);
6222
6223 if (tb[RTA_OIF])
6224 oif = nla_get_u32(nla: tb[RTA_OIF]);
6225
6226 if (tb[RTA_MARK])
6227 fl6.flowi6_mark = nla_get_u32(nla: tb[RTA_MARK]);
6228
6229 if (tb[RTA_UID])
6230 fl6.flowi6_uid = make_kuid(from: current_user_ns(),
6231 uid: nla_get_u32(nla: tb[RTA_UID]));
6232 else
6233 fl6.flowi6_uid = iif ? INVALID_UID : current_uid();
6234
6235 if (tb[RTA_SPORT])
6236 fl6.fl6_sport = nla_get_be16(nla: tb[RTA_SPORT]);
6237
6238 if (tb[RTA_DPORT])
6239 fl6.fl6_dport = nla_get_be16(nla: tb[RTA_DPORT]);
6240
6241 if (tb[RTA_IP_PROTO]) {
6242 err = rtm_getroute_parse_ip_proto(attr: tb[RTA_IP_PROTO],
6243 ip_proto: &fl6.flowi6_proto, AF_INET6,
6244 extack);
6245 if (err)
6246 goto errout;
6247 }
6248
6249 flowlabel = nla_get_be32_default(nla: tb[RTA_FLOWLABEL], defvalue: 0);
6250 fl6.flowlabel = ip6_make_flowinfo(tclass: rtm->rtm_tos, flowlabel);
6251
6252 if (iif) {
6253 struct net_device *dev;
6254 int flags = 0;
6255
6256 rcu_read_lock();
6257
6258 dev = dev_get_by_index_rcu(net, ifindex: iif);
6259 if (!dev) {
6260 rcu_read_unlock();
6261 err = -ENODEV;
6262 goto errout;
6263 }
6264
6265 fl6.flowi6_iif = iif;
6266
6267 if (!ipv6_addr_any(a: &fl6.saddr))
6268 flags |= RT6_LOOKUP_F_HAS_SADDR;
6269
6270 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags);
6271
6272 rcu_read_unlock();
6273 } else {
6274 fl6.flowi6_oif = oif;
6275
6276 dst = ip6_route_output(net, NULL, fl6: &fl6);
6277 }
6278
6279
6280 rt = dst_rt6_info(dst);
6281 if (rt->dst.error) {
6282 err = rt->dst.error;
6283 ip6_rt_put(rt);
6284 goto errout;
6285 }
6286
6287 if (rt == net->ipv6.ip6_null_entry) {
6288 err = rt->dst.error;
6289 ip6_rt_put(rt);
6290 goto errout;
6291 }
6292
6293 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
6294 if (!skb) {
6295 ip6_rt_put(rt);
6296 err = -ENOBUFS;
6297 goto errout;
6298 }
6299
6300 skb_dst_set(skb, dst: &rt->dst);
6301
6302 rcu_read_lock();
6303 from = rcu_dereference(rt->from);
6304 if (from) {
6305 if (fibmatch)
6306 err = rt6_fill_node(net, skb, rt: from, NULL, NULL, NULL,
6307 iif, RTM_NEWROUTE,
6308 NETLINK_CB(in_skb).portid,
6309 seq: nlh->nlmsg_seq, flags: 0);
6310 else
6311 err = rt6_fill_node(net, skb, rt: from, dst, dest: &fl6.daddr,
6312 src: &fl6.saddr, iif, RTM_NEWROUTE,
6313 NETLINK_CB(in_skb).portid,
6314 seq: nlh->nlmsg_seq, flags: 0);
6315 } else {
6316 err = -ENETUNREACH;
6317 }
6318 rcu_read_unlock();
6319
6320 if (err < 0) {
6321 kfree_skb(skb);
6322 goto errout;
6323 }
6324
6325 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
6326errout:
6327 return err;
6328}
6329
6330void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
6331 unsigned int nlm_flags)
6332{
6333 struct net *net = info->nl_net;
6334 struct sk_buff *skb;
6335 size_t sz;
6336 u32 seq;
6337 int err;
6338
6339 err = -ENOBUFS;
6340 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
6341
6342 rcu_read_lock();
6343 sz = rt6_nlmsg_size(f6i: rt);
6344retry:
6345 skb = nlmsg_new(payload: sz, GFP_ATOMIC);
6346 if (!skb)
6347 goto errout;
6348
6349 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, iif: 0,
6350 type: event, portid: info->portid, seq, flags: nlm_flags);
6351 if (err < 0) {
6352 kfree_skb(skb);
6353 /* -EMSGSIZE implies needed space grew under us. */
6354 if (err == -EMSGSIZE) {
6355 sz = max(rt6_nlmsg_size(rt), sz << 1);
6356 goto retry;
6357 }
6358 goto errout;
6359 }
6360
6361 rcu_read_unlock();
6362
6363 rtnl_notify(skb, net, pid: info->portid, RTNLGRP_IPV6_ROUTE,
6364 nlh: info->nlh, GFP_ATOMIC);
6365 return;
6366errout:
6367 rcu_read_unlock();
6368 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, error: err);
6369}
6370
6371void fib6_rt_update(struct net *net, struct fib6_info *rt,
6372 struct nl_info *info)
6373{
6374 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
6375 struct sk_buff *skb;
6376 int err = -ENOBUFS;
6377
6378 skb = nlmsg_new(payload: rt6_nlmsg_size(f6i: rt), flags: gfp_any());
6379 if (!skb)
6380 goto errout;
6381
6382 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, iif: 0,
6383 RTM_NEWROUTE, portid: info->portid, seq, NLM_F_REPLACE);
6384 if (err < 0) {
6385 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6386 WARN_ON(err == -EMSGSIZE);
6387 kfree_skb(skb);
6388 goto errout;
6389 }
6390 rtnl_notify(skb, net, pid: info->portid, RTNLGRP_IPV6_ROUTE,
6391 nlh: info->nlh, flags: gfp_any());
6392 return;
6393errout:
6394 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, error: err);
6395}
6396
6397void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i,
6398 bool offload, bool trap, bool offload_failed)
6399{
6400 struct sk_buff *skb;
6401 int err;
6402
6403 if (READ_ONCE(f6i->offload) == offload &&
6404 READ_ONCE(f6i->trap) == trap &&
6405 READ_ONCE(f6i->offload_failed) == offload_failed)
6406 return;
6407
6408 WRITE_ONCE(f6i->offload, offload);
6409 WRITE_ONCE(f6i->trap, trap);
6410
6411 /* 2 means send notifications only if offload_failed was changed. */
6412 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 &&
6413 READ_ONCE(f6i->offload_failed) == offload_failed)
6414 return;
6415
6416 WRITE_ONCE(f6i->offload_failed, offload_failed);
6417
6418 if (!rcu_access_pointer(f6i->fib6_node))
6419 /* The route was removed from the tree, do not send
6420 * notification.
6421 */
6422 return;
6423
6424 if (!net->ipv6.sysctl.fib_notify_on_flag_change)
6425 return;
6426
6427 skb = nlmsg_new(payload: rt6_nlmsg_size(f6i), GFP_KERNEL);
6428 if (!skb) {
6429 err = -ENOBUFS;
6430 goto errout;
6431 }
6432
6433 err = rt6_fill_node(net, skb, rt: f6i, NULL, NULL, NULL, iif: 0, RTM_NEWROUTE, portid: 0,
6434 seq: 0, flags: 0);
6435 if (err < 0) {
6436 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6437 WARN_ON(err == -EMSGSIZE);
6438 kfree_skb(skb);
6439 goto errout;
6440 }
6441
6442 rtnl_notify(skb, net, pid: 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL);
6443 return;
6444
6445errout:
6446 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, error: err);
6447}
6448EXPORT_SYMBOL(fib6_info_hw_flags_set);
6449
6450static int ip6_route_dev_notify(struct notifier_block *this,
6451 unsigned long event, void *ptr)
6452{
6453 struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
6454 struct net *net = dev_net(dev);
6455
6456 if (!(dev->flags & IFF_LOOPBACK))
6457 return NOTIFY_OK;
6458
6459 if (event == NETDEV_REGISTER) {
6460 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;
6461 net->ipv6.ip6_null_entry->dst.dev = dev;
6462 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev);
6463#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6464 net->ipv6.ip6_prohibit_entry->dst.dev = dev;
6465 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev);
6466 net->ipv6.ip6_blk_hole_entry->dst.dev = dev;
6467 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev);
6468#endif
6469 } else if (event == NETDEV_UNREGISTER &&
6470 dev->reg_state != NETREG_UNREGISTERED) {
6471 /* NETDEV_UNREGISTER could be fired for multiple times by
6472 * netdev_wait_allrefs(). Make sure we only call this once.
6473 */
6474 in6_dev_put_clear(pidev: &net->ipv6.ip6_null_entry->rt6i_idev);
6475#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6476 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev);
6477 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev);
6478#endif
6479 }
6480
6481 return NOTIFY_OK;
6482}
6483
6484/*
6485 * /proc
6486 */
6487
6488#ifdef CONFIG_PROC_FS
6489static int rt6_stats_seq_show(struct seq_file *seq, void *v)
6490{
6491 struct net *net = (struct net *)seq->private;
6492 seq_printf(m: seq, fmt: "%04x %04x %04x %04x %04x %04x %04x\n",
6493 net->ipv6.rt6_stats->fib_nodes,
6494 net->ipv6.rt6_stats->fib_route_nodes,
6495 atomic_read(v: &net->ipv6.rt6_stats->fib_rt_alloc),
6496 net->ipv6.rt6_stats->fib_rt_entries,
6497 net->ipv6.rt6_stats->fib_rt_cache,
6498 dst_entries_get_slow(dst: &net->ipv6.ip6_dst_ops),
6499 net->ipv6.rt6_stats->fib_discarded_routes);
6500
6501 return 0;
6502}
6503#endif /* CONFIG_PROC_FS */
6504
6505#ifdef CONFIG_SYSCTL
6506
6507static int ipv6_sysctl_rtcache_flush(const struct ctl_table *ctl, int write,
6508 void *buffer, size_t *lenp, loff_t *ppos)
6509{
6510 struct net *net;
6511 int delay;
6512 int ret;
6513 if (!write)
6514 return -EINVAL;
6515
6516 ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
6517 if (ret)
6518 return ret;
6519
6520 net = (struct net *)ctl->extra1;
6521 delay = net->ipv6.sysctl.flush_delay;
6522 fib6_run_gc(expires: delay <= 0 ? 0 : (unsigned long)delay, net, force: delay > 0);
6523 return 0;
6524}
6525
6526static struct ctl_table ipv6_route_table_template[] = {
6527 {
6528 .procname = "max_size",
6529 .data = &init_net.ipv6.sysctl.ip6_rt_max_size,
6530 .maxlen = sizeof(int),
6531 .mode = 0644,
6532 .proc_handler = proc_dointvec,
6533 },
6534 {
6535 .procname = "gc_thresh",
6536 .data = &ip6_dst_ops_template.gc_thresh,
6537 .maxlen = sizeof(int),
6538 .mode = 0644,
6539 .proc_handler = proc_dointvec,
6540 },
6541 {
6542 .procname = "flush",
6543 .data = &init_net.ipv6.sysctl.flush_delay,
6544 .maxlen = sizeof(int),
6545 .mode = 0200,
6546 .proc_handler = ipv6_sysctl_rtcache_flush
6547 },
6548 {
6549 .procname = "gc_min_interval",
6550 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
6551 .maxlen = sizeof(int),
6552 .mode = 0644,
6553 .proc_handler = proc_dointvec_jiffies,
6554 },
6555 {
6556 .procname = "gc_timeout",
6557 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout,
6558 .maxlen = sizeof(int),
6559 .mode = 0644,
6560 .proc_handler = proc_dointvec_jiffies,
6561 },
6562 {
6563 .procname = "gc_interval",
6564 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval,
6565 .maxlen = sizeof(int),
6566 .mode = 0644,
6567 .proc_handler = proc_dointvec_jiffies,
6568 },
6569 {
6570 .procname = "gc_elasticity",
6571 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity,
6572 .maxlen = sizeof(int),
6573 .mode = 0644,
6574 .proc_handler = proc_dointvec,
6575 },
6576 {
6577 .procname = "mtu_expires",
6578 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires,
6579 .maxlen = sizeof(int),
6580 .mode = 0644,
6581 .proc_handler = proc_dointvec_jiffies,
6582 },
6583 {
6584 .procname = "min_adv_mss",
6585 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss,
6586 .maxlen = sizeof(int),
6587 .mode = 0644,
6588 .proc_handler = proc_dointvec,
6589 },
6590 {
6591 .procname = "gc_min_interval_ms",
6592 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
6593 .maxlen = sizeof(int),
6594 .mode = 0644,
6595 .proc_handler = proc_dointvec_ms_jiffies,
6596 },
6597 {
6598 .procname = "skip_notify_on_dev_down",
6599 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down,
6600 .maxlen = sizeof(u8),
6601 .mode = 0644,
6602 .proc_handler = proc_dou8vec_minmax,
6603 .extra1 = SYSCTL_ZERO,
6604 .extra2 = SYSCTL_ONE,
6605 },
6606};
6607
6608struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
6609{
6610 struct ctl_table *table;
6611
6612 table = kmemdup(ipv6_route_table_template,
6613 sizeof(ipv6_route_table_template),
6614 GFP_KERNEL);
6615
6616 if (table) {
6617 table[0].data = &net->ipv6.sysctl.ip6_rt_max_size;
6618 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh;
6619 table[2].data = &net->ipv6.sysctl.flush_delay;
6620 table[2].extra1 = net;
6621 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
6622 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout;
6623 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval;
6624 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity;
6625 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires;
6626 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss;
6627 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
6628 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down;
6629 }
6630
6631 return table;
6632}
6633
6634size_t ipv6_route_sysctl_table_size(struct net *net)
6635{
6636 /* Don't export sysctls to unprivileged users */
6637 if (net->user_ns != &init_user_ns)
6638 return 1;
6639
6640 return ARRAY_SIZE(ipv6_route_table_template);
6641}
6642#endif
6643
6644static int __net_init ip6_route_net_init(struct net *net)
6645{
6646 int ret = -ENOMEM;
6647
6648 memcpy(to: &net->ipv6.ip6_dst_ops, from: &ip6_dst_ops_template,
6649 len: sizeof(net->ipv6.ip6_dst_ops));
6650
6651 if (dst_entries_init(dst: &net->ipv6.ip6_dst_ops) < 0)
6652 goto out_ip6_dst_ops;
6653
6654 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, with_fib6_nh: true);
6655 if (!net->ipv6.fib6_null_entry)
6656 goto out_ip6_dst_entries;
6657 memcpy(to: net->ipv6.fib6_null_entry, from: &fib6_null_entry_template,
6658 len: sizeof(*net->ipv6.fib6_null_entry));
6659
6660 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template,
6661 sizeof(*net->ipv6.ip6_null_entry),
6662 GFP_KERNEL);
6663 if (!net->ipv6.ip6_null_entry)
6664 goto out_fib6_null_entry;
6665 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops;
6666 dst_init_metrics(dst: &net->ipv6.ip6_null_entry->dst,
6667 src_metrics: ip6_template_metrics, read_only: true);
6668 INIT_LIST_HEAD(list: &net->ipv6.ip6_null_entry->dst.rt_uncached);
6669
6670#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6671 net->ipv6.fib6_has_custom_rules = false;
6672 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template,
6673 sizeof(*net->ipv6.ip6_prohibit_entry),
6674 GFP_KERNEL);
6675 if (!net->ipv6.ip6_prohibit_entry)
6676 goto out_ip6_null_entry;
6677 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops;
6678 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst,
6679 ip6_template_metrics, true);
6680 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->dst.rt_uncached);
6681
6682 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template,
6683 sizeof(*net->ipv6.ip6_blk_hole_entry),
6684 GFP_KERNEL);
6685 if (!net->ipv6.ip6_blk_hole_entry)
6686 goto out_ip6_prohibit_entry;
6687 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops;
6688 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst,
6689 ip6_template_metrics, true);
6690 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->dst.rt_uncached);
6691#ifdef CONFIG_IPV6_SUBTREES
6692 net->ipv6.fib6_routes_require_src = 0;
6693#endif
6694#endif
6695
6696 net->ipv6.sysctl.flush_delay = 0;
6697 net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
6698 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
6699 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
6700 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
6701 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9;
6702 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ;
6703 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;
6704 net->ipv6.sysctl.skip_notify_on_dev_down = 0;
6705
6706 atomic_set(v: &net->ipv6.ip6_rt_gc_expire, i: 30*HZ);
6707
6708 ret = 0;
6709out:
6710 return ret;
6711
6712#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6713out_ip6_prohibit_entry:
6714 kfree(net->ipv6.ip6_prohibit_entry);
6715out_ip6_null_entry:
6716 kfree(net->ipv6.ip6_null_entry);
6717#endif
6718out_fib6_null_entry:
6719 kfree(objp: net->ipv6.fib6_null_entry);
6720out_ip6_dst_entries:
6721 dst_entries_destroy(dst: &net->ipv6.ip6_dst_ops);
6722out_ip6_dst_ops:
6723 goto out;
6724}
6725
6726static void __net_exit ip6_route_net_exit(struct net *net)
6727{
6728 kfree(objp: net->ipv6.fib6_null_entry);
6729 kfree(objp: net->ipv6.ip6_null_entry);
6730#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6731 kfree(net->ipv6.ip6_prohibit_entry);
6732 kfree(net->ipv6.ip6_blk_hole_entry);
6733#endif
6734 dst_entries_destroy(dst: &net->ipv6.ip6_dst_ops);
6735}
6736
6737static int __net_init ip6_route_net_init_late(struct net *net)
6738{
6739#ifdef CONFIG_PROC_FS
6740 if (!proc_create_net("ipv6_route", 0, net->proc_net,
6741 &ipv6_route_seq_ops,
6742 sizeof(struct ipv6_route_iter)))
6743 return -ENOMEM;
6744
6745 if (!proc_create_net_single(name: "rt6_stats", mode: 0444, parent: net->proc_net,
6746 show: rt6_stats_seq_show, NULL)) {
6747 remove_proc_entry("ipv6_route", net->proc_net);
6748 return -ENOMEM;
6749 }
6750#endif
6751 return 0;
6752}
6753
6754static void __net_exit ip6_route_net_exit_late(struct net *net)
6755{
6756#ifdef CONFIG_PROC_FS
6757 remove_proc_entry("ipv6_route", net->proc_net);
6758 remove_proc_entry("rt6_stats", net->proc_net);
6759#endif
6760}
6761
6762static struct pernet_operations ip6_route_net_ops = {
6763 .init = ip6_route_net_init,
6764 .exit = ip6_route_net_exit,
6765};
6766
6767static int __net_init ipv6_inetpeer_init(struct net *net)
6768{
6769 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
6770
6771 if (!bp)
6772 return -ENOMEM;
6773 inet_peer_base_init(bp);
6774 net->ipv6.peers = bp;
6775 return 0;
6776}
6777
6778static void __net_exit ipv6_inetpeer_exit(struct net *net)
6779{
6780 struct inet_peer_base *bp = net->ipv6.peers;
6781
6782 net->ipv6.peers = NULL;
6783 inetpeer_invalidate_tree(bp);
6784 kfree(objp: bp);
6785}
6786
6787static struct pernet_operations ipv6_inetpeer_ops = {
6788 .init = ipv6_inetpeer_init,
6789 .exit = ipv6_inetpeer_exit,
6790};
6791
6792static struct pernet_operations ip6_route_net_late_ops = {
6793 .init = ip6_route_net_init_late,
6794 .exit = ip6_route_net_exit_late,
6795};
6796
6797static struct notifier_block ip6_route_dev_notifier = {
6798 .notifier_call = ip6_route_dev_notify,
6799 .priority = ADDRCONF_NOTIFY_PRIORITY - 10,
6800};
6801
6802void __init ip6_route_init_special_entries(void)
6803{
6804 /* Registering of the loopback is done before this portion of code,
6805 * the loopback reference in rt6_info will not be taken, do it
6806 * manually for init_net */
6807 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;
6808 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev;
6809 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev: init_net.loopback_dev);
6810 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6811 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev;
6812 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6813 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev;
6814 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6815 #endif
6816}
6817
6818#if IS_BUILTIN(CONFIG_IPV6)
6819#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6820DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt)
6821
6822BTF_ID_LIST_SINGLE(btf_fib6_info_id, struct, fib6_info)
6823
6824static const struct bpf_iter_seq_info ipv6_route_seq_info = {
6825 .seq_ops = &ipv6_route_seq_ops,
6826 .init_seq_private = bpf_iter_init_seq_net,
6827 .fini_seq_private = bpf_iter_fini_seq_net,
6828 .seq_priv_size = sizeof(struct ipv6_route_iter),
6829};
6830
6831static struct bpf_iter_reg ipv6_route_reg_info = {
6832 .target = "ipv6_route",
6833 .ctx_arg_info_size = 1,
6834 .ctx_arg_info = {
6835 { offsetof(struct bpf_iter__ipv6_route, rt),
6836 PTR_TO_BTF_ID_OR_NULL },
6837 },
6838 .seq_info = &ipv6_route_seq_info,
6839};
6840
6841static int __init bpf_iter_register(void)
6842{
6843 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id;
6844 return bpf_iter_reg_target(&ipv6_route_reg_info);
6845}
6846
6847static void bpf_iter_unregister(void)
6848{
6849 bpf_iter_unreg_target(&ipv6_route_reg_info);
6850}
6851#endif
6852#endif
6853
6854static const struct rtnl_msg_handler ip6_route_rtnl_msg_handlers[] __initconst_or_module = {
6855 {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_NEWROUTE,
6856 .doit = inet6_rtm_newroute, .flags = RTNL_FLAG_DOIT_UNLOCKED},
6857 {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_DELROUTE,
6858 .doit = inet6_rtm_delroute, .flags = RTNL_FLAG_DOIT_UNLOCKED},
6859 {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_GETROUTE,
6860 .doit = inet6_rtm_getroute, .flags = RTNL_FLAG_DOIT_UNLOCKED},
6861};
6862
6863int __init ip6_route_init(void)
6864{
6865 int ret;
6866 int cpu;
6867
6868 ret = -ENOMEM;
6869 ip6_dst_ops_template.kmem_cachep =
6870 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0,
6871 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
6872 if (!ip6_dst_ops_template.kmem_cachep)
6873 goto out;
6874
6875 ret = dst_entries_init(dst: &ip6_dst_blackhole_ops);
6876 if (ret)
6877 goto out_kmem_cache;
6878
6879 ret = register_pernet_subsys(&ipv6_inetpeer_ops);
6880 if (ret)
6881 goto out_dst_entries;
6882
6883 ret = register_pernet_subsys(&ip6_route_net_ops);
6884 if (ret)
6885 goto out_register_inetpeer;
6886
6887 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep;
6888
6889 ret = fib6_init();
6890 if (ret)
6891 goto out_register_subsys;
6892
6893 ret = xfrm6_init();
6894 if (ret)
6895 goto out_fib6_init;
6896
6897 ret = fib6_rules_init();
6898 if (ret)
6899 goto xfrm6_init;
6900
6901 ret = register_pernet_subsys(&ip6_route_net_late_ops);
6902 if (ret)
6903 goto fib6_rules_init;
6904
6905 ret = rtnl_register_many(ip6_route_rtnl_msg_handlers);
6906 if (ret < 0)
6907 goto out_register_late_subsys;
6908
6909 ret = register_netdevice_notifier(nb: &ip6_route_dev_notifier);
6910 if (ret)
6911 goto out_register_late_subsys;
6912
6913#if IS_BUILTIN(CONFIG_IPV6)
6914#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6915 ret = bpf_iter_register();
6916 if (ret)
6917 goto out_register_late_subsys;
6918#endif
6919#endif
6920
6921 for_each_possible_cpu(cpu) {
6922 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
6923
6924 INIT_LIST_HEAD(list: &ul->head);
6925 spin_lock_init(&ul->lock);
6926 }
6927
6928out:
6929 return ret;
6930
6931out_register_late_subsys:
6932 rtnl_unregister_all(PF_INET6);
6933 unregister_pernet_subsys(&ip6_route_net_late_ops);
6934fib6_rules_init:
6935 fib6_rules_cleanup();
6936xfrm6_init:
6937 xfrm6_fini();
6938out_fib6_init:
6939 fib6_gc_cleanup();
6940out_register_subsys:
6941 unregister_pernet_subsys(&ip6_route_net_ops);
6942out_register_inetpeer:
6943 unregister_pernet_subsys(&ipv6_inetpeer_ops);
6944out_dst_entries:
6945 dst_entries_destroy(dst: &ip6_dst_blackhole_ops);
6946out_kmem_cache:
6947 kmem_cache_destroy(s: ip6_dst_ops_template.kmem_cachep);
6948 goto out;
6949}
6950
6951void ip6_route_cleanup(void)
6952{
6953#if IS_BUILTIN(CONFIG_IPV6)
6954#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6955 bpf_iter_unregister();
6956#endif
6957#endif
6958 unregister_netdevice_notifier(nb: &ip6_route_dev_notifier);
6959 unregister_pernet_subsys(&ip6_route_net_late_ops);
6960 fib6_rules_cleanup();
6961 xfrm6_fini();
6962 fib6_gc_cleanup();
6963 unregister_pernet_subsys(&ipv6_inetpeer_ops);
6964 unregister_pernet_subsys(&ip6_route_net_ops);
6965 dst_entries_destroy(dst: &ip6_dst_blackhole_ops);
6966 kmem_cache_destroy(s: ip6_dst_ops_template.kmem_cachep);
6967}
6968