1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef _NET_CORE_DEV_H
3#define _NET_CORE_DEV_H
4
5#include <linux/cleanup.h>
6#include <linux/types.h>
7#include <linux/rwsem.h>
8#include <linux/netdevice.h>
9#include <net/netdev_lock.h>
10
11struct net;
12struct netlink_ext_ack;
13struct cpumask;
14
15/* Random bits of netdevice that don't need to be exposed */
16#define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */
17struct sd_flow_limit {
18 struct rcu_head rcu;
19 unsigned int count;
20 u8 log_buckets;
21 unsigned int history_head;
22 u16 history[FLOW_LIMIT_HISTORY];
23 u8 buckets[];
24};
25
26extern int netdev_flow_limit_table_len;
27
28struct napi_struct *
29netdev_napi_by_id_lock(struct net *net, unsigned int napi_id);
30struct net_device *dev_get_by_napi_id(unsigned int napi_id);
31
32struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex);
33struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net);
34struct net_device *
35netdev_xa_find_lock(struct net *net, struct net_device *dev,
36 unsigned long *index);
37
38DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T));
39
40#define for_each_netdev_lock_scoped(net, var_name, ifindex) \
41 for (struct net_device *var_name __free(netdev_unlock) = NULL; \
42 (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \
43 ifindex++)
44
45struct net_device *
46netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex);
47struct net_device *
48netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev,
49 unsigned long *index);
50
51DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *,
52 if (_T) netdev_unlock_ops_compat(_T));
53
54#define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \
55 for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \
56 (var_name = netdev_xa_find_lock_ops_compat(net, var_name, \
57 &ifindex)); \
58 ifindex++)
59
60#ifdef CONFIG_PROC_FS
61int __init dev_proc_init(void);
62#else
63#define dev_proc_init() 0
64#endif
65
66void linkwatch_init_dev(struct net_device *dev);
67void linkwatch_run_queue(void);
68
69void dev_addr_flush(struct net_device *dev);
70int dev_addr_init(struct net_device *dev);
71void dev_addr_check(struct net_device *dev);
72
73#if IS_ENABLED(CONFIG_NET_SHAPER)
74void net_shaper_flush_netdev(struct net_device *dev);
75void net_shaper_set_real_num_tx_queues(struct net_device *dev,
76 unsigned int txq);
77#else
78static inline void net_shaper_flush_netdev(struct net_device *dev) {}
79static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev,
80 unsigned int txq) {}
81#endif
82
83/* sysctls not referred to from outside net/core/ */
84extern int netdev_unregister_timeout_secs;
85extern int weight_p;
86extern int dev_weight_rx_bias;
87extern int dev_weight_tx_bias;
88
89extern struct rw_semaphore dev_addr_sem;
90
91/* rtnl helpers */
92extern struct list_head net_todo_list;
93void netdev_run_todo(void);
94
95/* netdev management, shared between various uAPI entry points */
96struct netdev_name_node {
97 struct hlist_node hlist;
98 struct list_head list;
99 struct net_device *dev;
100 const char *name;
101 struct rcu_head rcu;
102};
103
104int netdev_get_name(struct net *net, char *name, int ifindex);
105int netif_change_name(struct net_device *dev, const char *newname);
106int dev_change_name(struct net_device *dev, const char *newname);
107
108#define netdev_for_each_altname(dev, namenode) \
109 list_for_each_entry((namenode), &(dev)->name_node->list, list)
110#define netdev_for_each_altname_safe(dev, namenode, next) \
111 list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \
112 list)
113
114int netdev_name_node_alt_create(struct net_device *dev, const char *name);
115int netdev_name_node_alt_destroy(struct net_device *dev, const char *name);
116
117int dev_validate_mtu(struct net_device *dev, int mtu,
118 struct netlink_ext_ack *extack);
119int netif_set_mtu_ext(struct net_device *dev, int new_mtu,
120 struct netlink_ext_ack *extack);
121
122int dev_get_phys_port_id(struct net_device *dev,
123 struct netdev_phys_item_id *ppid);
124int dev_get_phys_port_name(struct net_device *dev,
125 char *name, size_t len);
126
127int netif_change_proto_down(struct net_device *dev, bool proto_down);
128int dev_change_proto_down(struct net_device *dev, bool proto_down);
129void netdev_change_proto_down_reason_locked(struct net_device *dev,
130 unsigned long mask, u32 value);
131
132typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
133int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
134 int fd, int expected_fd, u32 flags);
135
136int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
137int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
138void netif_set_group(struct net_device *dev, int new_group);
139void dev_set_group(struct net_device *dev, int new_group);
140int netif_change_carrier(struct net_device *dev, bool new_carrier);
141int dev_change_carrier(struct net_device *dev, bool new_carrier);
142
143void __dev_set_rx_mode(struct net_device *dev);
144
145void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
146 unsigned int gchanges, u32 portid,
147 const struct nlmsghdr *nlh);
148
149void unregister_netdevice_many_notify(struct list_head *head,
150 u32 portid, const struct nlmsghdr *nlh);
151
152static inline void netif_set_up(struct net_device *dev, bool value)
153{
154 if (value)
155 dev->flags |= IFF_UP;
156 else
157 dev->flags &= ~IFF_UP;
158
159 if (!netdev_need_ops_lock(dev))
160 netdev_lock(dev);
161 dev->up = value;
162 if (!netdev_need_ops_lock(dev))
163 netdev_unlock(dev);
164}
165
166static inline void netif_set_gso_max_size(struct net_device *dev,
167 unsigned int size)
168{
169 /* dev->gso_max_size is read locklessly from sk_setup_caps() */
170 WRITE_ONCE(dev->gso_max_size, size);
171 if (size <= GSO_LEGACY_MAX_SIZE)
172 WRITE_ONCE(dev->gso_ipv4_max_size, size);
173}
174
175static inline void netif_set_gso_max_segs(struct net_device *dev,
176 unsigned int segs)
177{
178 /* dev->gso_max_segs is read locklessly from sk_setup_caps() */
179 WRITE_ONCE(dev->gso_max_segs, segs);
180}
181
182static inline void netif_set_gro_max_size(struct net_device *dev,
183 unsigned int size)
184{
185 /* This pairs with the READ_ONCE() in skb_gro_receive() */
186 WRITE_ONCE(dev->gro_max_size, size);
187 if (size <= GRO_LEGACY_MAX_SIZE)
188 WRITE_ONCE(dev->gro_ipv4_max_size, size);
189}
190
191static inline void netif_set_gso_ipv4_max_size(struct net_device *dev,
192 unsigned int size)
193{
194 /* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */
195 WRITE_ONCE(dev->gso_ipv4_max_size, size);
196}
197
198static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
199 unsigned int size)
200{
201 /* This pairs with the READ_ONCE() in skb_gro_receive() */
202 WRITE_ONCE(dev->gro_ipv4_max_size, size);
203}
204
205/**
206 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs
207 * @n: napi struct to get the defer_hard_irqs field from
208 *
209 * Return: the per-NAPI value of the defar_hard_irqs field.
210 */
211static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n)
212{
213 return READ_ONCE(n->defer_hard_irqs);
214}
215
216/**
217 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi
218 * @n: napi_struct to set the defer_hard_irqs field
219 * @defer: the value the field should be set to
220 */
221static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer)
222{
223 WRITE_ONCE(n->defer_hard_irqs, defer);
224}
225
226/**
227 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev
228 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set
229 * @defer: the defer_hard_irqs value to set
230 */
231static inline void netdev_set_defer_hard_irqs(struct net_device *netdev,
232 u32 defer)
233{
234 unsigned int count = max(netdev->num_rx_queues,
235 netdev->num_tx_queues);
236 struct napi_struct *napi;
237 int i;
238
239 WRITE_ONCE(netdev->napi_defer_hard_irqs, defer);
240 list_for_each_entry(napi, &netdev->napi_list, dev_list)
241 napi_set_defer_hard_irqs(n: napi, defer);
242
243 for (i = 0; i < count; i++)
244 netdev->napi_config[i].defer_hard_irqs = defer;
245}
246
247/**
248 * napi_get_gro_flush_timeout - get the gro_flush_timeout
249 * @n: napi struct to get the gro_flush_timeout from
250 *
251 * Return: the per-NAPI value of the gro_flush_timeout field.
252 */
253static inline unsigned long
254napi_get_gro_flush_timeout(const struct napi_struct *n)
255{
256 return READ_ONCE(n->gro_flush_timeout);
257}
258
259/**
260 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi
261 * @n: napi struct to set the gro_flush_timeout
262 * @timeout: timeout value to set
263 *
264 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout
265 */
266static inline void napi_set_gro_flush_timeout(struct napi_struct *n,
267 unsigned long timeout)
268{
269 WRITE_ONCE(n->gro_flush_timeout, timeout);
270}
271
272/**
273 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs
274 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set
275 * @timeout: the timeout value to set
276 */
277static inline void netdev_set_gro_flush_timeout(struct net_device *netdev,
278 unsigned long timeout)
279{
280 unsigned int count = max(netdev->num_rx_queues,
281 netdev->num_tx_queues);
282 struct napi_struct *napi;
283 int i;
284
285 WRITE_ONCE(netdev->gro_flush_timeout, timeout);
286 list_for_each_entry(napi, &netdev->napi_list, dev_list)
287 napi_set_gro_flush_timeout(n: napi, timeout);
288
289 for (i = 0; i < count; i++)
290 netdev->napi_config[i].gro_flush_timeout = timeout;
291}
292
293/**
294 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout
295 * @n: napi struct to get the irq_suspend_timeout from
296 *
297 * Return: the per-NAPI value of the irq_suspend_timeout field.
298 */
299static inline unsigned long
300napi_get_irq_suspend_timeout(const struct napi_struct *n)
301{
302 return READ_ONCE(n->irq_suspend_timeout);
303}
304
305/**
306 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi
307 * @n: napi struct to set the irq_suspend_timeout
308 * @timeout: timeout value to set
309 *
310 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout
311 */
312static inline void napi_set_irq_suspend_timeout(struct napi_struct *n,
313 unsigned long timeout)
314{
315 WRITE_ONCE(n->irq_suspend_timeout, timeout);
316}
317
318static inline enum netdev_napi_threaded napi_get_threaded(struct napi_struct *n)
319{
320 if (test_bit(NAPI_STATE_THREADED, &n->state))
321 return NETDEV_NAPI_THREADED_ENABLED;
322
323 return NETDEV_NAPI_THREADED_DISABLED;
324}
325
326static inline enum netdev_napi_threaded
327napi_get_threaded_config(struct net_device *dev, struct napi_struct *n)
328{
329 if (n->config)
330 return n->config->threaded;
331 return dev->threaded;
332}
333
334int napi_set_threaded(struct napi_struct *n,
335 enum netdev_napi_threaded threaded);
336
337int netif_set_threaded(struct net_device *dev,
338 enum netdev_napi_threaded threaded);
339
340int rps_cpumask_housekeeping(struct cpumask *mask);
341
342#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
343void xdp_do_check_flushed(struct napi_struct *napi);
344#else
345static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
346#endif
347
348/* Best effort check that NAPI is not idle (can't be scheduled to run) */
349static inline void napi_assert_will_not_race(const struct napi_struct *napi)
350{
351 /* uninitialized instance, can't race */
352 if (!napi->poll_list.next)
353 return;
354
355 /* SCHED bit is set on disabled instances */
356 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state));
357 WARN_ON(READ_ONCE(napi->list_owner) != -1);
358}
359
360void kick_defer_list_purge(unsigned int cpu);
361
362#define XMIT_RECURSION_LIMIT 8
363
364#ifndef CONFIG_PREEMPT_RT
365static inline bool dev_xmit_recursion(void)
366{
367 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
368 XMIT_RECURSION_LIMIT);
369}
370
371static inline void dev_xmit_recursion_inc(void)
372{
373 __this_cpu_inc(softnet_data.xmit.recursion);
374}
375
376static inline void dev_xmit_recursion_dec(void)
377{
378 __this_cpu_dec(softnet_data.xmit.recursion);
379}
380#else
381static inline bool dev_xmit_recursion(void)
382{
383 return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT);
384}
385
386static inline void dev_xmit_recursion_inc(void)
387{
388 current->net_xmit.recursion++;
389}
390
391static inline void dev_xmit_recursion_dec(void)
392{
393 current->net_xmit.recursion--;
394}
395#endif
396
397int dev_set_hwtstamp_phylib(struct net_device *dev,
398 struct kernel_hwtstamp_config *cfg,
399 struct netlink_ext_ack *extack);
400int dev_get_hwtstamp_phylib(struct net_device *dev,
401 struct kernel_hwtstamp_config *cfg);
402int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg);
403
404#endif
405