1// SPDX-License-Identifier: GPL-2.0-only
2/* Kernel thread helper functions.
3 * Copyright (C) 2004 IBM Corporation, Rusty Russell.
4 * Copyright (C) 2009 Red Hat, Inc.
5 *
6 * Creation is done via kthreadd, so that we get a clean environment
7 * even if we're invoked from userspace (think modprobe, hotplug cpu,
8 * etc.).
9 */
10#include <uapi/linux/sched/types.h>
11#include <linux/mm.h>
12#include <linux/mmu_context.h>
13#include <linux/sched.h>
14#include <linux/sched/mm.h>
15#include <linux/sched/task.h>
16#include <linux/kthread.h>
17#include <linux/completion.h>
18#include <linux/err.h>
19#include <linux/cgroup.h>
20#include <linux/cpuset.h>
21#include <linux/unistd.h>
22#include <linux/file.h>
23#include <linux/export.h>
24#include <linux/mutex.h>
25#include <linux/slab.h>
26#include <linux/freezer.h>
27#include <linux/ptrace.h>
28#include <linux/uaccess.h>
29#include <linux/numa.h>
30#include <linux/sched/isolation.h>
31#include <trace/events/sched.h>
32
33
34static DEFINE_SPINLOCK(kthread_create_lock);
35static LIST_HEAD(kthread_create_list);
36struct task_struct *kthreadd_task;
37
38static LIST_HEAD(kthreads_hotplug);
39static DEFINE_MUTEX(kthreads_hotplug_lock);
40
41struct kthread_create_info
42{
43 /* Information passed to kthread() from kthreadd. */
44 char *full_name;
45 int (*threadfn)(void *data);
46 void *data;
47 int node;
48
49 /* Result passed back to kthread_create() from kthreadd. */
50 struct task_struct *result;
51 struct completion *done;
52
53 struct list_head list;
54};
55
56struct kthread {
57 unsigned long flags;
58 unsigned int cpu;
59 unsigned int node;
60 int started;
61 int result;
62 int (*threadfn)(void *);
63 void *data;
64 struct completion parked;
65 struct completion exited;
66#ifdef CONFIG_BLK_CGROUP
67 struct cgroup_subsys_state *blkcg_css;
68#endif
69 /* To store the full name if task comm is truncated. */
70 char *full_name;
71 struct task_struct *task;
72 struct list_head hotplug_node;
73 struct cpumask *preferred_affinity;
74};
75
76enum KTHREAD_BITS {
77 KTHREAD_IS_PER_CPU = 0,
78 KTHREAD_SHOULD_STOP,
79 KTHREAD_SHOULD_PARK,
80};
81
82static inline struct kthread *to_kthread(struct task_struct *k)
83{
84 WARN_ON(!(k->flags & PF_KTHREAD));
85 return k->worker_private;
86}
87
88/*
89 * Variant of to_kthread() that doesn't assume @p is a kthread.
90 *
91 * When "(p->flags & PF_KTHREAD)" is set the task is a kthread and will
92 * always remain a kthread. For kthreads p->worker_private always
93 * points to a struct kthread. For tasks that are not kthreads
94 * p->worker_private is used to point to other things.
95 *
96 * Return NULL for any task that is not a kthread.
97 */
98static inline struct kthread *__to_kthread(struct task_struct *p)
99{
100 void *kthread = p->worker_private;
101 if (kthread && !(p->flags & PF_KTHREAD))
102 kthread = NULL;
103 return kthread;
104}
105
106void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk)
107{
108 struct kthread *kthread = to_kthread(k: tsk);
109
110 if (!kthread || !kthread->full_name) {
111 strscpy(buf, tsk->comm, buf_size);
112 return;
113 }
114
115 strscpy_pad(buf, kthread->full_name, buf_size);
116}
117
118bool set_kthread_struct(struct task_struct *p)
119{
120 struct kthread *kthread;
121
122 if (WARN_ON_ONCE(to_kthread(p)))
123 return false;
124
125 kthread = kzalloc(sizeof(*kthread), GFP_KERNEL);
126 if (!kthread)
127 return false;
128
129 init_completion(x: &kthread->exited);
130 init_completion(x: &kthread->parked);
131 INIT_LIST_HEAD(list: &kthread->hotplug_node);
132 p->vfork_done = &kthread->exited;
133
134 kthread->task = p;
135 kthread->node = tsk_fork_get_node(current);
136 p->worker_private = kthread;
137 return true;
138}
139
140void free_kthread_struct(struct task_struct *k)
141{
142 struct kthread *kthread;
143
144 /*
145 * Can be NULL if kmalloc() in set_kthread_struct() failed.
146 */
147 kthread = to_kthread(k);
148 if (!kthread)
149 return;
150
151#ifdef CONFIG_BLK_CGROUP
152 WARN_ON_ONCE(kthread->blkcg_css);
153#endif
154 k->worker_private = NULL;
155 kfree(objp: kthread->full_name);
156 kfree(objp: kthread);
157}
158
159/**
160 * kthread_should_stop - should this kthread return now?
161 *
162 * When someone calls kthread_stop() on your kthread, it will be woken
163 * and this will return true. You should then return, and your return
164 * value will be passed through to kthread_stop().
165 */
166bool kthread_should_stop(void)
167{
168 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
169}
170EXPORT_SYMBOL(kthread_should_stop);
171
172static bool __kthread_should_park(struct task_struct *k)
173{
174 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
175}
176
177/**
178 * kthread_should_park - should this kthread park now?
179 *
180 * When someone calls kthread_park() on your kthread, it will be woken
181 * and this will return true. You should then do the necessary
182 * cleanup and call kthread_parkme()
183 *
184 * Similar to kthread_should_stop(), but this keeps the thread alive
185 * and in a park position. kthread_unpark() "restarts" the thread and
186 * calls the thread function again.
187 */
188bool kthread_should_park(void)
189{
190 return __kthread_should_park(current);
191}
192EXPORT_SYMBOL_GPL(kthread_should_park);
193
194bool kthread_should_stop_or_park(void)
195{
196 struct kthread *kthread = __to_kthread(current);
197
198 if (!kthread)
199 return false;
200
201 return kthread->flags & (BIT(KTHREAD_SHOULD_STOP) | BIT(KTHREAD_SHOULD_PARK));
202}
203
204/**
205 * kthread_freezable_should_stop - should this freezable kthread return now?
206 * @was_frozen: optional out parameter, indicates whether %current was frozen
207 *
208 * kthread_should_stop() for freezable kthreads, which will enter
209 * refrigerator if necessary. This function is safe from kthread_stop() /
210 * freezer deadlock and freezable kthreads should use this function instead
211 * of calling try_to_freeze() directly.
212 */
213bool kthread_freezable_should_stop(bool *was_frozen)
214{
215 bool frozen = false;
216
217 might_sleep();
218
219 if (unlikely(freezing(current)))
220 frozen = __refrigerator(check_kthr_stop: true);
221
222 if (was_frozen)
223 *was_frozen = frozen;
224
225 return kthread_should_stop();
226}
227EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
228
229/**
230 * kthread_func - return the function specified on kthread creation
231 * @task: kthread task in question
232 *
233 * Returns NULL if the task is not a kthread.
234 */
235void *kthread_func(struct task_struct *task)
236{
237 struct kthread *kthread = __to_kthread(p: task);
238 if (kthread)
239 return kthread->threadfn;
240 return NULL;
241}
242EXPORT_SYMBOL_GPL(kthread_func);
243
244/**
245 * kthread_data - return data value specified on kthread creation
246 * @task: kthread task in question
247 *
248 * Return the data value specified when kthread @task was created.
249 * The caller is responsible for ensuring the validity of @task when
250 * calling this function.
251 */
252void *kthread_data(struct task_struct *task)
253{
254 return to_kthread(k: task)->data;
255}
256EXPORT_SYMBOL_GPL(kthread_data);
257
258/**
259 * kthread_probe_data - speculative version of kthread_data()
260 * @task: possible kthread task in question
261 *
262 * @task could be a kthread task. Return the data value specified when it
263 * was created if accessible. If @task isn't a kthread task or its data is
264 * inaccessible for any reason, %NULL is returned. This function requires
265 * that @task itself is safe to dereference.
266 */
267void *kthread_probe_data(struct task_struct *task)
268{
269 struct kthread *kthread = __to_kthread(p: task);
270 void *data = NULL;
271
272 if (kthread)
273 copy_from_kernel_nofault(dst: &data, src: &kthread->data, size: sizeof(data));
274 return data;
275}
276
277static void __kthread_parkme(struct kthread *self)
278{
279 for (;;) {
280 /*
281 * TASK_PARKED is a special state; we must serialize against
282 * possible pending wakeups to avoid store-store collisions on
283 * task->state.
284 *
285 * Such a collision might possibly result in the task state
286 * changin from TASK_PARKED and us failing the
287 * wait_task_inactive() in kthread_park().
288 */
289 set_special_state(TASK_PARKED);
290 if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
291 break;
292
293 /*
294 * Thread is going to call schedule(), do not preempt it,
295 * or the caller of kthread_park() may spend more time in
296 * wait_task_inactive().
297 */
298 preempt_disable();
299 complete(&self->parked);
300 schedule_preempt_disabled();
301 preempt_enable();
302 }
303 __set_current_state(TASK_RUNNING);
304}
305
306void kthread_parkme(void)
307{
308 __kthread_parkme(self: to_kthread(current));
309}
310EXPORT_SYMBOL_GPL(kthread_parkme);
311
312/**
313 * kthread_exit - Cause the current kthread return @result to kthread_stop().
314 * @result: The integer value to return to kthread_stop().
315 *
316 * While kthread_exit can be called directly, it exists so that
317 * functions which do some additional work in non-modular code such as
318 * module_put_and_kthread_exit can be implemented.
319 *
320 * Does not return.
321 */
322void __noreturn kthread_exit(long result)
323{
324 struct kthread *kthread = to_kthread(current);
325 kthread->result = result;
326 if (!list_empty(head: &kthread->hotplug_node)) {
327 mutex_lock(lock: &kthreads_hotplug_lock);
328 list_del(entry: &kthread->hotplug_node);
329 mutex_unlock(lock: &kthreads_hotplug_lock);
330
331 if (kthread->preferred_affinity) {
332 kfree(objp: kthread->preferred_affinity);
333 kthread->preferred_affinity = NULL;
334 }
335 }
336 do_exit(error_code: 0);
337}
338EXPORT_SYMBOL(kthread_exit);
339
340/**
341 * kthread_complete_and_exit - Exit the current kthread.
342 * @comp: Completion to complete
343 * @code: The integer value to return to kthread_stop().
344 *
345 * If present, complete @comp and then return code to kthread_stop().
346 *
347 * A kernel thread whose module may be removed after the completion of
348 * @comp can use this function to exit safely.
349 *
350 * Does not return.
351 */
352void __noreturn kthread_complete_and_exit(struct completion *comp, long code)
353{
354 if (comp)
355 complete(comp);
356
357 kthread_exit(code);
358}
359EXPORT_SYMBOL(kthread_complete_and_exit);
360
361static void kthread_fetch_affinity(struct kthread *kthread, struct cpumask *cpumask)
362{
363 const struct cpumask *pref;
364
365 if (kthread->preferred_affinity) {
366 pref = kthread->preferred_affinity;
367 } else {
368 if (WARN_ON_ONCE(kthread->node == NUMA_NO_NODE))
369 return;
370 pref = cpumask_of_node(node: kthread->node);
371 }
372
373 cpumask_and(dstp: cpumask, src1p: pref, src2p: housekeeping_cpumask(type: HK_TYPE_KTHREAD));
374 if (cpumask_empty(srcp: cpumask))
375 cpumask_copy(dstp: cpumask, srcp: housekeeping_cpumask(type: HK_TYPE_KTHREAD));
376}
377
378static void kthread_affine_node(void)
379{
380 struct kthread *kthread = to_kthread(current);
381 cpumask_var_t affinity;
382
383 WARN_ON_ONCE(kthread_is_per_cpu(current));
384
385 if (kthread->node == NUMA_NO_NODE) {
386 housekeeping_affine(current, type: HK_TYPE_KTHREAD);
387 } else {
388 if (!zalloc_cpumask_var(mask: &affinity, GFP_KERNEL)) {
389 WARN_ON_ONCE(1);
390 return;
391 }
392
393 mutex_lock(lock: &kthreads_hotplug_lock);
394 WARN_ON_ONCE(!list_empty(&kthread->hotplug_node));
395 list_add_tail(new: &kthread->hotplug_node, head: &kthreads_hotplug);
396 /*
397 * The node cpumask is racy when read from kthread() but:
398 * - a racing CPU going down will either fail on the subsequent
399 * call to set_cpus_allowed_ptr() or be migrated to housekeepers
400 * afterwards by the scheduler.
401 * - a racing CPU going up will be handled by kthreads_online_cpu()
402 */
403 kthread_fetch_affinity(kthread, cpumask: affinity);
404 set_cpus_allowed_ptr(current, new_mask: affinity);
405 mutex_unlock(lock: &kthreads_hotplug_lock);
406
407 free_cpumask_var(mask: affinity);
408 }
409}
410
411static int kthread(void *_create)
412{
413 static const struct sched_param param = { .sched_priority = 0 };
414 /* Copy data: it's on kthread's stack */
415 struct kthread_create_info *create = _create;
416 int (*threadfn)(void *data) = create->threadfn;
417 void *data = create->data;
418 struct completion *done;
419 struct kthread *self;
420 int ret;
421
422 self = to_kthread(current);
423
424 /* Release the structure when caller killed by a fatal signal. */
425 done = xchg(&create->done, NULL);
426 if (!done) {
427 kfree(objp: create->full_name);
428 kfree(objp: create);
429 kthread_exit(-EINTR);
430 }
431
432 self->full_name = create->full_name;
433 self->threadfn = threadfn;
434 self->data = data;
435
436 /*
437 * The new thread inherited kthreadd's priority and CPU mask. Reset
438 * back to default in case they have been changed.
439 */
440 sched_setscheduler_nocheck(current, SCHED_NORMAL, &param);
441
442 /* OK, tell user we're spawned, wait for stop or wakeup */
443 __set_current_state(TASK_UNINTERRUPTIBLE);
444 create->result = current;
445 /*
446 * Thread is going to call schedule(), do not preempt it,
447 * or the creator may spend more time in wait_task_inactive().
448 */
449 preempt_disable();
450 complete(done);
451 schedule_preempt_disabled();
452 preempt_enable();
453
454 self->started = 1;
455
456 if (!(current->flags & PF_NO_SETAFFINITY) && !self->preferred_affinity)
457 kthread_affine_node();
458
459 ret = -EINTR;
460 if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
461 cgroup_kthread_ready();
462 __kthread_parkme(self);
463 ret = threadfn(data);
464 }
465 kthread_exit(ret);
466}
467
468/* called from kernel_clone() to get node information for about to be created task */
469int tsk_fork_get_node(struct task_struct *tsk)
470{
471#ifdef CONFIG_NUMA
472 if (tsk == kthreadd_task)
473 return tsk->pref_node_fork;
474#endif
475 return NUMA_NO_NODE;
476}
477
478static void create_kthread(struct kthread_create_info *create)
479{
480 int pid;
481
482#ifdef CONFIG_NUMA
483 current->pref_node_fork = create->node;
484#endif
485 /* We want our own signal handler (we take no signals by default). */
486 pid = kernel_thread(fn: kthread, arg: create, name: create->full_name,
487 CLONE_FS | CLONE_FILES | SIGCHLD);
488 if (pid < 0) {
489 /* Release the structure when caller killed by a fatal signal. */
490 struct completion *done = xchg(&create->done, NULL);
491
492 kfree(objp: create->full_name);
493 if (!done) {
494 kfree(objp: create);
495 return;
496 }
497 create->result = ERR_PTR(error: pid);
498 complete(done);
499 }
500}
501
502static __printf(4, 0)
503struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
504 void *data, int node,
505 const char namefmt[],
506 va_list args)
507{
508 DECLARE_COMPLETION_ONSTACK(done);
509 struct task_struct *task;
510 struct kthread_create_info *create = kmalloc(sizeof(*create),
511 GFP_KERNEL);
512
513 if (!create)
514 return ERR_PTR(error: -ENOMEM);
515 create->threadfn = threadfn;
516 create->data = data;
517 create->node = node;
518 create->done = &done;
519 create->full_name = kvasprintf(GFP_KERNEL, fmt: namefmt, args);
520 if (!create->full_name) {
521 task = ERR_PTR(error: -ENOMEM);
522 goto free_create;
523 }
524
525 spin_lock(lock: &kthread_create_lock);
526 list_add_tail(new: &create->list, head: &kthread_create_list);
527 spin_unlock(lock: &kthread_create_lock);
528
529 wake_up_process(tsk: kthreadd_task);
530 /*
531 * Wait for completion in killable state, for I might be chosen by
532 * the OOM killer while kthreadd is trying to allocate memory for
533 * new kernel thread.
534 */
535 if (unlikely(wait_for_completion_killable(&done))) {
536 /*
537 * If I was killed by a fatal signal before kthreadd (or new
538 * kernel thread) calls complete(), leave the cleanup of this
539 * structure to that thread.
540 */
541 if (xchg(&create->done, NULL))
542 return ERR_PTR(error: -EINTR);
543 /*
544 * kthreadd (or new kernel thread) will call complete()
545 * shortly.
546 */
547 wait_for_completion(&done);
548 }
549 task = create->result;
550free_create:
551 kfree(objp: create);
552 return task;
553}
554
555/**
556 * kthread_create_on_node - create a kthread.
557 * @threadfn: the function to run until signal_pending(current).
558 * @data: data ptr for @threadfn.
559 * @node: task and thread structures for the thread are allocated on this node
560 * @namefmt: printf-style name for the thread.
561 *
562 * Description: This helper function creates and names a kernel
563 * thread. The thread will be stopped: use wake_up_process() to start
564 * it. See also kthread_run(). The new thread has SCHED_NORMAL policy and
565 * is affine to all CPUs.
566 *
567 * If thread is going to be bound on a particular cpu, give its node
568 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
569 * When woken, the thread will run @threadfn() with @data as its
570 * argument. @threadfn() can either return directly if it is a
571 * standalone thread for which no one will call kthread_stop(), or
572 * return when 'kthread_should_stop()' is true (which means
573 * kthread_stop() has been called). The return value should be zero
574 * or a negative error number; it will be passed to kthread_stop().
575 *
576 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
577 */
578struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
579 void *data, int node,
580 const char namefmt[],
581 ...)
582{
583 struct task_struct *task;
584 va_list args;
585
586 va_start(args, namefmt);
587 task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
588 va_end(args);
589
590 return task;
591}
592EXPORT_SYMBOL(kthread_create_on_node);
593
594static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, unsigned int state)
595{
596 unsigned long flags;
597
598 if (!wait_task_inactive(p, match_state: state)) {
599 WARN_ON(1);
600 return;
601 }
602
603 /* It's safe because the task is inactive. */
604 raw_spin_lock_irqsave(&p->pi_lock, flags);
605 do_set_cpus_allowed(p, new_mask: mask);
606 p->flags |= PF_NO_SETAFFINITY;
607 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
608}
609
610static void __kthread_bind(struct task_struct *p, unsigned int cpu, unsigned int state)
611{
612 __kthread_bind_mask(p, cpumask_of(cpu), state);
613}
614
615void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
616{
617 struct kthread *kthread = to_kthread(k: p);
618 __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
619 WARN_ON_ONCE(kthread->started);
620}
621
622/**
623 * kthread_bind - bind a just-created kthread to a cpu.
624 * @p: thread created by kthread_create().
625 * @cpu: cpu (might not be online, must be possible) for @k to run on.
626 *
627 * Description: This function is equivalent to set_cpus_allowed(),
628 * except that @cpu doesn't need to be online, and the thread must be
629 * stopped (i.e., just returned from kthread_create()).
630 */
631void kthread_bind(struct task_struct *p, unsigned int cpu)
632{
633 struct kthread *kthread = to_kthread(k: p);
634 __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
635 WARN_ON_ONCE(kthread->started);
636}
637EXPORT_SYMBOL(kthread_bind);
638
639/**
640 * kthread_create_on_cpu - Create a cpu bound kthread
641 * @threadfn: the function to run until signal_pending(current).
642 * @data: data ptr for @threadfn.
643 * @cpu: The cpu on which the thread should be bound,
644 * @namefmt: printf-style name for the thread. Format is restricted
645 * to "name.*%u". Code fills in cpu number.
646 *
647 * Description: This helper function creates and names a kernel thread
648 */
649struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
650 void *data, unsigned int cpu,
651 const char *namefmt)
652{
653 struct task_struct *p;
654
655 p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
656 cpu);
657 if (IS_ERR(ptr: p))
658 return p;
659 kthread_bind(p, cpu);
660 /* CPU hotplug need to bind once again when unparking the thread. */
661 to_kthread(k: p)->cpu = cpu;
662 return p;
663}
664EXPORT_SYMBOL(kthread_create_on_cpu);
665
666void kthread_set_per_cpu(struct task_struct *k, int cpu)
667{
668 struct kthread *kthread = to_kthread(k);
669 if (!kthread)
670 return;
671
672 WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY));
673
674 if (cpu < 0) {
675 clear_bit(nr: KTHREAD_IS_PER_CPU, addr: &kthread->flags);
676 return;
677 }
678
679 kthread->cpu = cpu;
680 set_bit(nr: KTHREAD_IS_PER_CPU, addr: &kthread->flags);
681}
682
683bool kthread_is_per_cpu(struct task_struct *p)
684{
685 struct kthread *kthread = __to_kthread(p);
686 if (!kthread)
687 return false;
688
689 return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
690}
691
692/**
693 * kthread_unpark - unpark a thread created by kthread_create().
694 * @k: thread created by kthread_create().
695 *
696 * Sets kthread_should_park() for @k to return false, wakes it, and
697 * waits for it to return. If the thread is marked percpu then its
698 * bound to the cpu again.
699 */
700void kthread_unpark(struct task_struct *k)
701{
702 struct kthread *kthread = to_kthread(k);
703
704 if (!test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))
705 return;
706 /*
707 * Newly created kthread was parked when the CPU was offline.
708 * The binding was lost and we need to set it again.
709 */
710 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
711 __kthread_bind(p: k, cpu: kthread->cpu, TASK_PARKED);
712
713 clear_bit(nr: KTHREAD_SHOULD_PARK, addr: &kthread->flags);
714 /*
715 * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup.
716 */
717 wake_up_state(tsk: k, TASK_PARKED);
718}
719EXPORT_SYMBOL_GPL(kthread_unpark);
720
721/**
722 * kthread_park - park a thread created by kthread_create().
723 * @k: thread created by kthread_create().
724 *
725 * Sets kthread_should_park() for @k to return true, wakes it, and
726 * waits for it to return. This can also be called after kthread_create()
727 * instead of calling wake_up_process(): the thread will park without
728 * calling threadfn().
729 *
730 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
731 * If called by the kthread itself just the park bit is set.
732 */
733int kthread_park(struct task_struct *k)
734{
735 struct kthread *kthread = to_kthread(k);
736
737 if (WARN_ON(k->flags & PF_EXITING))
738 return -ENOSYS;
739
740 if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)))
741 return -EBUSY;
742
743 set_bit(nr: KTHREAD_SHOULD_PARK, addr: &kthread->flags);
744 if (k != current) {
745 wake_up_process(tsk: k);
746 /*
747 * Wait for __kthread_parkme() to complete(), this means we
748 * _will_ have TASK_PARKED and are about to call schedule().
749 */
750 wait_for_completion(&kthread->parked);
751 /*
752 * Now wait for that schedule() to complete and the task to
753 * get scheduled out.
754 */
755 WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED));
756 }
757
758 return 0;
759}
760EXPORT_SYMBOL_GPL(kthread_park);
761
762/**
763 * kthread_stop - stop a thread created by kthread_create().
764 * @k: thread created by kthread_create().
765 *
766 * Sets kthread_should_stop() for @k to return true, wakes it, and
767 * waits for it to exit. This can also be called after kthread_create()
768 * instead of calling wake_up_process(): the thread will exit without
769 * calling threadfn().
770 *
771 * If threadfn() may call kthread_exit() itself, the caller must ensure
772 * task_struct can't go away.
773 *
774 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
775 * was never called.
776 */
777int kthread_stop(struct task_struct *k)
778{
779 struct kthread *kthread;
780 int ret;
781
782 trace_sched_kthread_stop(t: k);
783
784 get_task_struct(t: k);
785 kthread = to_kthread(k);
786 set_bit(nr: KTHREAD_SHOULD_STOP, addr: &kthread->flags);
787 kthread_unpark(k);
788 set_tsk_thread_flag(tsk: k, TIF_NOTIFY_SIGNAL);
789 wake_up_process(tsk: k);
790 wait_for_completion(&kthread->exited);
791 ret = kthread->result;
792 put_task_struct(t: k);
793
794 trace_sched_kthread_stop_ret(ret);
795 return ret;
796}
797EXPORT_SYMBOL(kthread_stop);
798
799/**
800 * kthread_stop_put - stop a thread and put its task struct
801 * @k: thread created by kthread_create().
802 *
803 * Stops a thread created by kthread_create() and put its task_struct.
804 * Only use when holding an extra task struct reference obtained by
805 * calling get_task_struct().
806 */
807int kthread_stop_put(struct task_struct *k)
808{
809 int ret;
810
811 ret = kthread_stop(k);
812 put_task_struct(t: k);
813 return ret;
814}
815EXPORT_SYMBOL(kthread_stop_put);
816
817int kthreadd(void *unused)
818{
819 static const char comm[TASK_COMM_LEN] = "kthreadd";
820 struct task_struct *tsk = current;
821
822 /* Setup a clean context for our children to inherit. */
823 set_task_comm(tsk, comm);
824 ignore_signals(tsk);
825 set_cpus_allowed_ptr(p: tsk, new_mask: housekeeping_cpumask(type: HK_TYPE_KTHREAD));
826 set_mems_allowed(node_states[N_MEMORY]);
827
828 current->flags |= PF_NOFREEZE;
829 cgroup_init_kthreadd();
830
831 for (;;) {
832 set_current_state(TASK_INTERRUPTIBLE);
833 if (list_empty(head: &kthread_create_list))
834 schedule();
835 __set_current_state(TASK_RUNNING);
836
837 spin_lock(lock: &kthread_create_lock);
838 while (!list_empty(head: &kthread_create_list)) {
839 struct kthread_create_info *create;
840
841 create = list_entry(kthread_create_list.next,
842 struct kthread_create_info, list);
843 list_del_init(entry: &create->list);
844 spin_unlock(lock: &kthread_create_lock);
845
846 create_kthread(create);
847
848 spin_lock(lock: &kthread_create_lock);
849 }
850 spin_unlock(lock: &kthread_create_lock);
851 }
852
853 return 0;
854}
855
856int kthread_affine_preferred(struct task_struct *p, const struct cpumask *mask)
857{
858 struct kthread *kthread = to_kthread(k: p);
859 cpumask_var_t affinity;
860 unsigned long flags;
861 int ret = 0;
862
863 if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE) || kthread->started) {
864 WARN_ON(1);
865 return -EINVAL;
866 }
867
868 WARN_ON_ONCE(kthread->preferred_affinity);
869
870 if (!zalloc_cpumask_var(mask: &affinity, GFP_KERNEL))
871 return -ENOMEM;
872
873 kthread->preferred_affinity = kzalloc(sizeof(struct cpumask), GFP_KERNEL);
874 if (!kthread->preferred_affinity) {
875 ret = -ENOMEM;
876 goto out;
877 }
878
879 mutex_lock(lock: &kthreads_hotplug_lock);
880 cpumask_copy(dstp: kthread->preferred_affinity, srcp: mask);
881 WARN_ON_ONCE(!list_empty(&kthread->hotplug_node));
882 list_add_tail(new: &kthread->hotplug_node, head: &kthreads_hotplug);
883 kthread_fetch_affinity(kthread, cpumask: affinity);
884
885 /* It's safe because the task is inactive. */
886 raw_spin_lock_irqsave(&p->pi_lock, flags);
887 do_set_cpus_allowed(p, new_mask: affinity);
888 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
889
890 mutex_unlock(lock: &kthreads_hotplug_lock);
891out:
892 free_cpumask_var(mask: affinity);
893
894 return ret;
895}
896EXPORT_SYMBOL_GPL(kthread_affine_preferred);
897
898/*
899 * Re-affine kthreads according to their preferences
900 * and the newly online CPU. The CPU down part is handled
901 * by select_fallback_rq() which default re-affines to
902 * housekeepers from other nodes in case the preferred
903 * affinity doesn't apply anymore.
904 */
905static int kthreads_online_cpu(unsigned int cpu)
906{
907 cpumask_var_t affinity;
908 struct kthread *k;
909 int ret;
910
911 guard(mutex)(T: &kthreads_hotplug_lock);
912
913 if (list_empty(head: &kthreads_hotplug))
914 return 0;
915
916 if (!zalloc_cpumask_var(mask: &affinity, GFP_KERNEL))
917 return -ENOMEM;
918
919 ret = 0;
920
921 list_for_each_entry(k, &kthreads_hotplug, hotplug_node) {
922 if (WARN_ON_ONCE((k->task->flags & PF_NO_SETAFFINITY) ||
923 kthread_is_per_cpu(k->task))) {
924 ret = -EINVAL;
925 continue;
926 }
927 kthread_fetch_affinity(kthread: k, cpumask: affinity);
928 set_cpus_allowed_ptr(p: k->task, new_mask: affinity);
929 }
930
931 free_cpumask_var(mask: affinity);
932
933 return ret;
934}
935
936static int kthreads_init(void)
937{
938 return cpuhp_setup_state(state: CPUHP_AP_KTHREADS_ONLINE, name: "kthreads:online",
939 startup: kthreads_online_cpu, NULL);
940}
941early_initcall(kthreads_init);
942
943void __kthread_init_worker(struct kthread_worker *worker,
944 const char *name,
945 struct lock_class_key *key)
946{
947 memset(s: worker, c: 0, n: sizeof(struct kthread_worker));
948 raw_spin_lock_init(&worker->lock);
949 lockdep_set_class_and_name(&worker->lock, key, name);
950 INIT_LIST_HEAD(list: &worker->work_list);
951 INIT_LIST_HEAD(list: &worker->delayed_work_list);
952}
953EXPORT_SYMBOL_GPL(__kthread_init_worker);
954
955/**
956 * kthread_worker_fn - kthread function to process kthread_worker
957 * @worker_ptr: pointer to initialized kthread_worker
958 *
959 * This function implements the main cycle of kthread worker. It processes
960 * work_list until it is stopped with kthread_stop(). It sleeps when the queue
961 * is empty.
962 *
963 * The works are not allowed to keep any locks, disable preemption or interrupts
964 * when they finish. There is defined a safe point for freezing when one work
965 * finishes and before a new one is started.
966 *
967 * Also the works must not be handled by more than one worker at the same time,
968 * see also kthread_queue_work().
969 */
970int kthread_worker_fn(void *worker_ptr)
971{
972 struct kthread_worker *worker = worker_ptr;
973 struct kthread_work *work;
974
975 /*
976 * FIXME: Update the check and remove the assignment when all kthread
977 * worker users are created using kthread_create_worker*() functions.
978 */
979 WARN_ON(worker->task && worker->task != current);
980 worker->task = current;
981
982 if (worker->flags & KTW_FREEZABLE)
983 set_freezable();
984
985repeat:
986 set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */
987
988 if (kthread_should_stop()) {
989 __set_current_state(TASK_RUNNING);
990 raw_spin_lock_irq(&worker->lock);
991 worker->task = NULL;
992 raw_spin_unlock_irq(&worker->lock);
993 return 0;
994 }
995
996 work = NULL;
997 raw_spin_lock_irq(&worker->lock);
998 if (!list_empty(head: &worker->work_list)) {
999 work = list_first_entry(&worker->work_list,
1000 struct kthread_work, node);
1001 list_del_init(entry: &work->node);
1002 }
1003 worker->current_work = work;
1004 raw_spin_unlock_irq(&worker->lock);
1005
1006 if (work) {
1007 kthread_work_func_t func = work->func;
1008 __set_current_state(TASK_RUNNING);
1009 trace_sched_kthread_work_execute_start(work);
1010 work->func(work);
1011 /*
1012 * Avoid dereferencing work after this point. The trace
1013 * event only cares about the address.
1014 */
1015 trace_sched_kthread_work_execute_end(work, function: func);
1016 } else if (!freezing(current)) {
1017 schedule();
1018 } else {
1019 /*
1020 * Handle the case where the current remains
1021 * TASK_INTERRUPTIBLE. try_to_freeze() expects
1022 * the current to be TASK_RUNNING.
1023 */
1024 __set_current_state(TASK_RUNNING);
1025 }
1026
1027 try_to_freeze();
1028 cond_resched();
1029 goto repeat;
1030}
1031EXPORT_SYMBOL_GPL(kthread_worker_fn);
1032
1033static __printf(3, 0) struct kthread_worker *
1034__kthread_create_worker_on_node(unsigned int flags, int node,
1035 const char namefmt[], va_list args)
1036{
1037 struct kthread_worker *worker;
1038 struct task_struct *task;
1039
1040 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1041 if (!worker)
1042 return ERR_PTR(error: -ENOMEM);
1043
1044 kthread_init_worker(worker);
1045
1046 task = __kthread_create_on_node(threadfn: kthread_worker_fn, data: worker,
1047 node, namefmt, args);
1048 if (IS_ERR(ptr: task))
1049 goto fail_task;
1050
1051 worker->flags = flags;
1052 worker->task = task;
1053
1054 return worker;
1055
1056fail_task:
1057 kfree(objp: worker);
1058 return ERR_CAST(ptr: task);
1059}
1060
1061/**
1062 * kthread_create_worker_on_node - create a kthread worker
1063 * @flags: flags modifying the default behavior of the worker
1064 * @node: task structure for the thread is allocated on this node
1065 * @namefmt: printf-style name for the kthread worker (task).
1066 *
1067 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
1068 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
1069 * when the caller was killed by a fatal signal.
1070 */
1071struct kthread_worker *
1072kthread_create_worker_on_node(unsigned int flags, int node, const char namefmt[], ...)
1073{
1074 struct kthread_worker *worker;
1075 va_list args;
1076
1077 va_start(args, namefmt);
1078 worker = __kthread_create_worker_on_node(flags, node, namefmt, args);
1079 va_end(args);
1080
1081 return worker;
1082}
1083EXPORT_SYMBOL(kthread_create_worker_on_node);
1084
1085/**
1086 * kthread_create_worker_on_cpu - create a kthread worker and bind it
1087 * to a given CPU and the associated NUMA node.
1088 * @cpu: CPU number
1089 * @flags: flags modifying the default behavior of the worker
1090 * @namefmt: printf-style name for the thread. Format is restricted
1091 * to "name.*%u". Code fills in cpu number.
1092 *
1093 * Use a valid CPU number if you want to bind the kthread worker
1094 * to the given CPU and the associated NUMA node.
1095 *
1096 * A good practice is to add the cpu number also into the worker name.
1097 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
1098 *
1099 * CPU hotplug:
1100 * The kthread worker API is simple and generic. It just provides a way
1101 * to create, use, and destroy workers.
1102 *
1103 * It is up to the API user how to handle CPU hotplug. They have to decide
1104 * how to handle pending work items, prevent queuing new ones, and
1105 * restore the functionality when the CPU goes off and on. There are a
1106 * few catches:
1107 *
1108 * - CPU affinity gets lost when it is scheduled on an offline CPU.
1109 *
1110 * - The worker might not exist when the CPU was off when the user
1111 * created the workers.
1112 *
1113 * Good practice is to implement two CPU hotplug callbacks and to
1114 * destroy/create the worker when the CPU goes down/up.
1115 *
1116 * Return:
1117 * The pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
1118 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
1119 * when the caller was killed by a fatal signal.
1120 */
1121struct kthread_worker *
1122kthread_create_worker_on_cpu(int cpu, unsigned int flags,
1123 const char namefmt[])
1124{
1125 struct kthread_worker *worker;
1126
1127 worker = kthread_create_worker_on_node(flags, cpu_to_node(cpu), namefmt, cpu);
1128 if (!IS_ERR(ptr: worker))
1129 kthread_bind(worker->task, cpu);
1130
1131 return worker;
1132}
1133EXPORT_SYMBOL(kthread_create_worker_on_cpu);
1134
1135/*
1136 * Returns true when the work could not be queued at the moment.
1137 * It happens when it is already pending in a worker list
1138 * or when it is being cancelled.
1139 */
1140static inline bool queuing_blocked(struct kthread_worker *worker,
1141 struct kthread_work *work)
1142{
1143 lockdep_assert_held(&worker->lock);
1144
1145 return !list_empty(head: &work->node) || work->canceling;
1146}
1147
1148static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
1149 struct kthread_work *work)
1150{
1151 lockdep_assert_held(&worker->lock);
1152 WARN_ON_ONCE(!list_empty(&work->node));
1153 /* Do not use a work with >1 worker, see kthread_queue_work() */
1154 WARN_ON_ONCE(work->worker && work->worker != worker);
1155}
1156
1157/* insert @work before @pos in @worker */
1158static void kthread_insert_work(struct kthread_worker *worker,
1159 struct kthread_work *work,
1160 struct list_head *pos)
1161{
1162 kthread_insert_work_sanity_check(worker, work);
1163
1164 trace_sched_kthread_work_queue_work(worker, work);
1165
1166 list_add_tail(new: &work->node, head: pos);
1167 work->worker = worker;
1168 if (!worker->current_work && likely(worker->task))
1169 wake_up_process(tsk: worker->task);
1170}
1171
1172/**
1173 * kthread_queue_work - queue a kthread_work
1174 * @worker: target kthread_worker
1175 * @work: kthread_work to queue
1176 *
1177 * Queue @work to work processor @task for async execution. @task
1178 * must have been created with kthread_create_worker(). Returns %true
1179 * if @work was successfully queued, %false if it was already pending.
1180 *
1181 * Reinitialize the work if it needs to be used by another worker.
1182 * For example, when the worker was stopped and started again.
1183 */
1184bool kthread_queue_work(struct kthread_worker *worker,
1185 struct kthread_work *work)
1186{
1187 bool ret = false;
1188 unsigned long flags;
1189
1190 raw_spin_lock_irqsave(&worker->lock, flags);
1191 if (!queuing_blocked(worker, work)) {
1192 kthread_insert_work(worker, work, pos: &worker->work_list);
1193 ret = true;
1194 }
1195 raw_spin_unlock_irqrestore(&worker->lock, flags);
1196 return ret;
1197}
1198EXPORT_SYMBOL_GPL(kthread_queue_work);
1199
1200/**
1201 * kthread_delayed_work_timer_fn - callback that queues the associated kthread
1202 * delayed work when the timer expires.
1203 * @t: pointer to the expired timer
1204 *
1205 * The format of the function is defined by struct timer_list.
1206 * It should have been called from irqsafe timer with irq already off.
1207 */
1208void kthread_delayed_work_timer_fn(struct timer_list *t)
1209{
1210 struct kthread_delayed_work *dwork = timer_container_of(dwork, t,
1211 timer);
1212 struct kthread_work *work = &dwork->work;
1213 struct kthread_worker *worker = work->worker;
1214 unsigned long flags;
1215
1216 /*
1217 * This might happen when a pending work is reinitialized.
1218 * It means that it is used a wrong way.
1219 */
1220 if (WARN_ON_ONCE(!worker))
1221 return;
1222
1223 raw_spin_lock_irqsave(&worker->lock, flags);
1224 /* Work must not be used with >1 worker, see kthread_queue_work(). */
1225 WARN_ON_ONCE(work->worker != worker);
1226
1227 /* Move the work from worker->delayed_work_list. */
1228 WARN_ON_ONCE(list_empty(&work->node));
1229 list_del_init(entry: &work->node);
1230 if (!work->canceling)
1231 kthread_insert_work(worker, work, pos: &worker->work_list);
1232
1233 raw_spin_unlock_irqrestore(&worker->lock, flags);
1234}
1235EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
1236
1237static void __kthread_queue_delayed_work(struct kthread_worker *worker,
1238 struct kthread_delayed_work *dwork,
1239 unsigned long delay)
1240{
1241 struct timer_list *timer = &dwork->timer;
1242 struct kthread_work *work = &dwork->work;
1243
1244 WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
1245
1246 /*
1247 * If @delay is 0, queue @dwork->work immediately. This is for
1248 * both optimization and correctness. The earliest @timer can
1249 * expire is on the closest next tick and delayed_work users depend
1250 * on that there's no such delay when @delay is 0.
1251 */
1252 if (!delay) {
1253 kthread_insert_work(worker, work, pos: &worker->work_list);
1254 return;
1255 }
1256
1257 /* Be paranoid and try to detect possible races already now. */
1258 kthread_insert_work_sanity_check(worker, work);
1259
1260 list_add(new: &work->node, head: &worker->delayed_work_list);
1261 work->worker = worker;
1262 timer->expires = jiffies + delay;
1263 add_timer(timer);
1264}
1265
1266/**
1267 * kthread_queue_delayed_work - queue the associated kthread work
1268 * after a delay.
1269 * @worker: target kthread_worker
1270 * @dwork: kthread_delayed_work to queue
1271 * @delay: number of jiffies to wait before queuing
1272 *
1273 * If the work has not been pending it starts a timer that will queue
1274 * the work after the given @delay. If @delay is zero, it queues the
1275 * work immediately.
1276 *
1277 * Return: %false if the @work has already been pending. It means that
1278 * either the timer was running or the work was queued. It returns %true
1279 * otherwise.
1280 */
1281bool kthread_queue_delayed_work(struct kthread_worker *worker,
1282 struct kthread_delayed_work *dwork,
1283 unsigned long delay)
1284{
1285 struct kthread_work *work = &dwork->work;
1286 unsigned long flags;
1287 bool ret = false;
1288
1289 raw_spin_lock_irqsave(&worker->lock, flags);
1290
1291 if (!queuing_blocked(worker, work)) {
1292 __kthread_queue_delayed_work(worker, dwork, delay);
1293 ret = true;
1294 }
1295
1296 raw_spin_unlock_irqrestore(&worker->lock, flags);
1297 return ret;
1298}
1299EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
1300
1301struct kthread_flush_work {
1302 struct kthread_work work;
1303 struct completion done;
1304};
1305
1306static void kthread_flush_work_fn(struct kthread_work *work)
1307{
1308 struct kthread_flush_work *fwork =
1309 container_of(work, struct kthread_flush_work, work);
1310 complete(&fwork->done);
1311}
1312
1313/**
1314 * kthread_flush_work - flush a kthread_work
1315 * @work: work to flush
1316 *
1317 * If @work is queued or executing, wait for it to finish execution.
1318 */
1319void kthread_flush_work(struct kthread_work *work)
1320{
1321 struct kthread_flush_work fwork = {
1322 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1323 COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1324 };
1325 struct kthread_worker *worker;
1326 bool noop = false;
1327
1328 worker = work->worker;
1329 if (!worker)
1330 return;
1331
1332 raw_spin_lock_irq(&worker->lock);
1333 /* Work must not be used with >1 worker, see kthread_queue_work(). */
1334 WARN_ON_ONCE(work->worker != worker);
1335
1336 if (!list_empty(head: &work->node))
1337 kthread_insert_work(worker, work: &fwork.work, pos: work->node.next);
1338 else if (worker->current_work == work)
1339 kthread_insert_work(worker, work: &fwork.work,
1340 pos: worker->work_list.next);
1341 else
1342 noop = true;
1343
1344 raw_spin_unlock_irq(&worker->lock);
1345
1346 if (!noop)
1347 wait_for_completion(&fwork.done);
1348}
1349EXPORT_SYMBOL_GPL(kthread_flush_work);
1350
1351/*
1352 * Make sure that the timer is neither set nor running and could
1353 * not manipulate the work list_head any longer.
1354 *
1355 * The function is called under worker->lock. The lock is temporary
1356 * released but the timer can't be set again in the meantime.
1357 */
1358static void kthread_cancel_delayed_work_timer(struct kthread_work *work,
1359 unsigned long *flags)
1360{
1361 struct kthread_delayed_work *dwork =
1362 container_of(work, struct kthread_delayed_work, work);
1363 struct kthread_worker *worker = work->worker;
1364
1365 /*
1366 * timer_delete_sync() must be called to make sure that the timer
1367 * callback is not running. The lock must be temporary released
1368 * to avoid a deadlock with the callback. In the meantime,
1369 * any queuing is blocked by setting the canceling counter.
1370 */
1371 work->canceling++;
1372 raw_spin_unlock_irqrestore(&worker->lock, *flags);
1373 timer_delete_sync(timer: &dwork->timer);
1374 raw_spin_lock_irqsave(&worker->lock, *flags);
1375 work->canceling--;
1376}
1377
1378/*
1379 * This function removes the work from the worker queue.
1380 *
1381 * It is called under worker->lock. The caller must make sure that
1382 * the timer used by delayed work is not running, e.g. by calling
1383 * kthread_cancel_delayed_work_timer().
1384 *
1385 * The work might still be in use when this function finishes. See the
1386 * current_work proceed by the worker.
1387 *
1388 * Return: %true if @work was pending and successfully canceled,
1389 * %false if @work was not pending
1390 */
1391static bool __kthread_cancel_work(struct kthread_work *work)
1392{
1393 /*
1394 * Try to remove the work from a worker list. It might either
1395 * be from worker->work_list or from worker->delayed_work_list.
1396 */
1397 if (!list_empty(head: &work->node)) {
1398 list_del_init(entry: &work->node);
1399 return true;
1400 }
1401
1402 return false;
1403}
1404
1405/**
1406 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
1407 * @worker: kthread worker to use
1408 * @dwork: kthread delayed work to queue
1409 * @delay: number of jiffies to wait before queuing
1410 *
1411 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
1412 * modify @dwork's timer so that it expires after @delay. If @delay is zero,
1413 * @work is guaranteed to be queued immediately.
1414 *
1415 * Return: %false if @dwork was idle and queued, %true otherwise.
1416 *
1417 * A special case is when the work is being canceled in parallel.
1418 * It might be caused either by the real kthread_cancel_delayed_work_sync()
1419 * or yet another kthread_mod_delayed_work() call. We let the other command
1420 * win and return %true here. The return value can be used for reference
1421 * counting and the number of queued works stays the same. Anyway, the caller
1422 * is supposed to synchronize these operations a reasonable way.
1423 *
1424 * This function is safe to call from any context including IRQ handler.
1425 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
1426 * for details.
1427 */
1428bool kthread_mod_delayed_work(struct kthread_worker *worker,
1429 struct kthread_delayed_work *dwork,
1430 unsigned long delay)
1431{
1432 struct kthread_work *work = &dwork->work;
1433 unsigned long flags;
1434 int ret;
1435
1436 raw_spin_lock_irqsave(&worker->lock, flags);
1437
1438 /* Do not bother with canceling when never queued. */
1439 if (!work->worker) {
1440 ret = false;
1441 goto fast_queue;
1442 }
1443
1444 /* Work must not be used with >1 worker, see kthread_queue_work() */
1445 WARN_ON_ONCE(work->worker != worker);
1446
1447 /*
1448 * Temporary cancel the work but do not fight with another command
1449 * that is canceling the work as well.
1450 *
1451 * It is a bit tricky because of possible races with another
1452 * mod_delayed_work() and cancel_delayed_work() callers.
1453 *
1454 * The timer must be canceled first because worker->lock is released
1455 * when doing so. But the work can be removed from the queue (list)
1456 * only when it can be queued again so that the return value can
1457 * be used for reference counting.
1458 */
1459 kthread_cancel_delayed_work_timer(work, flags: &flags);
1460 if (work->canceling) {
1461 /* The number of works in the queue does not change. */
1462 ret = true;
1463 goto out;
1464 }
1465 ret = __kthread_cancel_work(work);
1466
1467fast_queue:
1468 __kthread_queue_delayed_work(worker, dwork, delay);
1469out:
1470 raw_spin_unlock_irqrestore(&worker->lock, flags);
1471 return ret;
1472}
1473EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
1474
1475static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
1476{
1477 struct kthread_worker *worker = work->worker;
1478 unsigned long flags;
1479 int ret = false;
1480
1481 if (!worker)
1482 goto out;
1483
1484 raw_spin_lock_irqsave(&worker->lock, flags);
1485 /* Work must not be used with >1 worker, see kthread_queue_work(). */
1486 WARN_ON_ONCE(work->worker != worker);
1487
1488 if (is_dwork)
1489 kthread_cancel_delayed_work_timer(work, flags: &flags);
1490
1491 ret = __kthread_cancel_work(work);
1492
1493 if (worker->current_work != work)
1494 goto out_fast;
1495
1496 /*
1497 * The work is in progress and we need to wait with the lock released.
1498 * In the meantime, block any queuing by setting the canceling counter.
1499 */
1500 work->canceling++;
1501 raw_spin_unlock_irqrestore(&worker->lock, flags);
1502 kthread_flush_work(work);
1503 raw_spin_lock_irqsave(&worker->lock, flags);
1504 work->canceling--;
1505
1506out_fast:
1507 raw_spin_unlock_irqrestore(&worker->lock, flags);
1508out:
1509 return ret;
1510}
1511
1512/**
1513 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
1514 * @work: the kthread work to cancel
1515 *
1516 * Cancel @work and wait for its execution to finish. This function
1517 * can be used even if the work re-queues itself. On return from this
1518 * function, @work is guaranteed to be not pending or executing on any CPU.
1519 *
1520 * kthread_cancel_work_sync(&delayed_work->work) must not be used for
1521 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
1522 *
1523 * The caller must ensure that the worker on which @work was last
1524 * queued can't be destroyed before this function returns.
1525 *
1526 * Return: %true if @work was pending, %false otherwise.
1527 */
1528bool kthread_cancel_work_sync(struct kthread_work *work)
1529{
1530 return __kthread_cancel_work_sync(work, is_dwork: false);
1531}
1532EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);
1533
1534/**
1535 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
1536 * wait for it to finish.
1537 * @dwork: the kthread delayed work to cancel
1538 *
1539 * This is kthread_cancel_work_sync() for delayed works.
1540 *
1541 * Return: %true if @dwork was pending, %false otherwise.
1542 */
1543bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
1544{
1545 return __kthread_cancel_work_sync(work: &dwork->work, is_dwork: true);
1546}
1547EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);
1548
1549/**
1550 * kthread_flush_worker - flush all current works on a kthread_worker
1551 * @worker: worker to flush
1552 *
1553 * Wait until all currently executing or pending works on @worker are
1554 * finished.
1555 */
1556void kthread_flush_worker(struct kthread_worker *worker)
1557{
1558 struct kthread_flush_work fwork = {
1559 KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1560 COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1561 };
1562
1563 kthread_queue_work(worker, &fwork.work);
1564 wait_for_completion(&fwork.done);
1565}
1566EXPORT_SYMBOL_GPL(kthread_flush_worker);
1567
1568/**
1569 * kthread_destroy_worker - destroy a kthread worker
1570 * @worker: worker to be destroyed
1571 *
1572 * Flush and destroy @worker. The simple flush is enough because the kthread
1573 * worker API is used only in trivial scenarios. There are no multi-step state
1574 * machines needed.
1575 *
1576 * Note that this function is not responsible for handling delayed work, so
1577 * caller should be responsible for queuing or canceling all delayed work items
1578 * before invoke this function.
1579 */
1580void kthread_destroy_worker(struct kthread_worker *worker)
1581{
1582 struct task_struct *task;
1583
1584 task = worker->task;
1585 if (WARN_ON(!task))
1586 return;
1587
1588 kthread_flush_worker(worker);
1589 kthread_stop(task);
1590 WARN_ON(!list_empty(&worker->delayed_work_list));
1591 WARN_ON(!list_empty(&worker->work_list));
1592 kfree(objp: worker);
1593}
1594EXPORT_SYMBOL(kthread_destroy_worker);
1595
1596/**
1597 * kthread_use_mm - make the calling kthread operate on an address space
1598 * @mm: address space to operate on
1599 */
1600void kthread_use_mm(struct mm_struct *mm)
1601{
1602 struct mm_struct *active_mm;
1603 struct task_struct *tsk = current;
1604
1605 WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1606 WARN_ON_ONCE(tsk->mm);
1607
1608 /*
1609 * It is possible for mm to be the same as tsk->active_mm, but
1610 * we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm),
1611 * because these references are not equivalent.
1612 */
1613 mmgrab(mm);
1614
1615 task_lock(p: tsk);
1616 /* Hold off tlb flush IPIs while switching mm's */
1617 local_irq_disable();
1618 active_mm = tsk->active_mm;
1619 tsk->active_mm = mm;
1620 tsk->mm = mm;
1621 membarrier_update_current_mm(next_mm: mm);
1622 switch_mm_irqs_off(prev: active_mm, next: mm, tsk);
1623 local_irq_enable();
1624 task_unlock(p: tsk);
1625#ifdef finish_arch_post_lock_switch
1626 finish_arch_post_lock_switch();
1627#endif
1628
1629 /*
1630 * When a kthread starts operating on an address space, the loop
1631 * in membarrier_{private,global}_expedited() may not observe
1632 * that tsk->mm, and not issue an IPI. Membarrier requires a
1633 * memory barrier after storing to tsk->mm, before accessing
1634 * user-space memory. A full memory barrier for membarrier
1635 * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by
1636 * mmdrop_lazy_tlb().
1637 */
1638 mmdrop_lazy_tlb(mm: active_mm);
1639}
1640EXPORT_SYMBOL_GPL(kthread_use_mm);
1641
1642/**
1643 * kthread_unuse_mm - reverse the effect of kthread_use_mm()
1644 * @mm: address space to operate on
1645 */
1646void kthread_unuse_mm(struct mm_struct *mm)
1647{
1648 struct task_struct *tsk = current;
1649
1650 WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1651 WARN_ON_ONCE(!tsk->mm);
1652
1653 task_lock(p: tsk);
1654 /*
1655 * When a kthread stops operating on an address space, the loop
1656 * in membarrier_{private,global}_expedited() may not observe
1657 * that tsk->mm, and not issue an IPI. Membarrier requires a
1658 * memory barrier after accessing user-space memory, before
1659 * clearing tsk->mm.
1660 */
1661 smp_mb__after_spinlock();
1662 local_irq_disable();
1663 tsk->mm = NULL;
1664 membarrier_update_current_mm(NULL);
1665 mmgrab_lazy_tlb(mm);
1666 /* active_mm is still 'mm' */
1667 enter_lazy_tlb(mm, tsk);
1668 local_irq_enable();
1669 task_unlock(p: tsk);
1670
1671 mmdrop(mm);
1672}
1673EXPORT_SYMBOL_GPL(kthread_unuse_mm);
1674
1675#ifdef CONFIG_BLK_CGROUP
1676/**
1677 * kthread_associate_blkcg - associate blkcg to current kthread
1678 * @css: the cgroup info
1679 *
1680 * Current thread must be a kthread. The thread is running jobs on behalf of
1681 * other threads. In some cases, we expect the jobs attach cgroup info of
1682 * original threads instead of that of current thread. This function stores
1683 * original thread's cgroup info in current kthread context for later
1684 * retrieval.
1685 */
1686void kthread_associate_blkcg(struct cgroup_subsys_state *css)
1687{
1688 struct kthread *kthread;
1689
1690 if (!(current->flags & PF_KTHREAD))
1691 return;
1692 kthread = to_kthread(current);
1693 if (!kthread)
1694 return;
1695
1696 if (kthread->blkcg_css) {
1697 css_put(css: kthread->blkcg_css);
1698 kthread->blkcg_css = NULL;
1699 }
1700 if (css) {
1701 css_get(css);
1702 kthread->blkcg_css = css;
1703 }
1704}
1705EXPORT_SYMBOL(kthread_associate_blkcg);
1706
1707/**
1708 * kthread_blkcg - get associated blkcg css of current kthread
1709 *
1710 * Current thread must be a kthread.
1711 */
1712struct cgroup_subsys_state *kthread_blkcg(void)
1713{
1714 struct kthread *kthread;
1715
1716 if (current->flags & PF_KTHREAD) {
1717 kthread = to_kthread(current);
1718 if (kthread)
1719 return kthread->blkcg_css;
1720 }
1721 return NULL;
1722}
1723#endif
1724