1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10#include <linux/sched/isolation.h>
11#include "sched.h"
12
13enum hk_flags {
14 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
15 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
16 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE),
17};
18
19DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
20EXPORT_SYMBOL_GPL(housekeeping_overridden);
21
22struct housekeeping {
23 cpumask_var_t cpumasks[HK_TYPE_MAX];
24 unsigned long flags;
25};
26
27static struct housekeeping housekeeping;
28
29bool housekeeping_enabled(enum hk_type type)
30{
31 return !!(housekeeping.flags & BIT(type));
32}
33EXPORT_SYMBOL_GPL(housekeeping_enabled);
34
35int housekeeping_any_cpu(enum hk_type type)
36{
37 int cpu;
38
39 if (static_branch_unlikely(&housekeeping_overridden)) {
40 if (housekeeping.flags & BIT(type)) {
41 cpu = sched_numa_find_closest(cpus: housekeeping.cpumasks[type], smp_processor_id());
42 if (cpu < nr_cpu_ids)
43 return cpu;
44
45 cpu = cpumask_any_and_distribute(src1p: housekeeping.cpumasks[type], cpu_online_mask);
46 if (likely(cpu < nr_cpu_ids))
47 return cpu;
48 /*
49 * Unless we have another problem this can only happen
50 * at boot time before start_secondary() brings the 1st
51 * housekeeping CPU up.
52 */
53 WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
54 type != HK_TYPE_TIMER);
55 }
56 }
57 return smp_processor_id();
58}
59EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
60
61const struct cpumask *housekeeping_cpumask(enum hk_type type)
62{
63 if (static_branch_unlikely(&housekeeping_overridden))
64 if (housekeeping.flags & BIT(type))
65 return housekeeping.cpumasks[type];
66 return cpu_possible_mask;
67}
68EXPORT_SYMBOL_GPL(housekeeping_cpumask);
69
70void housekeeping_affine(struct task_struct *t, enum hk_type type)
71{
72 if (static_branch_unlikely(&housekeeping_overridden))
73 if (housekeeping.flags & BIT(type))
74 set_cpus_allowed_ptr(p: t, new_mask: housekeeping.cpumasks[type]);
75}
76EXPORT_SYMBOL_GPL(housekeeping_affine);
77
78bool housekeeping_test_cpu(int cpu, enum hk_type type)
79{
80 if (static_branch_unlikely(&housekeeping_overridden))
81 if (housekeeping.flags & BIT(type))
82 return cpumask_test_cpu(cpu, cpumask: housekeeping.cpumasks[type]);
83 return true;
84}
85EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
86
87void __init housekeeping_init(void)
88{
89 enum hk_type type;
90
91 if (!housekeeping.flags)
92 return;
93
94 static_branch_enable(&housekeeping_overridden);
95
96 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE)
97 sched_tick_offload_init();
98
99 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
100 /* We need at least one CPU to handle housekeeping work */
101 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
102 }
103}
104
105static void __init housekeeping_setup_type(enum hk_type type,
106 cpumask_var_t housekeeping_staging)
107{
108
109 alloc_bootmem_cpumask_var(mask: &housekeeping.cpumasks[type]);
110 cpumask_copy(dstp: housekeeping.cpumasks[type],
111 srcp: housekeeping_staging);
112}
113
114static int __init housekeeping_setup(char *str, unsigned long flags)
115{
116 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
117 unsigned int first_cpu;
118 int err = 0;
119
120 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) {
121 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
122 pr_warn("Housekeeping: nohz unsupported."
123 " Build with CONFIG_NO_HZ_FULL\n");
124 return 0;
125 }
126 }
127
128 alloc_bootmem_cpumask_var(mask: &non_housekeeping_mask);
129 if (cpulist_parse(buf: str, dstp: non_housekeeping_mask) < 0) {
130 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
131 goto free_non_housekeeping_mask;
132 }
133
134 alloc_bootmem_cpumask_var(mask: &housekeeping_staging);
135 cpumask_andnot(dstp: housekeeping_staging,
136 cpu_possible_mask, src2p: non_housekeeping_mask);
137
138 first_cpu = cpumask_first_and(cpu_present_mask, srcp2: housekeeping_staging);
139 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
140 __cpumask_set_cpu(smp_processor_id(), dstp: housekeeping_staging);
141 __cpumask_clear_cpu(smp_processor_id(), dstp: non_housekeeping_mask);
142 if (!housekeeping.flags) {
143 pr_warn("Housekeeping: must include one present CPU, "
144 "using boot CPU:%d\n", smp_processor_id());
145 }
146 }
147
148 if (cpumask_empty(srcp: non_housekeeping_mask))
149 goto free_housekeeping_staging;
150
151 if (!housekeeping.flags) {
152 /* First setup call ("nohz_full=" or "isolcpus=") */
153 enum hk_type type;
154
155 for_each_set_bit(type, &flags, HK_TYPE_MAX)
156 housekeeping_setup_type(type, housekeeping_staging);
157 } else {
158 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
159 enum hk_type type;
160 unsigned long iter_flags = flags & housekeeping.flags;
161
162 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
163 if (!cpumask_equal(src1p: housekeeping_staging,
164 src2p: housekeeping.cpumasks[type])) {
165 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
166 goto free_housekeeping_staging;
167 }
168 }
169
170 iter_flags = flags & ~housekeeping.flags;
171
172 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
173 housekeeping_setup_type(type, housekeeping_staging);
174 }
175
176 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE))
177 tick_nohz_full_setup(cpumask: non_housekeeping_mask);
178
179 housekeeping.flags |= flags;
180 err = 1;
181
182free_housekeeping_staging:
183 free_bootmem_cpumask_var(mask: housekeeping_staging);
184free_non_housekeeping_mask:
185 free_bootmem_cpumask_var(mask: non_housekeeping_mask);
186
187 return err;
188}
189
190static int __init housekeeping_nohz_full_setup(char *str)
191{
192 unsigned long flags;
193
194 flags = HK_FLAG_KERNEL_NOISE;
195
196 return housekeeping_setup(str, flags);
197}
198__setup("nohz_full=", housekeeping_nohz_full_setup);
199
200static int __init housekeeping_isolcpus_setup(char *str)
201{
202 unsigned long flags = 0;
203 bool illegal = false;
204 char *par;
205 int len;
206
207 while (isalpha(*str)) {
208 /*
209 * isolcpus=nohz is equivalent to nohz_full.
210 */
211 if (!strncmp(str, "nohz,", 5)) {
212 str += 5;
213 flags |= HK_FLAG_KERNEL_NOISE;
214 continue;
215 }
216
217 if (!strncmp(str, "domain,", 7)) {
218 str += 7;
219 flags |= HK_FLAG_DOMAIN;
220 continue;
221 }
222
223 if (!strncmp(str, "managed_irq,", 12)) {
224 str += 12;
225 flags |= HK_FLAG_MANAGED_IRQ;
226 continue;
227 }
228
229 /*
230 * Skip unknown sub-parameter and validate that it is not
231 * containing an invalid character.
232 */
233 for (par = str, len = 0; *str && *str != ','; str++, len++) {
234 if (!isalpha(*str) && *str != '_')
235 illegal = true;
236 }
237
238 if (illegal) {
239 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
240 return 0;
241 }
242
243 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
244 str++;
245 }
246
247 /* Default behaviour for isolcpus without flags */
248 if (!flags)
249 flags |= HK_FLAG_DOMAIN;
250
251 return housekeeping_setup(str, flags);
252}
253__setup("isolcpus=", housekeeping_isolcpus_setup);
254