1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/cpumask.h>
4#include <linux/delay.h>
5#include <linux/smp.h>
6#include <linux/string_choices.h>
7
8#include <asm/io_apic.h>
9
10#include "local.h"
11
12DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);
13
14#ifdef CONFIG_SMP
15static int apic_ipi_shorthand_off __ro_after_init;
16
17static __init int apic_ipi_shorthand(char *str)
18{
19 get_option(str: &str, pint: &apic_ipi_shorthand_off);
20 return 1;
21}
22__setup("no_ipi_broadcast=", apic_ipi_shorthand);
23
24static int __init print_ipi_mode(void)
25{
26 pr_info("IPI shorthand broadcast: %s\n",
27 str_disabled_enabled(apic_ipi_shorthand_off));
28 return 0;
29}
30late_initcall(print_ipi_mode);
31
32void apic_smt_update(void)
33{
34 /*
35 * Do not switch to broadcast mode if:
36 * - Disabled on the command line
37 * - Only a single CPU is online
38 * - Not all present CPUs have been at least booted once
39 *
40 * The latter is important as the local APIC might be in some
41 * random state and a broadcast might cause havoc. That's
42 * especially true for NMI broadcasting.
43 */
44 if (apic_ipi_shorthand_off || num_online_cpus() == 1 ||
45 !cpumask_equal(cpu_present_mask, src2p: &cpus_booted_once_mask)) {
46 static_branch_disable(&apic_use_ipi_shorthand);
47 } else {
48 static_branch_enable(&apic_use_ipi_shorthand);
49 }
50}
51
52void apic_send_IPI_allbutself(unsigned int vector)
53{
54 if (num_online_cpus() < 2)
55 return;
56
57 if (static_branch_likely(&apic_use_ipi_shorthand))
58 __apic_send_IPI_allbutself(vector);
59 else
60 __apic_send_IPI_mask_allbutself(cpu_online_mask, vector);
61}
62
63/*
64 * Send a 'reschedule' IPI to another CPU. It goes straight through and
65 * wastes no time serializing anything. Worst case is that we lose a
66 * reschedule ...
67 */
68void native_smp_send_reschedule(int cpu)
69{
70 if (unlikely(cpu_is_offline(cpu))) {
71 WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
72 return;
73 }
74 __apic_send_IPI(cpu, RESCHEDULE_VECTOR);
75}
76
77void native_send_call_func_single_ipi(int cpu)
78{
79 __apic_send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
80}
81
82void native_send_call_func_ipi(const struct cpumask *mask)
83{
84 if (static_branch_likely(&apic_use_ipi_shorthand)) {
85 unsigned int cpu = smp_processor_id();
86
87 if (!cpumask_or_equal(src1p: mask, cpumask_of(cpu), cpu_online_mask))
88 goto sendmask;
89
90 if (cpumask_test_cpu(cpu, cpumask: mask))
91 __apic_send_IPI_all(CALL_FUNCTION_VECTOR);
92 else if (num_online_cpus() > 1)
93 __apic_send_IPI_allbutself(CALL_FUNCTION_VECTOR);
94 return;
95 }
96
97sendmask:
98 __apic_send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
99}
100
101void apic_send_nmi_to_offline_cpu(unsigned int cpu)
102{
103 if (WARN_ON_ONCE(!apic->nmi_to_offline_cpu))
104 return;
105 if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, &cpus_booted_once_mask)))
106 return;
107 apic->send_IPI(cpu, NMI_VECTOR);
108}
109#endif /* CONFIG_SMP */
110
111static inline int __prepare_ICR2(unsigned int mask)
112{
113 return SET_XAPIC_DEST_FIELD(mask);
114}
115
116u32 apic_mem_wait_icr_idle_timeout(void)
117{
118 int cnt;
119
120 for (cnt = 0; cnt < 1000; cnt++) {
121 if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY))
122 return 0;
123 inc_irq_stat(icr_read_retry_count);
124 udelay(usec: 100);
125 }
126 return APIC_ICR_BUSY;
127}
128
129void apic_mem_wait_icr_idle(void)
130{
131 while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
132 cpu_relax();
133}
134
135/*
136 * This is safe against interruption because it only writes the lower 32
137 * bits of the APIC_ICR register. The destination field is ignored for
138 * short hand IPIs.
139 *
140 * wait_icr_idle()
141 * write(ICR2, dest)
142 * NMI
143 * wait_icr_idle()
144 * write(ICR)
145 * wait_icr_idle()
146 * write(ICR)
147 *
148 * This function does not need to disable interrupts as there is no ICR2
149 * interaction. The memory write is direct except when the machine is
150 * affected by the 11AP Pentium erratum, which turns the plain write into
151 * an XCHG operation.
152 */
153static void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
154{
155 /*
156 * Wait for the previous ICR command to complete. Use
157 * safe_apic_wait_icr_idle() for the NMI vector as there have been
158 * issues where otherwise the system hangs when the panic CPU tries
159 * to stop the others before launching the kdump kernel.
160 */
161 if (unlikely(vector == NMI_VECTOR))
162 apic_mem_wait_icr_idle_timeout();
163 else
164 apic_mem_wait_icr_idle();
165
166 /* Destination field (ICR2) and the destination mode are ignored */
167 native_apic_mem_write(APIC_ICR, v: __prepare_ICR(shortcut, vector, dest: 0));
168}
169
170/*
171 * This is used to send an IPI with no shorthand notation (the destination is
172 * specified in bits 56 to 63 of the ICR).
173 */
174void __default_send_IPI_dest_field(unsigned int dest_mask, int vector,
175 unsigned int dest_mode)
176{
177 /* See comment in __default_send_IPI_shortcut() */
178 if (unlikely(vector == NMI_VECTOR))
179 apic_mem_wait_icr_idle_timeout();
180 else
181 apic_mem_wait_icr_idle();
182
183 /* Set the IPI destination field in the ICR */
184 native_apic_mem_write(APIC_ICR2, v: __prepare_ICR2(mask: dest_mask));
185 /* Send it with the proper destination mode */
186 native_apic_mem_write(APIC_ICR, v: __prepare_ICR(shortcut: 0, vector, dest: dest_mode));
187}
188
189void default_send_IPI_single_phys(int cpu, int vector)
190{
191 unsigned long flags;
192
193 local_irq_save(flags);
194 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
195 vector, APIC_DEST_PHYSICAL);
196 local_irq_restore(flags);
197}
198
199void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
200{
201 unsigned long flags;
202 unsigned long cpu;
203
204 local_irq_save(flags);
205 for_each_cpu(cpu, mask) {
206 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
207 cpu), vector, APIC_DEST_PHYSICAL);
208 }
209 local_irq_restore(flags);
210}
211
212void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
213 int vector)
214{
215 unsigned int cpu, this_cpu = smp_processor_id();
216 unsigned long flags;
217
218 local_irq_save(flags);
219 for_each_cpu(cpu, mask) {
220 if (cpu == this_cpu)
221 continue;
222 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
223 cpu), vector, APIC_DEST_PHYSICAL);
224 }
225 local_irq_restore(flags);
226}
227
228/*
229 * Helper function for APICs which insist on cpumasks
230 */
231void default_send_IPI_single(int cpu, int vector)
232{
233 __apic_send_IPI_mask(cpumask_of(cpu), vector);
234}
235
236void default_send_IPI_allbutself(int vector)
237{
238 __default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
239}
240
241void default_send_IPI_all(int vector)
242{
243 __default_send_IPI_shortcut(APIC_DEST_ALLINC, vector);
244}
245
246void default_send_IPI_self(int vector)
247{
248 __default_send_IPI_shortcut(APIC_DEST_SELF, vector);
249}
250
251#ifdef CONFIG_X86_32
252void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector)
253{
254 unsigned long flags;
255 unsigned int cpu;
256
257 local_irq_save(flags);
258 for_each_cpu(cpu, mask)
259 __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
260 local_irq_restore(flags);
261}
262
263void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
264 int vector)
265{
266 unsigned int cpu, this_cpu = smp_processor_id();
267 unsigned long flags;
268
269 local_irq_save(flags);
270 for_each_cpu(cpu, mask) {
271 if (cpu == this_cpu)
272 continue;
273 __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
274 }
275 local_irq_restore(flags);
276}
277
278void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
279{
280 unsigned long mask = cpumask_bits(cpumask)[0];
281 unsigned long flags;
282
283 if (!mask)
284 return;
285
286 local_irq_save(flags);
287 WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
288 __default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
289 local_irq_restore(flags);
290}
291#endif
292