1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * processor_idle - idle state submodule to the ACPI processor driver
4 *
5 * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
6 * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
7 * Copyright (C) 2004, 2005 Dominik Brodowski <linux@brodo.de>
8 * Copyright (C) 2004 Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
9 * - Added processor hotplug support
10 * Copyright (C) 2005 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
11 * - Added support for C3 on SMP
12 */
13#define pr_fmt(fmt) "ACPI: " fmt
14
15#include <linux/module.h>
16#include <linux/acpi.h>
17#include <linux/dmi.h>
18#include <linux/sched.h> /* need_resched() */
19#include <linux/tick.h>
20#include <linux/cpuidle.h>
21#include <linux/cpu.h>
22#include <linux/minmax.h>
23#include <linux/perf_event.h>
24#include <acpi/processor.h>
25#include <linux/context_tracking.h>
26
27#include "internal.h"
28
29/*
30 * Include the apic definitions for x86 to have the APIC timer related defines
31 * available also for UP (on SMP it gets magically included via linux/smp.h).
32 * asm/acpi.h is not an option, as it would require more include magic. Also
33 * creating an empty asm-ia64/apic.h would just trade pest vs. cholera.
34 */
35#ifdef CONFIG_X86
36#include <asm/apic.h>
37#include <asm/cpu.h>
38#endif
39
40#define ACPI_IDLE_STATE_START (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX) ? 1 : 0)
41
42static unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER;
43module_param(max_cstate, uint, 0400);
44static bool nocst __read_mostly;
45module_param(nocst, bool, 0400);
46static bool bm_check_disable __read_mostly;
47module_param(bm_check_disable, bool, 0400);
48
49static unsigned int latency_factor __read_mostly = 2;
50module_param(latency_factor, uint, 0644);
51
52static DEFINE_PER_CPU(struct cpuidle_device *, acpi_cpuidle_device);
53
54static struct cpuidle_driver acpi_idle_driver = {
55 .name = "acpi_idle",
56 .owner = THIS_MODULE,
57};
58
59#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
60void acpi_idle_rescan_dead_smt_siblings(void)
61{
62 if (cpuidle_get_driver() == &acpi_idle_driver)
63 arch_cpu_rescan_dead_smt_siblings();
64}
65
66static
67DEFINE_PER_CPU(struct acpi_processor_cx * [CPUIDLE_STATE_MAX], acpi_cstate);
68
69static int disabled_by_idle_boot_param(void)
70{
71 return boot_option_idle_override == IDLE_POLL ||
72 boot_option_idle_override == IDLE_HALT;
73}
74
75/*
76 * IBM ThinkPad R40e crashes mysteriously when going into C2 or C3.
77 * For now disable this. Probably a bug somewhere else.
78 *
79 * To skip this limit, boot/load with a large max_cstate limit.
80 */
81static int set_max_cstate(const struct dmi_system_id *id)
82{
83 if (max_cstate > ACPI_PROCESSOR_MAX_POWER)
84 return 0;
85
86 pr_notice("%s detected - limiting to C%ld max_cstate."
87 " Override with \"processor.max_cstate=%d\"\n", id->ident,
88 (long)id->driver_data, ACPI_PROCESSOR_MAX_POWER + 1);
89
90 max_cstate = (long)id->driver_data;
91
92 return 0;
93}
94
95static const struct dmi_system_id processor_power_dmi_table[] = {
96 { set_max_cstate, "Clevo 5600D", {
97 DMI_MATCH(DMI_BIOS_VENDOR,"Phoenix Technologies LTD"),
98 DMI_MATCH(DMI_BIOS_VERSION,"SHE845M0.86C.0013.D.0302131307")},
99 (void *)2},
100 { set_max_cstate, "Pavilion zv5000", {
101 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
102 DMI_MATCH(DMI_PRODUCT_NAME,"Pavilion zv5000 (DS502A#ABA)")},
103 (void *)1},
104 { set_max_cstate, "Asus L8400B", {
105 DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer Inc."),
106 DMI_MATCH(DMI_PRODUCT_NAME,"L8400B series Notebook PC")},
107 (void *)1},
108 {},
109};
110
111
112/*
113 * Callers should disable interrupts before the call and enable
114 * interrupts after return.
115 */
116static void __cpuidle acpi_safe_halt(void)
117{
118 if (!tif_need_resched()) {
119 raw_safe_halt();
120 raw_local_irq_disable();
121 }
122}
123
124#ifdef ARCH_APICTIMER_STOPS_ON_C3
125
126/*
127 * Some BIOS implementations switch to C3 in the published C2 state.
128 * This seems to be a common problem on AMD boxen, but other vendors
129 * are affected too. We pick the most conservative approach: we assume
130 * that the local APIC stops in both C2 and C3.
131 */
132static void lapic_timer_check_state(int state, struct acpi_processor *pr,
133 struct acpi_processor_cx *cx)
134{
135 struct acpi_processor_power *pwr = &pr->power;
136 u8 type = local_apic_timer_c2_ok ? ACPI_STATE_C3 : ACPI_STATE_C2;
137
138 if (cpu_has(&cpu_data(pr->id), X86_FEATURE_ARAT))
139 return;
140
141 if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E))
142 type = ACPI_STATE_C1;
143
144 /*
145 * Check, if one of the previous states already marked the lapic
146 * unstable
147 */
148 if (pwr->timer_broadcast_on_state < state)
149 return;
150
151 if (cx->type >= type)
152 pr->power.timer_broadcast_on_state = state;
153}
154
155static void __lapic_timer_propagate_broadcast(void *arg)
156{
157 struct acpi_processor *pr = arg;
158
159 if (pr->power.timer_broadcast_on_state < INT_MAX)
160 tick_broadcast_enable();
161 else
162 tick_broadcast_disable();
163}
164
165static void lapic_timer_propagate_broadcast(struct acpi_processor *pr)
166{
167 smp_call_function_single(cpuid: pr->id, func: __lapic_timer_propagate_broadcast,
168 info: (void *)pr, wait: 1);
169}
170
171/* Power(C) State timer broadcast control */
172static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
173 struct acpi_processor_cx *cx)
174{
175 return cx - pr->power.states >= pr->power.timer_broadcast_on_state;
176}
177
178#else
179
180static void lapic_timer_check_state(int state, struct acpi_processor *pr,
181 struct acpi_processor_cx *cstate) { }
182static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) { }
183
184static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
185 struct acpi_processor_cx *cx)
186{
187 return false;
188}
189
190#endif
191
192#if defined(CONFIG_X86)
193static void tsc_check_state(int state)
194{
195 switch (boot_cpu_data.x86_vendor) {
196 case X86_VENDOR_HYGON:
197 case X86_VENDOR_AMD:
198 case X86_VENDOR_INTEL:
199 case X86_VENDOR_CENTAUR:
200 case X86_VENDOR_ZHAOXIN:
201 /*
202 * AMD Fam10h TSC will tick in all
203 * C/P/S0/S1 states when this bit is set.
204 */
205 if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
206 return;
207 fallthrough;
208 default:
209 /* TSC could halt in idle, so notify users */
210 if (state > ACPI_STATE_C1)
211 mark_tsc_unstable(reason: "TSC halts in idle");
212 }
213}
214#else
215static void tsc_check_state(int state) { return; }
216#endif
217
218static int acpi_processor_get_power_info_fadt(struct acpi_processor *pr)
219{
220
221 if (!pr->pblk)
222 return -ENODEV;
223
224 /* if info is obtained from pblk/fadt, type equals state */
225 pr->power.states[ACPI_STATE_C2].type = ACPI_STATE_C2;
226 pr->power.states[ACPI_STATE_C3].type = ACPI_STATE_C3;
227
228#ifndef CONFIG_HOTPLUG_CPU
229 /*
230 * Check for P_LVL2_UP flag before entering C2 and above on
231 * an SMP system.
232 */
233 if ((num_online_cpus() > 1) &&
234 !(acpi_gbl_FADT.flags & ACPI_FADT_C2_MP_SUPPORTED))
235 return -ENODEV;
236#endif
237
238 /* determine C2 and C3 address from pblk */
239 pr->power.states[ACPI_STATE_C2].address = pr->pblk + 4;
240 pr->power.states[ACPI_STATE_C3].address = pr->pblk + 5;
241
242 /* determine latencies from FADT */
243 pr->power.states[ACPI_STATE_C2].latency = acpi_gbl_FADT.c2_latency;
244 pr->power.states[ACPI_STATE_C3].latency = acpi_gbl_FADT.c3_latency;
245
246 /*
247 * FADT specified C2 latency must be less than or equal to
248 * 100 microseconds.
249 */
250 if (acpi_gbl_FADT.c2_latency > ACPI_PROCESSOR_MAX_C2_LATENCY) {
251 acpi_handle_debug(pr->handle, "C2 latency too large [%d]\n",
252 acpi_gbl_FADT.c2_latency);
253 /* invalidate C2 */
254 pr->power.states[ACPI_STATE_C2].address = 0;
255 }
256
257 /*
258 * FADT supplied C3 latency must be less than or equal to
259 * 1000 microseconds.
260 */
261 if (acpi_gbl_FADT.c3_latency > ACPI_PROCESSOR_MAX_C3_LATENCY) {
262 acpi_handle_debug(pr->handle, "C3 latency too large [%d]\n",
263 acpi_gbl_FADT.c3_latency);
264 /* invalidate C3 */
265 pr->power.states[ACPI_STATE_C3].address = 0;
266 }
267
268 acpi_handle_debug(pr->handle, "lvl2[0x%08x] lvl3[0x%08x]\n",
269 pr->power.states[ACPI_STATE_C2].address,
270 pr->power.states[ACPI_STATE_C3].address);
271
272 snprintf(buf: pr->power.states[ACPI_STATE_C2].desc,
273 ACPI_CX_DESC_LEN, fmt: "ACPI P_LVL2 IOPORT 0x%x",
274 pr->power.states[ACPI_STATE_C2].address);
275 snprintf(buf: pr->power.states[ACPI_STATE_C3].desc,
276 ACPI_CX_DESC_LEN, fmt: "ACPI P_LVL3 IOPORT 0x%x",
277 pr->power.states[ACPI_STATE_C3].address);
278
279 if (!pr->power.states[ACPI_STATE_C2].address &&
280 !pr->power.states[ACPI_STATE_C3].address)
281 return -ENODEV;
282
283 return 0;
284}
285
286static int acpi_processor_get_power_info_default(struct acpi_processor *pr)
287{
288 if (!pr->power.states[ACPI_STATE_C1].valid) {
289 /* set the first C-State to C1 */
290 /* all processors need to support C1 */
291 pr->power.states[ACPI_STATE_C1].type = ACPI_STATE_C1;
292 pr->power.states[ACPI_STATE_C1].valid = 1;
293 pr->power.states[ACPI_STATE_C1].entry_method = ACPI_CSTATE_HALT;
294
295 snprintf(buf: pr->power.states[ACPI_STATE_C1].desc,
296 ACPI_CX_DESC_LEN, fmt: "ACPI HLT");
297 }
298 /* the C0 state only exists as a filler in our array */
299 pr->power.states[ACPI_STATE_C0].valid = 1;
300 return 0;
301}
302
303static int acpi_processor_get_power_info_cst(struct acpi_processor *pr)
304{
305 int ret;
306
307 if (nocst)
308 return -ENODEV;
309
310 ret = acpi_processor_evaluate_cst(handle: pr->handle, cpu: pr->id, info: &pr->power);
311 if (ret)
312 return ret;
313
314 if (!pr->power.count)
315 return -EFAULT;
316
317 pr->flags.has_cst = 1;
318 return 0;
319}
320
321static void acpi_processor_power_verify_c3(struct acpi_processor *pr,
322 struct acpi_processor_cx *cx)
323{
324 static int bm_check_flag = -1;
325 static int bm_control_flag = -1;
326
327
328 if (!cx->address)
329 return;
330
331 /*
332 * PIIX4 Erratum #18: We don't support C3 when Type-F (fast)
333 * DMA transfers are used by any ISA device to avoid livelock.
334 * Note that we could disable Type-F DMA (as recommended by
335 * the erratum), but this is known to disrupt certain ISA
336 * devices thus we take the conservative approach.
337 */
338 if (errata.piix4.fdma) {
339 acpi_handle_debug(pr->handle,
340 "C3 not supported on PIIX4 with Type-F DMA\n");
341 return;
342 }
343
344 /* All the logic here assumes flags.bm_check is same across all CPUs */
345 if (bm_check_flag == -1) {
346 /* Determine whether bm_check is needed based on CPU */
347 acpi_processor_power_init_bm_check(flags: &(pr->flags), cpu: pr->id);
348 bm_check_flag = pr->flags.bm_check;
349 bm_control_flag = pr->flags.bm_control;
350 } else {
351 pr->flags.bm_check = bm_check_flag;
352 pr->flags.bm_control = bm_control_flag;
353 }
354
355 if (pr->flags.bm_check) {
356 if (!pr->flags.bm_control) {
357 if (pr->flags.has_cst != 1) {
358 /* bus mastering control is necessary */
359 acpi_handle_debug(pr->handle,
360 "C3 support requires BM control\n");
361 return;
362 } else {
363 /* Here we enter C3 without bus mastering */
364 acpi_handle_debug(pr->handle,
365 "C3 support without BM control\n");
366 }
367 }
368 } else {
369 /*
370 * WBINVD should be set in fadt, for C3 state to be
371 * supported on when bm_check is not required.
372 */
373 if (!(acpi_gbl_FADT.flags & ACPI_FADT_WBINVD)) {
374 acpi_handle_debug(pr->handle,
375 "Cache invalidation should work properly"
376 " for C3 to be enabled on SMP systems\n");
377 return;
378 }
379 }
380
381 /*
382 * Otherwise we've met all of our C3 requirements.
383 * Normalize the C3 latency to expidite policy. Enable
384 * checking of bus mastering status (bm_check) so we can
385 * use this in our C3 policy
386 */
387 cx->valid = 1;
388
389 /*
390 * On older chipsets, BM_RLD needs to be set
391 * in order for Bus Master activity to wake the
392 * system from C3. Newer chipsets handle DMA
393 * during C3 automatically and BM_RLD is a NOP.
394 * In either case, the proper way to
395 * handle BM_RLD is to set it and leave it set.
396 */
397 acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_RLD, value: 1);
398}
399
400static void acpi_cst_latency_sort(struct acpi_processor_cx *states, size_t length)
401{
402 int i, j, k;
403
404 for (i = 1; i < length; i++) {
405 if (!states[i].valid)
406 continue;
407
408 for (j = i - 1, k = i; j >= 0; j--) {
409 if (!states[j].valid)
410 continue;
411
412 if (states[j].latency > states[k].latency)
413 swap(states[j].latency, states[k].latency);
414
415 k = j;
416 }
417 }
418}
419
420static int acpi_processor_power_verify(struct acpi_processor *pr)
421{
422 unsigned int i;
423 unsigned int working = 0;
424 unsigned int last_latency = 0;
425 unsigned int last_type = 0;
426 bool buggy_latency = false;
427
428 pr->power.timer_broadcast_on_state = INT_MAX;
429
430 for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
431 struct acpi_processor_cx *cx = &pr->power.states[i];
432
433 switch (cx->type) {
434 case ACPI_STATE_C1:
435 cx->valid = 1;
436 break;
437
438 case ACPI_STATE_C2:
439 if (!cx->address)
440 break;
441 cx->valid = 1;
442 break;
443
444 case ACPI_STATE_C3:
445 acpi_processor_power_verify_c3(pr, cx);
446 break;
447 }
448 if (!cx->valid)
449 continue;
450 if (cx->type >= last_type && cx->latency < last_latency)
451 buggy_latency = true;
452 last_latency = cx->latency;
453 last_type = cx->type;
454
455 lapic_timer_check_state(state: i, pr, cx);
456 tsc_check_state(state: cx->type);
457 working++;
458 }
459
460 if (buggy_latency) {
461 pr_notice("FW issue: working around C-state latencies out of order\n");
462 acpi_cst_latency_sort(states: &pr->power.states[1], length: max_cstate);
463 }
464
465 lapic_timer_propagate_broadcast(pr);
466
467 return working;
468}
469
470static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
471{
472 int result;
473
474 /* NOTE: the idle thread may not be running while calling
475 * this function */
476
477 /* Zero initialize all the C-states info. */
478 memset(s: pr->power.states, c: 0, n: sizeof(pr->power.states));
479
480 result = acpi_processor_get_power_info_cst(pr);
481 if (result == -ENODEV)
482 result = acpi_processor_get_power_info_fadt(pr);
483
484 if (result)
485 return result;
486
487 acpi_processor_get_power_info_default(pr);
488
489 pr->power.count = acpi_processor_power_verify(pr);
490 pr->flags.power = 1;
491
492 return 0;
493}
494
495/**
496 * acpi_idle_bm_check - checks if bus master activity was detected
497 */
498static int acpi_idle_bm_check(void)
499{
500 u32 bm_status = 0;
501
502 if (bm_check_disable)
503 return 0;
504
505 acpi_read_bit_register(ACPI_BITREG_BUS_MASTER_STATUS, return_value: &bm_status);
506 if (bm_status)
507 acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_STATUS, value: 1);
508 /*
509 * PIIX4 Erratum #18: Note that BM_STS doesn't always reflect
510 * the true state of bus mastering activity; forcing us to
511 * manually check the BMIDEA bit of each IDE channel.
512 */
513 else if (errata.piix4.bmisx) {
514 if ((inb_p(port: errata.piix4.bmisx + 0x02) & 0x01)
515 || (inb_p(port: errata.piix4.bmisx + 0x0A) & 0x01))
516 bm_status = 1;
517 }
518 return bm_status;
519}
520
521static __cpuidle void io_idle(unsigned long addr)
522{
523 /* IO port based C-state */
524 inb(port: addr);
525
526#ifdef CONFIG_X86
527 /* No delay is needed if we are in guest */
528 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
529 return;
530 /*
531 * Modern (>=Nehalem) Intel systems use ACPI via intel_idle,
532 * not this code. Assume that any Intel systems using this
533 * are ancient and may need the dummy wait. This also assumes
534 * that the motivating chipset issue was Intel-only.
535 */
536 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
537 return;
538#endif
539 /*
540 * Dummy wait op - must do something useless after P_LVL2 read
541 * because chipsets cannot guarantee that STPCLK# signal gets
542 * asserted in time to freeze execution properly
543 *
544 * This workaround has been in place since the original ACPI
545 * implementation was merged, circa 2002.
546 *
547 * If a profile is pointing to this instruction, please first
548 * consider moving your system to a more modern idle
549 * mechanism.
550 */
551 inl(port: acpi_gbl_FADT.xpm_timer_block.address);
552}
553
554/**
555 * acpi_idle_do_entry - enter idle state using the appropriate method
556 * @cx: cstate data
557 *
558 * Caller disables interrupt before call and enables interrupt after return.
559 */
560static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
561{
562 perf_lopwr_cb(mode: true);
563
564 if (cx->entry_method == ACPI_CSTATE_FFH) {
565 /* Call into architectural FFH based C-state */
566 acpi_processor_ffh_cstate_enter(cstate: cx);
567 } else if (cx->entry_method == ACPI_CSTATE_HALT) {
568 acpi_safe_halt();
569 } else {
570 io_idle(addr: cx->address);
571 }
572
573 perf_lopwr_cb(mode: false);
574}
575
576/**
577 * acpi_idle_play_dead - enters an ACPI state for long-term idle (i.e. off-lining)
578 * @dev: the target CPU
579 * @index: the index of suggested state
580 */
581static void acpi_idle_play_dead(struct cpuidle_device *dev, int index)
582{
583 struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
584
585 ACPI_FLUSH_CPU_CACHE();
586
587 while (1) {
588
589 if (cx->entry_method == ACPI_CSTATE_HALT)
590 raw_safe_halt();
591 else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
592 io_idle(addr: cx->address);
593 } else if (cx->entry_method == ACPI_CSTATE_FFH) {
594 acpi_processor_ffh_play_dead(cx);
595 } else
596 return;
597 }
598}
599
600static __always_inline bool acpi_idle_fallback_to_c1(struct acpi_processor *pr)
601{
602 return IS_ENABLED(CONFIG_HOTPLUG_CPU) && !pr->flags.has_cst &&
603 !(acpi_gbl_FADT.flags & ACPI_FADT_C2_MP_SUPPORTED);
604}
605
606static int c3_cpu_count;
607static DEFINE_RAW_SPINLOCK(c3_lock);
608
609/**
610 * acpi_idle_enter_bm - enters C3 with proper BM handling
611 * @drv: cpuidle driver
612 * @pr: Target processor
613 * @cx: Target state context
614 * @index: index of target state
615 */
616static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv,
617 struct acpi_processor *pr,
618 struct acpi_processor_cx *cx,
619 int index)
620{
621 static struct acpi_processor_cx safe_cx = {
622 .entry_method = ACPI_CSTATE_HALT,
623 };
624
625 /*
626 * disable bus master
627 * bm_check implies we need ARB_DIS
628 * bm_control implies whether we can do ARB_DIS
629 *
630 * That leaves a case where bm_check is set and bm_control is not set.
631 * In that case we cannot do much, we enter C3 without doing anything.
632 */
633 bool dis_bm = pr->flags.bm_control;
634
635 instrumentation_begin();
636
637 /* If we can skip BM, demote to a safe state. */
638 if (!cx->bm_sts_skip && acpi_idle_bm_check()) {
639 dis_bm = false;
640 index = drv->safe_state_index;
641 if (index >= 0) {
642 cx = this_cpu_read(acpi_cstate[index]);
643 } else {
644 cx = &safe_cx;
645 index = -EBUSY;
646 }
647 }
648
649 if (dis_bm) {
650 raw_spin_lock(&c3_lock);
651 c3_cpu_count++;
652 /* Disable bus master arbitration when all CPUs are in C3 */
653 if (c3_cpu_count == num_online_cpus())
654 acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, value: 1);
655 raw_spin_unlock(&c3_lock);
656 }
657
658 ct_cpuidle_enter();
659
660 acpi_idle_do_entry(cx);
661
662 ct_cpuidle_exit();
663
664 /* Re-enable bus master arbitration */
665 if (dis_bm) {
666 raw_spin_lock(&c3_lock);
667 acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, value: 0);
668 c3_cpu_count--;
669 raw_spin_unlock(&c3_lock);
670 }
671
672 instrumentation_end();
673
674 return index;
675}
676
677static int __cpuidle acpi_idle_enter(struct cpuidle_device *dev,
678 struct cpuidle_driver *drv, int index)
679{
680 struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
681 struct acpi_processor *pr;
682
683 pr = __this_cpu_read(processors);
684 if (unlikely(!pr))
685 return -EINVAL;
686
687 if (cx->type != ACPI_STATE_C1) {
688 if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check)
689 return acpi_idle_enter_bm(drv, pr, cx, index);
690
691 /* C2 to C1 demotion. */
692 if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) {
693 index = ACPI_IDLE_STATE_START;
694 cx = per_cpu(acpi_cstate[index], dev->cpu);
695 }
696 }
697
698 if (cx->type == ACPI_STATE_C3)
699 ACPI_FLUSH_CPU_CACHE();
700
701 acpi_idle_do_entry(cx);
702
703 return index;
704}
705
706static int __cpuidle acpi_idle_enter_s2idle(struct cpuidle_device *dev,
707 struct cpuidle_driver *drv, int index)
708{
709 struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
710
711 if (cx->type == ACPI_STATE_C3) {
712 struct acpi_processor *pr = __this_cpu_read(processors);
713
714 if (unlikely(!pr))
715 return 0;
716
717 if (pr->flags.bm_check) {
718 u8 bm_sts_skip = cx->bm_sts_skip;
719
720 /* Don't check BM_STS, do an unconditional ARB_DIS for S2IDLE */
721 cx->bm_sts_skip = 1;
722 acpi_idle_enter_bm(drv, pr, cx, index);
723 cx->bm_sts_skip = bm_sts_skip;
724
725 return 0;
726 } else {
727 ACPI_FLUSH_CPU_CACHE();
728 }
729 }
730 acpi_idle_do_entry(cx);
731
732 return 0;
733}
734
735static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
736 struct cpuidle_device *dev)
737{
738 int i, count = ACPI_IDLE_STATE_START;
739 struct acpi_processor_cx *cx;
740 struct cpuidle_state *state;
741
742 if (max_cstate == 0)
743 max_cstate = 1;
744
745 for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
746 state = &acpi_idle_driver.states[count];
747 cx = &pr->power.states[i];
748
749 if (!cx->valid)
750 continue;
751
752 per_cpu(acpi_cstate[count], dev->cpu) = cx;
753
754 if (lapic_timer_needs_broadcast(pr, cx))
755 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
756
757 if (cx->type == ACPI_STATE_C3) {
758 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
759 if (pr->flags.bm_check)
760 state->flags |= CPUIDLE_FLAG_RCU_IDLE;
761 }
762
763 count++;
764 if (count == CPUIDLE_STATE_MAX)
765 break;
766 }
767
768 if (!count)
769 return -EINVAL;
770
771 return 0;
772}
773
774static int acpi_processor_setup_cstates(struct acpi_processor *pr)
775{
776 int i, count;
777 struct acpi_processor_cx *cx;
778 struct cpuidle_state *state;
779 struct cpuidle_driver *drv = &acpi_idle_driver;
780
781 if (max_cstate == 0)
782 max_cstate = 1;
783
784 if (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX)) {
785 cpuidle_poll_state_init(drv);
786 count = 1;
787 } else {
788 count = 0;
789 }
790
791 for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
792 cx = &pr->power.states[i];
793
794 if (!cx->valid)
795 continue;
796
797 state = &drv->states[count];
798 snprintf(buf: state->name, CPUIDLE_NAME_LEN, fmt: "C%d", i);
799 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
800 state->exit_latency = cx->latency;
801 state->target_residency = cx->latency * latency_factor;
802 state->enter = acpi_idle_enter;
803
804 state->flags = 0;
805
806 state->enter_dead = acpi_idle_play_dead;
807
808 if (cx->type == ACPI_STATE_C1 || cx->type == ACPI_STATE_C2)
809 drv->safe_state_index = count;
810
811 /*
812 * Halt-induced C1 is not good for ->enter_s2idle, because it
813 * re-enables interrupts on exit. Moreover, C1 is generally not
814 * particularly interesting from the suspend-to-idle angle, so
815 * avoid C1 and the situations in which we may need to fall back
816 * to it altogether.
817 */
818 if (cx->type != ACPI_STATE_C1 && !acpi_idle_fallback_to_c1(pr))
819 state->enter_s2idle = acpi_idle_enter_s2idle;
820
821 count++;
822 if (count == CPUIDLE_STATE_MAX)
823 break;
824 }
825
826 drv->state_count = count;
827
828 if (!count)
829 return -EINVAL;
830
831 return 0;
832}
833
834static inline void acpi_processor_cstate_first_run_checks(void)
835{
836 static int first_run;
837
838 if (first_run)
839 return;
840 dmi_check_system(list: processor_power_dmi_table);
841 max_cstate = acpi_processor_cstate_check(max_cstate);
842 if (max_cstate < ACPI_C_STATES_MAX)
843 pr_notice("processor limited to max C-state %d\n", max_cstate);
844
845 first_run++;
846
847 if (nocst)
848 return;
849
850 acpi_processor_claim_cst_control();
851}
852#else
853
854static inline int disabled_by_idle_boot_param(void) { return 0; }
855static inline void acpi_processor_cstate_first_run_checks(void) { }
856static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
857{
858 return -ENODEV;
859}
860
861static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
862 struct cpuidle_device *dev)
863{
864 return -EINVAL;
865}
866
867static int acpi_processor_setup_cstates(struct acpi_processor *pr)
868{
869 return -EINVAL;
870}
871
872#endif /* CONFIG_ACPI_PROCESSOR_CSTATE */
873
874struct acpi_lpi_states_array {
875 unsigned int size;
876 unsigned int composite_states_size;
877 struct acpi_lpi_state *entries;
878 struct acpi_lpi_state *composite_states[ACPI_PROCESSOR_MAX_POWER];
879};
880
881static int obj_get_integer(union acpi_object *obj, u32 *value)
882{
883 if (obj->type != ACPI_TYPE_INTEGER)
884 return -EINVAL;
885
886 *value = obj->integer.value;
887 return 0;
888}
889
890static int acpi_processor_evaluate_lpi(acpi_handle handle,
891 struct acpi_lpi_states_array *info)
892{
893 acpi_status status;
894 int ret = 0;
895 int pkg_count, state_idx = 1, loop;
896 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
897 union acpi_object *lpi_data;
898 struct acpi_lpi_state *lpi_state;
899
900 status = acpi_evaluate_object(object: handle, pathname: "_LPI", NULL, return_object_buffer: &buffer);
901 if (ACPI_FAILURE(status)) {
902 acpi_handle_debug(handle, "No _LPI, giving up\n");
903 return -ENODEV;
904 }
905
906 lpi_data = buffer.pointer;
907
908 /* There must be at least 4 elements = 3 elements + 1 package */
909 if (!lpi_data || lpi_data->type != ACPI_TYPE_PACKAGE ||
910 lpi_data->package.count < 4) {
911 pr_debug("not enough elements in _LPI\n");
912 ret = -ENODATA;
913 goto end;
914 }
915
916 pkg_count = lpi_data->package.elements[2].integer.value;
917
918 /* Validate number of power states. */
919 if (pkg_count < 1 || pkg_count != lpi_data->package.count - 3) {
920 pr_debug("count given by _LPI is not valid\n");
921 ret = -ENODATA;
922 goto end;
923 }
924
925 lpi_state = kcalloc(pkg_count, sizeof(*lpi_state), GFP_KERNEL);
926 if (!lpi_state) {
927 ret = -ENOMEM;
928 goto end;
929 }
930
931 info->size = pkg_count;
932 info->entries = lpi_state;
933
934 /* LPI States start at index 3 */
935 for (loop = 3; state_idx <= pkg_count; loop++, state_idx++, lpi_state++) {
936 union acpi_object *element, *pkg_elem, *obj;
937
938 element = &lpi_data->package.elements[loop];
939 if (element->type != ACPI_TYPE_PACKAGE || element->package.count < 7)
940 continue;
941
942 pkg_elem = element->package.elements;
943
944 obj = pkg_elem + 6;
945 if (obj->type == ACPI_TYPE_BUFFER) {
946 struct acpi_power_register *reg;
947
948 reg = (struct acpi_power_register *)obj->buffer.pointer;
949 if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO &&
950 reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)
951 continue;
952
953 lpi_state->address = reg->address;
954 lpi_state->entry_method =
955 reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE ?
956 ACPI_CSTATE_FFH : ACPI_CSTATE_SYSTEMIO;
957 } else if (obj->type == ACPI_TYPE_INTEGER) {
958 lpi_state->entry_method = ACPI_CSTATE_INTEGER;
959 lpi_state->address = obj->integer.value;
960 } else {
961 continue;
962 }
963
964 /* elements[7,8] skipped for now i.e. Residency/Usage counter*/
965
966 obj = pkg_elem + 9;
967 if (obj->type == ACPI_TYPE_STRING)
968 strscpy(lpi_state->desc, obj->string.pointer,
969 ACPI_CX_DESC_LEN);
970
971 lpi_state->index = state_idx;
972 if (obj_get_integer(obj: pkg_elem + 0, value: &lpi_state->min_residency)) {
973 pr_debug("No min. residency found, assuming 10 us\n");
974 lpi_state->min_residency = 10;
975 }
976
977 if (obj_get_integer(obj: pkg_elem + 1, value: &lpi_state->wake_latency)) {
978 pr_debug("No wakeup residency found, assuming 10 us\n");
979 lpi_state->wake_latency = 10;
980 }
981
982 if (obj_get_integer(obj: pkg_elem + 2, value: &lpi_state->flags))
983 lpi_state->flags = 0;
984
985 if (obj_get_integer(obj: pkg_elem + 3, value: &lpi_state->arch_flags))
986 lpi_state->arch_flags = 0;
987
988 if (obj_get_integer(obj: pkg_elem + 4, value: &lpi_state->res_cnt_freq))
989 lpi_state->res_cnt_freq = 1;
990
991 if (obj_get_integer(obj: pkg_elem + 5, value: &lpi_state->enable_parent_state))
992 lpi_state->enable_parent_state = 0;
993 }
994
995 acpi_handle_debug(handle, "Found %d power states\n", state_idx);
996end:
997 kfree(objp: buffer.pointer);
998 return ret;
999}
1000
1001/**
1002 * combine_lpi_states - combine local and parent LPI states to form a composite LPI state
1003 *
1004 * @local: local LPI state
1005 * @parent: parent LPI state
1006 * @result: composite LPI state
1007 */
1008static bool combine_lpi_states(struct acpi_lpi_state *local,
1009 struct acpi_lpi_state *parent,
1010 struct acpi_lpi_state *result)
1011{
1012 if (parent->entry_method == ACPI_CSTATE_INTEGER) {
1013 if (!parent->address) /* 0 means autopromotable */
1014 return false;
1015 result->address = local->address + parent->address;
1016 } else {
1017 result->address = parent->address;
1018 }
1019
1020 result->min_residency = max(local->min_residency, parent->min_residency);
1021 result->wake_latency = local->wake_latency + parent->wake_latency;
1022 result->enable_parent_state = parent->enable_parent_state;
1023 result->entry_method = local->entry_method;
1024
1025 result->flags = parent->flags;
1026 result->arch_flags = parent->arch_flags;
1027 result->index = parent->index;
1028
1029 strscpy(result->desc, local->desc, ACPI_CX_DESC_LEN);
1030 strlcat(result->desc, "+", ACPI_CX_DESC_LEN);
1031 strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN);
1032 return true;
1033}
1034
1035#define ACPI_LPI_STATE_FLAGS_ENABLED BIT(0)
1036
1037static void stash_composite_state(struct acpi_lpi_states_array *curr_level,
1038 struct acpi_lpi_state *t)
1039{
1040 curr_level->composite_states[curr_level->composite_states_size++] = t;
1041}
1042
1043static unsigned int flatten_lpi_states(struct acpi_processor *pr,
1044 unsigned int flat_state_cnt,
1045 struct acpi_lpi_states_array *curr_level,
1046 struct acpi_lpi_states_array *prev_level)
1047{
1048 int i, j, state_count = curr_level->size;
1049 struct acpi_lpi_state *p, *t = curr_level->entries;
1050
1051 curr_level->composite_states_size = 0;
1052 for (j = 0; j < state_count; j++, t++) {
1053 struct acpi_lpi_state *flpi;
1054
1055 if (!(t->flags & ACPI_LPI_STATE_FLAGS_ENABLED))
1056 continue;
1057
1058 if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER) {
1059 pr_warn("Limiting number of LPI states to max (%d)\n",
1060 ACPI_PROCESSOR_MAX_POWER);
1061 pr_warn("Please increase ACPI_PROCESSOR_MAX_POWER if needed.\n");
1062 break;
1063 }
1064
1065 flpi = &pr->power.lpi_states[flat_state_cnt];
1066
1067 if (!prev_level) { /* leaf/processor node */
1068 memcpy(to: flpi, from: t, len: sizeof(*t));
1069 stash_composite_state(curr_level, t: flpi);
1070 flat_state_cnt++;
1071 continue;
1072 }
1073
1074 for (i = 0; i < prev_level->composite_states_size; i++) {
1075 p = prev_level->composite_states[i];
1076 if (t->index <= p->enable_parent_state &&
1077 combine_lpi_states(local: p, parent: t, result: flpi)) {
1078 stash_composite_state(curr_level, t: flpi);
1079 flat_state_cnt++;
1080 flpi++;
1081 }
1082 }
1083 }
1084
1085 kfree(objp: curr_level->entries);
1086 return flat_state_cnt;
1087}
1088
1089int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
1090{
1091 return -EOPNOTSUPP;
1092}
1093
1094static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
1095{
1096 int ret, i;
1097 acpi_status status;
1098 acpi_handle handle = pr->handle, pr_ahandle;
1099 struct acpi_device *d = NULL;
1100 struct acpi_lpi_states_array info[2], *tmp, *prev, *curr;
1101 unsigned int state_count;
1102
1103 /* make sure our architecture has support */
1104 ret = acpi_processor_ffh_lpi_probe(cpu: pr->id);
1105 if (ret == -EOPNOTSUPP)
1106 return ret;
1107
1108 if (!osc_pc_lpi_support_confirmed)
1109 return -EOPNOTSUPP;
1110
1111 if (!acpi_has_method(handle, name: "_LPI"))
1112 return -EINVAL;
1113
1114 prev = &info[0];
1115 curr = &info[1];
1116 handle = pr->handle;
1117 ret = acpi_processor_evaluate_lpi(handle, info: prev);
1118 if (ret)
1119 return ret;
1120 state_count = flatten_lpi_states(pr, flat_state_cnt: 0, curr_level: prev, NULL);
1121
1122 status = acpi_get_parent(object: handle, out_handle: &pr_ahandle);
1123 while (ACPI_SUCCESS(status)) {
1124 d = acpi_fetch_acpi_dev(handle: pr_ahandle);
1125 if (!d)
1126 break;
1127
1128 handle = pr_ahandle;
1129
1130 if (strcmp(acpi_device_hid(device: d), ACPI_PROCESSOR_CONTAINER_HID))
1131 break;
1132
1133 /* can be optional ? */
1134 if (!acpi_has_method(handle, name: "_LPI"))
1135 break;
1136
1137 ret = acpi_processor_evaluate_lpi(handle, info: curr);
1138 if (ret)
1139 break;
1140
1141 /* flatten all the LPI states in this level of hierarchy */
1142 state_count = flatten_lpi_states(pr, flat_state_cnt: state_count, curr_level: curr, prev_level: prev);
1143
1144 tmp = prev, prev = curr, curr = tmp;
1145
1146 status = acpi_get_parent(object: handle, out_handle: &pr_ahandle);
1147 }
1148
1149 /* reset the index after flattening */
1150 for (i = 0; i < state_count; i++)
1151 pr->power.lpi_states[i].index = i;
1152
1153 pr->power.count = state_count;
1154
1155 /* Tell driver that _LPI is supported. */
1156 pr->flags.has_lpi = 1;
1157 pr->flags.power = 1;
1158
1159 return 0;
1160}
1161
1162int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi)
1163{
1164 return -ENODEV;
1165}
1166
1167/**
1168 * acpi_idle_lpi_enter - enters an ACPI any LPI state
1169 * @dev: the target CPU
1170 * @drv: cpuidle driver containing cpuidle state info
1171 * @index: index of target state
1172 *
1173 * Return: 0 for success or negative value for error
1174 */
1175static int acpi_idle_lpi_enter(struct cpuidle_device *dev,
1176 struct cpuidle_driver *drv, int index)
1177{
1178 struct acpi_processor *pr;
1179 struct acpi_lpi_state *lpi;
1180
1181 pr = __this_cpu_read(processors);
1182
1183 if (unlikely(!pr))
1184 return -EINVAL;
1185
1186 lpi = &pr->power.lpi_states[index];
1187 if (lpi->entry_method == ACPI_CSTATE_FFH)
1188 return acpi_processor_ffh_lpi_enter(lpi);
1189
1190 return -EINVAL;
1191}
1192
1193static int acpi_processor_setup_lpi_states(struct acpi_processor *pr)
1194{
1195 int i;
1196 struct acpi_lpi_state *lpi;
1197 struct cpuidle_state *state;
1198 struct cpuidle_driver *drv = &acpi_idle_driver;
1199
1200 if (!pr->flags.has_lpi)
1201 return -EOPNOTSUPP;
1202
1203 for (i = 0; i < pr->power.count && i < CPUIDLE_STATE_MAX; i++) {
1204 lpi = &pr->power.lpi_states[i];
1205
1206 state = &drv->states[i];
1207 snprintf(buf: state->name, CPUIDLE_NAME_LEN, fmt: "LPI-%d", i);
1208 strscpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN);
1209 state->exit_latency = lpi->wake_latency;
1210 state->target_residency = lpi->min_residency;
1211 state->flags |= arch_get_idle_state_flags(arch_flags: lpi->arch_flags);
1212 if (i != 0 && lpi->entry_method == ACPI_CSTATE_FFH)
1213 state->flags |= CPUIDLE_FLAG_RCU_IDLE;
1214 state->enter = acpi_idle_lpi_enter;
1215 drv->safe_state_index = i;
1216 }
1217
1218 drv->state_count = i;
1219
1220 return 0;
1221}
1222
1223/**
1224 * acpi_processor_setup_cpuidle_states- prepares and configures cpuidle
1225 * global state data i.e. idle routines
1226 *
1227 * @pr: the ACPI processor
1228 */
1229static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr)
1230{
1231 int i;
1232 struct cpuidle_driver *drv = &acpi_idle_driver;
1233
1234 if (!pr->flags.power_setup_done || !pr->flags.power)
1235 return -EINVAL;
1236
1237 drv->safe_state_index = -1;
1238 for (i = ACPI_IDLE_STATE_START; i < CPUIDLE_STATE_MAX; i++) {
1239 drv->states[i].name[0] = '\0';
1240 drv->states[i].desc[0] = '\0';
1241 }
1242
1243 if (pr->flags.has_lpi)
1244 return acpi_processor_setup_lpi_states(pr);
1245
1246 return acpi_processor_setup_cstates(pr);
1247}
1248
1249/**
1250 * acpi_processor_setup_cpuidle_dev - prepares and configures CPUIDLE
1251 * device i.e. per-cpu data
1252 *
1253 * @pr: the ACPI processor
1254 * @dev : the cpuidle device
1255 */
1256static int acpi_processor_setup_cpuidle_dev(struct acpi_processor *pr,
1257 struct cpuidle_device *dev)
1258{
1259 if (!pr->flags.power_setup_done || !pr->flags.power || !dev)
1260 return -EINVAL;
1261
1262 dev->cpu = pr->id;
1263 if (pr->flags.has_lpi)
1264 return acpi_processor_ffh_lpi_probe(cpu: pr->id);
1265
1266 return acpi_processor_setup_cpuidle_cx(pr, dev);
1267}
1268
1269static int acpi_processor_get_power_info(struct acpi_processor *pr)
1270{
1271 int ret;
1272
1273 ret = acpi_processor_get_lpi_info(pr);
1274 if (ret)
1275 ret = acpi_processor_get_cstate_info(pr);
1276
1277 return ret;
1278}
1279
1280int acpi_processor_hotplug(struct acpi_processor *pr)
1281{
1282 int ret = 0;
1283 struct cpuidle_device *dev;
1284
1285 if (disabled_by_idle_boot_param())
1286 return 0;
1287
1288 if (!pr->flags.power_setup_done)
1289 return -ENODEV;
1290
1291 dev = per_cpu(acpi_cpuidle_device, pr->id);
1292 cpuidle_pause_and_lock();
1293 cpuidle_disable_device(dev);
1294 ret = acpi_processor_get_power_info(pr);
1295 if (!ret && pr->flags.power) {
1296 acpi_processor_setup_cpuidle_dev(pr, dev);
1297 ret = cpuidle_enable_device(dev);
1298 }
1299 cpuidle_resume_and_unlock();
1300
1301 return ret;
1302}
1303
1304int acpi_processor_power_state_has_changed(struct acpi_processor *pr)
1305{
1306 int cpu;
1307 struct acpi_processor *_pr;
1308 struct cpuidle_device *dev;
1309
1310 if (disabled_by_idle_boot_param())
1311 return 0;
1312
1313 if (!pr->flags.power_setup_done)
1314 return -ENODEV;
1315
1316 /*
1317 * FIXME: Design the ACPI notification to make it once per
1318 * system instead of once per-cpu. This condition is a hack
1319 * to make the code that updates C-States be called once.
1320 */
1321
1322 if (pr->id == 0 && cpuidle_get_driver() == &acpi_idle_driver) {
1323
1324 /* Protect against cpu-hotplug */
1325 cpus_read_lock();
1326 cpuidle_pause_and_lock();
1327
1328 /* Disable all cpuidle devices */
1329 for_each_online_cpu(cpu) {
1330 _pr = per_cpu(processors, cpu);
1331 if (!_pr || !_pr->flags.power_setup_done)
1332 continue;
1333 dev = per_cpu(acpi_cpuidle_device, cpu);
1334 cpuidle_disable_device(dev);
1335 }
1336
1337 /* Populate Updated C-state information */
1338 acpi_processor_get_power_info(pr);
1339 acpi_processor_setup_cpuidle_states(pr);
1340
1341 /* Enable all cpuidle devices */
1342 for_each_online_cpu(cpu) {
1343 _pr = per_cpu(processors, cpu);
1344 if (!_pr || !_pr->flags.power_setup_done)
1345 continue;
1346 acpi_processor_get_power_info(pr: _pr);
1347 if (_pr->flags.power) {
1348 dev = per_cpu(acpi_cpuidle_device, cpu);
1349 acpi_processor_setup_cpuidle_dev(pr: _pr, dev);
1350 cpuidle_enable_device(dev);
1351 }
1352 }
1353 cpuidle_resume_and_unlock();
1354 cpus_read_unlock();
1355 }
1356
1357 return 0;
1358}
1359
1360void acpi_processor_register_idle_driver(void)
1361{
1362 struct acpi_processor *pr;
1363 int ret = -ENODEV;
1364 int cpu;
1365
1366 /*
1367 * Acpi idle driver is used by all possible CPUs.
1368 * Install the idle handler by the processor power info of one in them.
1369 * Note that we use previously set idle handler will be used on
1370 * platforms that only support C1.
1371 */
1372 for_each_cpu(cpu, (struct cpumask *)cpu_possible_mask) {
1373 pr = per_cpu(processors, cpu);
1374 if (!pr)
1375 continue;
1376
1377 ret = acpi_processor_get_power_info(pr);
1378 if (!ret) {
1379 pr->flags.power_setup_done = 1;
1380 acpi_processor_setup_cpuidle_states(pr);
1381 break;
1382 }
1383 }
1384
1385 if (ret) {
1386 pr_debug("No ACPI power information from any CPUs.\n");
1387 return;
1388 }
1389
1390 ret = cpuidle_register_driver(drv: &acpi_idle_driver);
1391 if (ret) {
1392 pr_debug("register %s failed.\n", acpi_idle_driver.name);
1393 return;
1394 }
1395 pr_debug("%s registered with cpuidle.\n", acpi_idle_driver.name);
1396}
1397
1398void acpi_processor_unregister_idle_driver(void)
1399{
1400 cpuidle_unregister_driver(drv: &acpi_idle_driver);
1401}
1402
1403void acpi_processor_power_init(struct acpi_processor *pr)
1404{
1405 struct cpuidle_device *dev;
1406
1407 /*
1408 * The code below only works if the current cpuidle driver is the ACPI
1409 * idle driver.
1410 */
1411 if (cpuidle_get_driver() != &acpi_idle_driver)
1412 return;
1413
1414 if (disabled_by_idle_boot_param())
1415 return;
1416
1417 acpi_processor_cstate_first_run_checks();
1418
1419 if (!acpi_processor_get_power_info(pr))
1420 pr->flags.power_setup_done = 1;
1421
1422 if (!pr->flags.power)
1423 return;
1424
1425 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1426 if (!dev)
1427 return;
1428
1429 per_cpu(acpi_cpuidle_device, pr->id) = dev;
1430
1431 acpi_processor_setup_cpuidle_dev(pr, dev);
1432
1433 /*
1434 * Register a cpuidle device for this CPU. The cpuidle driver using
1435 * this device is expected to be registered.
1436 */
1437 if (cpuidle_register_device(dev)) {
1438 per_cpu(acpi_cpuidle_device, pr->id) = NULL;
1439 kfree(objp: dev);
1440 }
1441}
1442
1443void acpi_processor_power_exit(struct acpi_processor *pr)
1444{
1445 struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
1446
1447 if (disabled_by_idle_boot_param())
1448 return;
1449
1450 if (pr->flags.power) {
1451 cpuidle_unregister_device(dev);
1452 kfree(objp: dev);
1453 }
1454
1455 pr->flags.power_setup_done = 0;
1456}
1457
1458MODULE_IMPORT_NS("ACPI_PROCESSOR_IDLE");
1459