1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * processor_throttling.c - Throttling submodule of the ACPI processor driver
4 *
5 * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
6 * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
7 * Copyright (C) 2004 Dominik Brodowski <linux@brodo.de>
8 * Copyright (C) 2004 Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
9 * - Added processor hotplug support
10 */
11
12#define pr_fmt(fmt) "ACPI: " fmt
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/slab.h>
17#include <linux/init.h>
18#include <linux/sched.h>
19#include <linux/cpufreq.h>
20#include <linux/acpi.h>
21#include <linux/uaccess.h>
22#include <acpi/processor.h>
23#include <asm/io.h>
24#ifdef CONFIG_X86
25#include <asm/msr.h>
26#endif
27
28/* ignore_tpc:
29 * 0 -> acpi processor driver doesn't ignore _TPC values
30 * 1 -> acpi processor driver ignores _TPC values
31 */
32static int ignore_tpc;
33module_param(ignore_tpc, int, 0644);
34MODULE_PARM_DESC(ignore_tpc, "Disable broken BIOS _TPC throttling support");
35
36struct throttling_tstate {
37 unsigned int cpu; /* cpu nr */
38 int target_state; /* target T-state */
39};
40
41struct acpi_processor_throttling_arg {
42 struct acpi_processor *pr;
43 int target_state;
44 bool force;
45};
46
47#define THROTTLING_PRECHANGE (1)
48#define THROTTLING_POSTCHANGE (2)
49
50static int acpi_processor_get_throttling(struct acpi_processor *pr);
51static int __acpi_processor_set_throttling(struct acpi_processor *pr,
52 int state, bool force, bool direct);
53
54static int acpi_processor_update_tsd_coord(void)
55{
56 int count_target;
57 int retval = 0;
58 unsigned int i, j;
59 cpumask_var_t covered_cpus;
60 struct acpi_processor *pr, *match_pr;
61 struct acpi_tsd_package *pdomain, *match_pdomain;
62 struct acpi_processor_throttling *pthrottling, *match_pthrottling;
63
64 if (!zalloc_cpumask_var(mask: &covered_cpus, GFP_KERNEL))
65 return -ENOMEM;
66
67 /*
68 * Now that we have _TSD data from all CPUs, lets setup T-state
69 * coordination between all CPUs.
70 */
71 for_each_possible_cpu(i) {
72 pr = per_cpu(processors, i);
73 if (!pr)
74 continue;
75
76 /* Basic validity check for domain info */
77 pthrottling = &(pr->throttling);
78
79 /*
80 * If tsd package for one cpu is invalid, the coordination
81 * among all CPUs is thought as invalid.
82 * Maybe it is ugly.
83 */
84 if (!pthrottling->tsd_valid_flag) {
85 retval = -EINVAL;
86 break;
87 }
88 }
89 if (retval)
90 goto err_ret;
91
92 for_each_possible_cpu(i) {
93 pr = per_cpu(processors, i);
94 if (!pr)
95 continue;
96
97 if (cpumask_test_cpu(cpu: i, cpumask: covered_cpus))
98 continue;
99 pthrottling = &pr->throttling;
100
101 pdomain = &(pthrottling->domain_info);
102 cpumask_set_cpu(cpu: i, dstp: pthrottling->shared_cpu_map);
103 cpumask_set_cpu(cpu: i, dstp: covered_cpus);
104 /*
105 * If the number of processor in the TSD domain is 1, it is
106 * unnecessary to parse the coordination for this CPU.
107 */
108 if (pdomain->num_processors <= 1)
109 continue;
110
111 /* Validate the Domain info */
112 count_target = pdomain->num_processors;
113
114 for_each_possible_cpu(j) {
115 if (i == j)
116 continue;
117
118 match_pr = per_cpu(processors, j);
119 if (!match_pr)
120 continue;
121
122 match_pthrottling = &(match_pr->throttling);
123 match_pdomain = &(match_pthrottling->domain_info);
124 if (match_pdomain->domain != pdomain->domain)
125 continue;
126
127 /* Here i and j are in the same domain.
128 * If two TSD packages have the same domain, they
129 * should have the same num_porcessors and
130 * coordination type. Otherwise it will be regarded
131 * as illegal.
132 */
133 if (match_pdomain->num_processors != count_target) {
134 retval = -EINVAL;
135 goto err_ret;
136 }
137
138 if (pdomain->coord_type != match_pdomain->coord_type) {
139 retval = -EINVAL;
140 goto err_ret;
141 }
142
143 cpumask_set_cpu(cpu: j, dstp: covered_cpus);
144 cpumask_set_cpu(cpu: j, dstp: pthrottling->shared_cpu_map);
145 }
146 for_each_possible_cpu(j) {
147 if (i == j)
148 continue;
149
150 match_pr = per_cpu(processors, j);
151 if (!match_pr)
152 continue;
153
154 match_pthrottling = &(match_pr->throttling);
155 match_pdomain = &(match_pthrottling->domain_info);
156 if (match_pdomain->domain != pdomain->domain)
157 continue;
158
159 /*
160 * If some CPUS have the same domain, they
161 * will have the same shared_cpu_map.
162 */
163 cpumask_copy(dstp: match_pthrottling->shared_cpu_map,
164 srcp: pthrottling->shared_cpu_map);
165 }
166 }
167
168err_ret:
169 free_cpumask_var(mask: covered_cpus);
170
171 for_each_possible_cpu(i) {
172 pr = per_cpu(processors, i);
173 if (!pr)
174 continue;
175
176 /*
177 * Assume no coordination on any error parsing domain info.
178 * The coordination type will be forced as SW_ALL.
179 */
180 if (retval) {
181 pthrottling = &(pr->throttling);
182 cpumask_clear(dstp: pthrottling->shared_cpu_map);
183 cpumask_set_cpu(cpu: i, dstp: pthrottling->shared_cpu_map);
184 pthrottling->shared_type = DOMAIN_COORD_TYPE_SW_ALL;
185 }
186 }
187
188 return retval;
189}
190
191/*
192 * Update the T-state coordination after the _TSD
193 * data for all cpus is obtained.
194 */
195void acpi_processor_throttling_init(void)
196{
197 if (acpi_processor_update_tsd_coord())
198 pr_debug("Assume no T-state coordination\n");
199}
200
201static int acpi_processor_throttling_notifier(unsigned long event, void *data)
202{
203 struct throttling_tstate *p_tstate = data;
204 struct acpi_processor *pr;
205 unsigned int cpu;
206 int target_state;
207 struct acpi_processor_limit *p_limit;
208 struct acpi_processor_throttling *p_throttling;
209
210 cpu = p_tstate->cpu;
211 pr = per_cpu(processors, cpu);
212 if (!pr) {
213 pr_debug("Invalid pr pointer\n");
214 return 0;
215 }
216 if (!pr->flags.throttling) {
217 acpi_handle_debug(pr->handle,
218 "Throttling control unsupported on CPU %d\n",
219 cpu);
220 return 0;
221 }
222 target_state = p_tstate->target_state;
223 p_throttling = &(pr->throttling);
224 switch (event) {
225 case THROTTLING_PRECHANGE:
226 /*
227 * Prechange event is used to choose one proper t-state,
228 * which meets the limits of thermal, user and _TPC.
229 */
230 p_limit = &pr->limit;
231 if (p_limit->thermal.tx > target_state)
232 target_state = p_limit->thermal.tx;
233 if (p_limit->user.tx > target_state)
234 target_state = p_limit->user.tx;
235 if (pr->throttling_platform_limit > target_state)
236 target_state = pr->throttling_platform_limit;
237 if (target_state >= p_throttling->state_count) {
238 pr_warn("Exceed the limit of T-state\n");
239 target_state = p_throttling->state_count - 1;
240 }
241 p_tstate->target_state = target_state;
242 acpi_handle_debug(pr->handle,
243 "PreChange Event: target T-state of CPU %d is T%d\n",
244 cpu, target_state);
245 break;
246 case THROTTLING_POSTCHANGE:
247 /*
248 * Postchange event is only used to update the
249 * T-state flag of acpi_processor_throttling.
250 */
251 p_throttling->state = target_state;
252 acpi_handle_debug(pr->handle,
253 "PostChange Event: CPU %d is switched to T%d\n",
254 cpu, target_state);
255 break;
256 default:
257 pr_warn("Unsupported Throttling notifier event\n");
258 break;
259 }
260
261 return 0;
262}
263
264/*
265 * _TPC - Throttling Present Capabilities
266 */
267static int acpi_processor_get_platform_limit(struct acpi_processor *pr)
268{
269 acpi_status status = 0;
270 unsigned long long tpc = 0;
271
272 if (!pr)
273 return -EINVAL;
274
275 if (ignore_tpc)
276 goto end;
277
278 status = acpi_evaluate_integer(handle: pr->handle, pathname: "_TPC", NULL, data: &tpc);
279 if (ACPI_FAILURE(status)) {
280 if (status != AE_NOT_FOUND)
281 acpi_evaluation_failure_warn(handle: pr->handle, name: "_TPC", status);
282
283 return -ENODEV;
284 }
285
286end:
287 pr->throttling_platform_limit = (int)tpc;
288 return 0;
289}
290
291int acpi_processor_tstate_has_changed(struct acpi_processor *pr)
292{
293 int result = 0;
294 int throttling_limit;
295 int current_state;
296 struct acpi_processor_limit *limit;
297 int target_state;
298
299 if (ignore_tpc)
300 return 0;
301
302 result = acpi_processor_get_platform_limit(pr);
303 if (result) {
304 /* Throttling Limit is unsupported */
305 return result;
306 }
307
308 throttling_limit = pr->throttling_platform_limit;
309 if (throttling_limit >= pr->throttling.state_count) {
310 /* Uncorrect Throttling Limit */
311 return -EINVAL;
312 }
313
314 current_state = pr->throttling.state;
315 if (current_state > throttling_limit) {
316 /*
317 * The current state can meet the requirement of
318 * _TPC limit. But it is reasonable that OSPM changes
319 * t-states from high to low for better performance.
320 * Of course the limit condition of thermal
321 * and user should be considered.
322 */
323 limit = &pr->limit;
324 target_state = throttling_limit;
325 if (limit->thermal.tx > target_state)
326 target_state = limit->thermal.tx;
327 if (limit->user.tx > target_state)
328 target_state = limit->user.tx;
329 } else if (current_state == throttling_limit) {
330 /*
331 * Unnecessary to change the throttling state
332 */
333 return 0;
334 } else {
335 /*
336 * If the current state is lower than the limit of _TPC, it
337 * will be forced to switch to the throttling state defined
338 * by throttling_platfor_limit.
339 * Because the previous state meets with the limit condition
340 * of thermal and user, it is unnecessary to check it again.
341 */
342 target_state = throttling_limit;
343 }
344 return acpi_processor_set_throttling(pr, state: target_state, force: false);
345}
346
347/*
348 * This function is used to reevaluate whether the T-state is valid
349 * after one CPU is onlined/offlined.
350 * It is noted that it won't reevaluate the following properties for
351 * the T-state.
352 * 1. Control method.
353 * 2. the number of supported T-state
354 * 3. TSD domain
355 */
356void acpi_processor_reevaluate_tstate(struct acpi_processor *pr,
357 bool is_dead)
358{
359 int result = 0;
360
361 if (is_dead) {
362 /* When one CPU is offline, the T-state throttling
363 * will be invalidated.
364 */
365 pr->flags.throttling = 0;
366 return;
367 }
368 /* the following is to recheck whether the T-state is valid for
369 * the online CPU
370 */
371 if (!pr->throttling.state_count) {
372 /* If the number of T-state is invalid, it is
373 * invalidated.
374 */
375 pr->flags.throttling = 0;
376 return;
377 }
378 pr->flags.throttling = 1;
379
380 /* Disable throttling (if enabled). We'll let subsequent
381 * policy (e.g.thermal) decide to lower performance if it
382 * so chooses, but for now we'll crank up the speed.
383 */
384
385 result = acpi_processor_get_throttling(pr);
386 if (result)
387 goto end;
388
389 if (pr->throttling.state) {
390 result = acpi_processor_set_throttling(pr, state: 0, force: false);
391 if (result)
392 goto end;
393 }
394
395end:
396 if (result)
397 pr->flags.throttling = 0;
398}
399/*
400 * _PTC - Processor Throttling Control (and status) register location
401 */
402static int acpi_processor_get_throttling_control(struct acpi_processor *pr)
403{
404 int result = 0;
405 acpi_status status = 0;
406 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
407 union acpi_object *ptc = NULL;
408 union acpi_object obj;
409 struct acpi_processor_throttling *throttling;
410
411 status = acpi_evaluate_object(object: pr->handle, pathname: "_PTC", NULL, return_object_buffer: &buffer);
412 if (ACPI_FAILURE(status)) {
413 if (status != AE_NOT_FOUND)
414 acpi_evaluation_failure_warn(handle: pr->handle, name: "_PTC", status);
415
416 return -ENODEV;
417 }
418
419 ptc = (union acpi_object *)buffer.pointer;
420 if (!ptc || (ptc->type != ACPI_TYPE_PACKAGE)
421 || (ptc->package.count != 2)) {
422 pr_err("Invalid _PTC data\n");
423 result = -EFAULT;
424 goto end;
425 }
426
427 /*
428 * control_register
429 */
430
431 obj = ptc->package.elements[0];
432
433 if ((obj.type != ACPI_TYPE_BUFFER)
434 || (obj.buffer.length < sizeof(struct acpi_ptc_register))
435 || (obj.buffer.pointer == NULL)) {
436 pr_err("Invalid _PTC data (control_register)\n");
437 result = -EFAULT;
438 goto end;
439 }
440 memcpy(to: &pr->throttling.control_register, from: obj.buffer.pointer,
441 len: sizeof(struct acpi_ptc_register));
442
443 /*
444 * status_register
445 */
446
447 obj = ptc->package.elements[1];
448
449 if ((obj.type != ACPI_TYPE_BUFFER)
450 || (obj.buffer.length < sizeof(struct acpi_ptc_register))
451 || (obj.buffer.pointer == NULL)) {
452 pr_err("Invalid _PTC data (status_register)\n");
453 result = -EFAULT;
454 goto end;
455 }
456
457 memcpy(to: &pr->throttling.status_register, from: obj.buffer.pointer,
458 len: sizeof(struct acpi_ptc_register));
459
460 throttling = &pr->throttling;
461
462 if ((throttling->control_register.bit_width +
463 throttling->control_register.bit_offset) > 32) {
464 pr_err("Invalid _PTC control register\n");
465 result = -EFAULT;
466 goto end;
467 }
468
469 if ((throttling->status_register.bit_width +
470 throttling->status_register.bit_offset) > 32) {
471 pr_err("Invalid _PTC status register\n");
472 result = -EFAULT;
473 goto end;
474 }
475
476end:
477 kfree(objp: buffer.pointer);
478
479 return result;
480}
481
482/*
483 * _TSS - Throttling Supported States
484 */
485static int acpi_processor_get_throttling_states(struct acpi_processor *pr)
486{
487 int result = 0;
488 acpi_status status = AE_OK;
489 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
490 struct acpi_buffer format = { sizeof("NNNNN"), "NNNNN" };
491 struct acpi_buffer state = { 0, NULL };
492 union acpi_object *tss = NULL;
493 int i;
494
495 status = acpi_evaluate_object(object: pr->handle, pathname: "_TSS", NULL, return_object_buffer: &buffer);
496 if (ACPI_FAILURE(status)) {
497 if (status != AE_NOT_FOUND)
498 acpi_evaluation_failure_warn(handle: pr->handle, name: "_TSS", status);
499
500 return -ENODEV;
501 }
502
503 tss = buffer.pointer;
504 if (!tss || (tss->type != ACPI_TYPE_PACKAGE)) {
505 pr_err("Invalid _TSS data\n");
506 result = -EFAULT;
507 goto end;
508 }
509
510 acpi_handle_debug(pr->handle, "Found %d throttling states\n",
511 tss->package.count);
512
513 pr->throttling.state_count = tss->package.count;
514 pr->throttling.states_tss =
515 kmalloc_array(tss->package.count,
516 sizeof(struct acpi_processor_tx_tss),
517 GFP_KERNEL);
518 if (!pr->throttling.states_tss) {
519 result = -ENOMEM;
520 goto end;
521 }
522
523 for (i = 0; i < pr->throttling.state_count; i++) {
524
525 struct acpi_processor_tx_tss *tx =
526 (struct acpi_processor_tx_tss *)&(pr->throttling.
527 states_tss[i]);
528
529 state.length = sizeof(struct acpi_processor_tx_tss);
530 state.pointer = tx;
531
532 acpi_handle_debug(pr->handle, "Extracting state %d\n", i);
533
534 status = acpi_extract_package(package: &(tss->package.elements[i]),
535 format: &format, buffer: &state);
536 if (ACPI_FAILURE(status)) {
537 acpi_handle_warn(pr->handle, "Invalid _TSS data: %s\n",
538 acpi_format_exception(status));
539 result = -EFAULT;
540 kfree(objp: pr->throttling.states_tss);
541 goto end;
542 }
543
544 if (!tx->freqpercentage) {
545 pr_err("Invalid _TSS data: freq is zero\n");
546 result = -EFAULT;
547 kfree(objp: pr->throttling.states_tss);
548 goto end;
549 }
550 }
551
552end:
553 kfree(objp: buffer.pointer);
554
555 return result;
556}
557
558/*
559 * _TSD - T-State Dependencies
560 */
561static int acpi_processor_get_tsd(struct acpi_processor *pr)
562{
563 int result = 0;
564 acpi_status status = AE_OK;
565 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
566 struct acpi_buffer format = { sizeof("NNNNN"), "NNNNN" };
567 struct acpi_buffer state = { 0, NULL };
568 union acpi_object *tsd = NULL;
569 struct acpi_tsd_package *pdomain;
570 struct acpi_processor_throttling *pthrottling;
571
572 pthrottling = &pr->throttling;
573 pthrottling->tsd_valid_flag = 0;
574
575 status = acpi_evaluate_object(object: pr->handle, pathname: "_TSD", NULL, return_object_buffer: &buffer);
576 if (ACPI_FAILURE(status)) {
577 if (status != AE_NOT_FOUND)
578 acpi_evaluation_failure_warn(handle: pr->handle, name: "_TSD", status);
579
580 return -ENODEV;
581 }
582
583 tsd = buffer.pointer;
584 if (!tsd || (tsd->type != ACPI_TYPE_PACKAGE)) {
585 pr_err("Invalid _TSD data\n");
586 result = -EFAULT;
587 goto end;
588 }
589
590 if (tsd->package.count != 1) {
591 pr_err("Invalid _TSD data\n");
592 result = -EFAULT;
593 goto end;
594 }
595
596 pdomain = &(pr->throttling.domain_info);
597
598 state.length = sizeof(struct acpi_tsd_package);
599 state.pointer = pdomain;
600
601 status = acpi_extract_package(package: &(tsd->package.elements[0]),
602 format: &format, buffer: &state);
603 if (ACPI_FAILURE(status)) {
604 pr_err("Invalid _TSD data\n");
605 result = -EFAULT;
606 goto end;
607 }
608
609 if (pdomain->num_entries != ACPI_TSD_REV0_ENTRIES) {
610 pr_err("Unknown _TSD:num_entries\n");
611 result = -EFAULT;
612 goto end;
613 }
614
615 if (pdomain->revision != ACPI_TSD_REV0_REVISION) {
616 pr_err("Unknown _TSD:revision\n");
617 result = -EFAULT;
618 goto end;
619 }
620
621 pthrottling = &pr->throttling;
622 pthrottling->tsd_valid_flag = 1;
623 pthrottling->shared_type = pdomain->coord_type;
624 cpumask_set_cpu(cpu: pr->id, dstp: pthrottling->shared_cpu_map);
625 /*
626 * If the coordination type is not defined in ACPI spec,
627 * the tsd_valid_flag will be clear and coordination type
628 * will be forecd as DOMAIN_COORD_TYPE_SW_ALL.
629 */
630 if (pdomain->coord_type != DOMAIN_COORD_TYPE_SW_ALL &&
631 pdomain->coord_type != DOMAIN_COORD_TYPE_SW_ANY &&
632 pdomain->coord_type != DOMAIN_COORD_TYPE_HW_ALL) {
633 pthrottling->tsd_valid_flag = 0;
634 pthrottling->shared_type = DOMAIN_COORD_TYPE_SW_ALL;
635 }
636
637end:
638 kfree(objp: buffer.pointer);
639 return result;
640}
641
642/* --------------------------------------------------------------------------
643 Throttling Control
644 -------------------------------------------------------------------------- */
645static int acpi_processor_get_throttling_fadt(struct acpi_processor *pr)
646{
647 int state = 0;
648 u32 value = 0;
649 u32 duty_mask = 0;
650 u32 duty_value = 0;
651
652 if (!pr)
653 return -EINVAL;
654
655 if (!pr->flags.throttling)
656 return -ENODEV;
657
658 /*
659 * We don't care about error returns - we just try to mark
660 * these reserved so that nobody else is confused into thinking
661 * that this region might be unused..
662 *
663 * (In particular, allocating the IO range for Cardbus)
664 */
665 request_region(pr->throttling.address, 6, "ACPI CPU throttle");
666
667 pr->throttling.state = 0;
668
669 duty_mask = pr->throttling.state_count - 1;
670
671 duty_mask <<= pr->throttling.duty_offset;
672
673 local_irq_disable();
674
675 value = inl(port: pr->throttling.address);
676
677 /*
678 * Compute the current throttling state when throttling is enabled
679 * (bit 4 is on).
680 */
681 if (value & 0x10) {
682 duty_value = value & duty_mask;
683 duty_value >>= pr->throttling.duty_offset;
684
685 if (duty_value)
686 state = pr->throttling.state_count - duty_value;
687 }
688
689 pr->throttling.state = state;
690
691 local_irq_enable();
692
693 acpi_handle_debug(pr->handle,
694 "Throttling state is T%d (%d%% throttling applied)\n",
695 state, pr->throttling.states[state].performance);
696
697 return 0;
698}
699
700#ifdef CONFIG_X86
701static int acpi_throttling_rdmsr(u64 *value)
702{
703 u64 msr_high, msr_low;
704 u64 msr = 0;
705 int ret = -1;
706
707 if ((this_cpu_read(cpu_info.x86_vendor) != X86_VENDOR_INTEL) ||
708 !this_cpu_has(X86_FEATURE_ACPI)) {
709 pr_err("HARDWARE addr space,NOT supported yet\n");
710 } else {
711 msr_low = 0;
712 msr_high = 0;
713 rdmsr_safe(MSR_IA32_THERM_CONTROL,
714 (u32 *)&msr_low, (u32 *) &msr_high);
715 msr = (msr_high << 32) | msr_low;
716 *value = (u64) msr;
717 ret = 0;
718 }
719 return ret;
720}
721
722static int acpi_throttling_wrmsr(u64 value)
723{
724 int ret = -1;
725 u64 msr;
726
727 if ((this_cpu_read(cpu_info.x86_vendor) != X86_VENDOR_INTEL) ||
728 !this_cpu_has(X86_FEATURE_ACPI)) {
729 pr_err("HARDWARE addr space,NOT supported yet\n");
730 } else {
731 msr = value;
732 wrmsr_safe(MSR_IA32_THERM_CONTROL,
733 low: msr & 0xffffffff, high: msr >> 32);
734 ret = 0;
735 }
736 return ret;
737}
738#else
739static int acpi_throttling_rdmsr(u64 *value)
740{
741 pr_err("HARDWARE addr space,NOT supported yet\n");
742 return -1;
743}
744
745static int acpi_throttling_wrmsr(u64 value)
746{
747 pr_err("HARDWARE addr space,NOT supported yet\n");
748 return -1;
749}
750#endif
751
752static int acpi_read_throttling_status(struct acpi_processor *pr,
753 u64 *value)
754{
755 u32 bit_width, bit_offset;
756 u32 ptc_value;
757 u64 ptc_mask;
758 struct acpi_processor_throttling *throttling;
759 int ret = -1;
760
761 throttling = &pr->throttling;
762 switch (throttling->status_register.space_id) {
763 case ACPI_ADR_SPACE_SYSTEM_IO:
764 bit_width = throttling->status_register.bit_width;
765 bit_offset = throttling->status_register.bit_offset;
766
767 acpi_os_read_port(address: (acpi_io_address) throttling->status_register.
768 address, value: &ptc_value,
769 width: (u32) (bit_width + bit_offset));
770 ptc_mask = (1 << bit_width) - 1;
771 *value = (u64) ((ptc_value >> bit_offset) & ptc_mask);
772 ret = 0;
773 break;
774 case ACPI_ADR_SPACE_FIXED_HARDWARE:
775 ret = acpi_throttling_rdmsr(value);
776 break;
777 default:
778 pr_err("Unknown addr space %d\n",
779 (u32) (throttling->status_register.space_id));
780 }
781 return ret;
782}
783
784static int acpi_write_throttling_state(struct acpi_processor *pr,
785 u64 value)
786{
787 u32 bit_width, bit_offset;
788 u64 ptc_value;
789 u64 ptc_mask;
790 struct acpi_processor_throttling *throttling;
791 int ret = -1;
792
793 throttling = &pr->throttling;
794 switch (throttling->control_register.space_id) {
795 case ACPI_ADR_SPACE_SYSTEM_IO:
796 bit_width = throttling->control_register.bit_width;
797 bit_offset = throttling->control_register.bit_offset;
798 ptc_mask = (1 << bit_width) - 1;
799 ptc_value = value & ptc_mask;
800
801 acpi_os_write_port(address: (acpi_io_address) throttling->
802 control_register.address,
803 value: (u32) (ptc_value << bit_offset),
804 width: (u32) (bit_width + bit_offset));
805 ret = 0;
806 break;
807 case ACPI_ADR_SPACE_FIXED_HARDWARE:
808 ret = acpi_throttling_wrmsr(value);
809 break;
810 default:
811 pr_err("Unknown addr space %d\n",
812 (u32) (throttling->control_register.space_id));
813 }
814 return ret;
815}
816
817static int acpi_get_throttling_state(struct acpi_processor *pr,
818 u64 value)
819{
820 int i;
821
822 for (i = 0; i < pr->throttling.state_count; i++) {
823 struct acpi_processor_tx_tss *tx =
824 (struct acpi_processor_tx_tss *)&(pr->throttling.
825 states_tss[i]);
826 if (tx->control == value)
827 return i;
828 }
829 return -1;
830}
831
832static int acpi_get_throttling_value(struct acpi_processor *pr,
833 int state, u64 *value)
834{
835 int ret = -1;
836
837 if (state >= 0 && state <= pr->throttling.state_count) {
838 struct acpi_processor_tx_tss *tx =
839 (struct acpi_processor_tx_tss *)&(pr->throttling.
840 states_tss[state]);
841 *value = tx->control;
842 ret = 0;
843 }
844 return ret;
845}
846
847static int acpi_processor_get_throttling_ptc(struct acpi_processor *pr)
848{
849 int state = 0;
850 int ret;
851 u64 value;
852
853 if (!pr)
854 return -EINVAL;
855
856 if (!pr->flags.throttling)
857 return -ENODEV;
858
859 pr->throttling.state = 0;
860
861 value = 0;
862 ret = acpi_read_throttling_status(pr, value: &value);
863 if (ret >= 0) {
864 state = acpi_get_throttling_state(pr, value);
865 if (state == -1) {
866 acpi_handle_debug(pr->handle,
867 "Invalid throttling state, reset\n");
868 state = 0;
869 ret = __acpi_processor_set_throttling(pr, state, force: true,
870 direct: true);
871 if (ret)
872 return ret;
873 }
874 pr->throttling.state = state;
875 }
876
877 return 0;
878}
879
880static long __acpi_processor_get_throttling(void *data)
881{
882 struct acpi_processor *pr = data;
883
884 return pr->throttling.acpi_processor_get_throttling(pr);
885}
886
887static int acpi_processor_get_throttling(struct acpi_processor *pr)
888{
889 if (!pr)
890 return -EINVAL;
891
892 if (!pr->flags.throttling)
893 return -ENODEV;
894
895 /*
896 * This is either called from the CPU hotplug callback of
897 * processor_driver or via the ACPI probe function. In the latter
898 * case the CPU is not guaranteed to be online. Both call sites are
899 * protected against CPU hotplug.
900 */
901 if (!cpu_online(cpu: pr->id))
902 return -ENODEV;
903
904 return call_on_cpu(cpu: pr->id, fn: __acpi_processor_get_throttling, arg: pr, direct: false);
905}
906
907static int acpi_processor_get_fadt_info(struct acpi_processor *pr)
908{
909 int i, step;
910
911 if (!pr->throttling.address) {
912 acpi_handle_debug(pr->handle, "No throttling register\n");
913 return -EINVAL;
914 } else if (!pr->throttling.duty_width) {
915 acpi_handle_debug(pr->handle, "No throttling states\n");
916 return -EINVAL;
917 }
918 /* TBD: Support duty_cycle values that span bit 4. */
919 else if ((pr->throttling.duty_offset + pr->throttling.duty_width) > 4) {
920 pr_warn("duty_cycle spans bit 4\n");
921 return -EINVAL;
922 }
923
924 pr->throttling.state_count = 1 << acpi_gbl_FADT.duty_width;
925
926 /*
927 * Compute state values. Note that throttling displays a linear power
928 * performance relationship (at 50% performance the CPU will consume
929 * 50% power). Values are in 1/10th of a percent to preserve accuracy.
930 */
931
932 step = (1000 / pr->throttling.state_count);
933
934 for (i = 0; i < pr->throttling.state_count; i++) {
935 pr->throttling.states[i].performance = 1000 - step * i;
936 pr->throttling.states[i].power = 1000 - step * i;
937 }
938 return 0;
939}
940
941static int acpi_processor_set_throttling_fadt(struct acpi_processor *pr,
942 int state, bool force)
943{
944 u32 value = 0;
945 u32 duty_mask = 0;
946 u32 duty_value = 0;
947
948 if (!pr)
949 return -EINVAL;
950
951 if ((state < 0) || (state > (pr->throttling.state_count - 1)))
952 return -EINVAL;
953
954 if (!pr->flags.throttling)
955 return -ENODEV;
956
957 if (!force && (state == pr->throttling.state))
958 return 0;
959
960 if (state < pr->throttling_platform_limit)
961 return -EPERM;
962 /*
963 * Calculate the duty_value and duty_mask.
964 */
965 if (state) {
966 duty_value = pr->throttling.state_count - state;
967
968 duty_value <<= pr->throttling.duty_offset;
969
970 /* Used to clear all duty_value bits */
971 duty_mask = pr->throttling.state_count - 1;
972
973 duty_mask <<= acpi_gbl_FADT.duty_offset;
974 duty_mask = ~duty_mask;
975 }
976
977 local_irq_disable();
978
979 /*
980 * Disable throttling by writing a 0 to bit 4. Note that we must
981 * turn it off before you can change the duty_value.
982 */
983 value = inl(port: pr->throttling.address);
984 if (value & 0x10) {
985 value &= 0xFFFFFFEF;
986 outl(value, port: pr->throttling.address);
987 }
988
989 /*
990 * Write the new duty_value and then enable throttling. Note
991 * that a state value of 0 leaves throttling disabled.
992 */
993 if (state) {
994 value &= duty_mask;
995 value |= duty_value;
996 outl(value, port: pr->throttling.address);
997
998 value |= 0x00000010;
999 outl(value, port: pr->throttling.address);
1000 }
1001
1002 pr->throttling.state = state;
1003
1004 local_irq_enable();
1005
1006 acpi_handle_debug(pr->handle,
1007 "Throttling state set to T%d (%d%%)\n", state,
1008 (pr->throttling.states[state].performance ? pr->
1009 throttling.states[state].performance / 10 : 0));
1010
1011 return 0;
1012}
1013
1014static int acpi_processor_set_throttling_ptc(struct acpi_processor *pr,
1015 int state, bool force)
1016{
1017 int ret;
1018 u64 value;
1019
1020 if (!pr)
1021 return -EINVAL;
1022
1023 if ((state < 0) || (state > (pr->throttling.state_count - 1)))
1024 return -EINVAL;
1025
1026 if (!pr->flags.throttling)
1027 return -ENODEV;
1028
1029 if (!force && (state == pr->throttling.state))
1030 return 0;
1031
1032 if (state < pr->throttling_platform_limit)
1033 return -EPERM;
1034
1035 value = 0;
1036 ret = acpi_get_throttling_value(pr, state, value: &value);
1037 if (ret >= 0) {
1038 acpi_write_throttling_state(pr, value);
1039 pr->throttling.state = state;
1040 }
1041
1042 return 0;
1043}
1044
1045static long acpi_processor_throttling_fn(void *data)
1046{
1047 struct acpi_processor_throttling_arg *arg = data;
1048 struct acpi_processor *pr = arg->pr;
1049
1050 return pr->throttling.acpi_processor_set_throttling(pr,
1051 arg->target_state, arg->force);
1052}
1053
1054static int __acpi_processor_set_throttling(struct acpi_processor *pr,
1055 int state, bool force, bool direct)
1056{
1057 int ret = 0;
1058 unsigned int i;
1059 struct acpi_processor *match_pr;
1060 struct acpi_processor_throttling *p_throttling;
1061 struct acpi_processor_throttling_arg arg;
1062 struct throttling_tstate t_state;
1063
1064 if (!pr)
1065 return -EINVAL;
1066
1067 if (!pr->flags.throttling)
1068 return -ENODEV;
1069
1070 if ((state < 0) || (state > (pr->throttling.state_count - 1)))
1071 return -EINVAL;
1072
1073 if (cpu_is_offline(pr->id)) {
1074 /*
1075 * the cpu pointed by pr->id is offline. Unnecessary to change
1076 * the throttling state any more.
1077 */
1078 return -ENODEV;
1079 }
1080
1081 t_state.target_state = state;
1082 p_throttling = &(pr->throttling);
1083
1084 /*
1085 * The throttling notifier will be called for every
1086 * affected cpu in order to get one proper T-state.
1087 * The notifier event is THROTTLING_PRECHANGE.
1088 */
1089 for_each_cpu_and(i, cpu_online_mask, p_throttling->shared_cpu_map) {
1090 t_state.cpu = i;
1091 acpi_processor_throttling_notifier(THROTTLING_PRECHANGE,
1092 data: &t_state);
1093 }
1094 /*
1095 * The function of acpi_processor_set_throttling will be called
1096 * to switch T-state. If the coordination type is SW_ALL or HW_ALL,
1097 * it is necessary to call it for every affected cpu. Otherwise
1098 * it can be called only for the cpu pointed by pr.
1099 */
1100 if (p_throttling->shared_type == DOMAIN_COORD_TYPE_SW_ANY) {
1101 arg.pr = pr;
1102 arg.target_state = state;
1103 arg.force = force;
1104 ret = call_on_cpu(cpu: pr->id, fn: acpi_processor_throttling_fn, arg: &arg,
1105 direct);
1106 } else {
1107 /*
1108 * When the T-state coordination is SW_ALL or HW_ALL,
1109 * it is necessary to set T-state for every affected
1110 * cpus.
1111 */
1112 for_each_cpu_and(i, cpu_online_mask,
1113 p_throttling->shared_cpu_map) {
1114 match_pr = per_cpu(processors, i);
1115 /*
1116 * If the pointer is invalid, we will report the
1117 * error message and continue.
1118 */
1119 if (!match_pr) {
1120 acpi_handle_debug(pr->handle,
1121 "Invalid Pointer for CPU %d\n", i);
1122 continue;
1123 }
1124 /*
1125 * If the throttling control is unsupported on CPU i,
1126 * we will report the error message and continue.
1127 */
1128 if (!match_pr->flags.throttling) {
1129 acpi_handle_debug(pr->handle,
1130 "Throttling Control unsupported on CPU %d\n", i);
1131 continue;
1132 }
1133
1134 arg.pr = match_pr;
1135 arg.target_state = state;
1136 arg.force = force;
1137 ret = call_on_cpu(cpu: pr->id, fn: acpi_processor_throttling_fn,
1138 arg: &arg, direct);
1139 }
1140 }
1141 /*
1142 * After the set_throttling is called, the
1143 * throttling notifier is called for every
1144 * affected cpu to update the T-states.
1145 * The notifier event is THROTTLING_POSTCHANGE
1146 */
1147 for_each_cpu_and(i, cpu_online_mask, p_throttling->shared_cpu_map) {
1148 t_state.cpu = i;
1149 acpi_processor_throttling_notifier(THROTTLING_POSTCHANGE,
1150 data: &t_state);
1151 }
1152
1153 return ret;
1154}
1155
1156int acpi_processor_set_throttling(struct acpi_processor *pr, int state,
1157 bool force)
1158{
1159 return __acpi_processor_set_throttling(pr, state, force, direct: false);
1160}
1161
1162int acpi_processor_get_throttling_info(struct acpi_processor *pr)
1163{
1164 int result = 0;
1165 struct acpi_processor_throttling *pthrottling;
1166
1167 acpi_handle_debug(pr->handle,
1168 "pblk_address[0x%08x] duty_offset[%d] duty_width[%d]\n",
1169 pr->throttling.address,
1170 pr->throttling.duty_offset,
1171 pr->throttling.duty_width);
1172
1173 /*
1174 * Evaluate _PTC, _TSS and _TPC
1175 * They must all be present or none of them can be used.
1176 */
1177 if (acpi_processor_get_throttling_control(pr) ||
1178 acpi_processor_get_throttling_states(pr) ||
1179 acpi_processor_get_platform_limit(pr)) {
1180 pr->throttling.acpi_processor_get_throttling =
1181 &acpi_processor_get_throttling_fadt;
1182 pr->throttling.acpi_processor_set_throttling =
1183 &acpi_processor_set_throttling_fadt;
1184 if (acpi_processor_get_fadt_info(pr))
1185 return 0;
1186 } else {
1187 pr->throttling.acpi_processor_get_throttling =
1188 &acpi_processor_get_throttling_ptc;
1189 pr->throttling.acpi_processor_set_throttling =
1190 &acpi_processor_set_throttling_ptc;
1191 }
1192
1193 /*
1194 * If TSD package for one CPU can't be parsed successfully, it means
1195 * that this CPU will have no coordination with other CPUs.
1196 */
1197 if (acpi_processor_get_tsd(pr)) {
1198 pthrottling = &pr->throttling;
1199 pthrottling->tsd_valid_flag = 0;
1200 cpumask_set_cpu(cpu: pr->id, dstp: pthrottling->shared_cpu_map);
1201 pthrottling->shared_type = DOMAIN_COORD_TYPE_SW_ALL;
1202 }
1203
1204 /*
1205 * PIIX4 Errata: We don't support throttling on the original PIIX4.
1206 * This shouldn't be an issue as few (if any) mobile systems ever
1207 * used this part.
1208 */
1209 if (errata.piix4.throttle) {
1210 acpi_handle_debug(pr->handle,
1211 "Throttling not supported on PIIX4 A- or B-step\n");
1212 return 0;
1213 }
1214
1215 acpi_handle_debug(pr->handle, "Found %d throttling states\n",
1216 pr->throttling.state_count);
1217
1218 pr->flags.throttling = 1;
1219
1220 /*
1221 * Disable throttling (if enabled). We'll let subsequent policy (e.g.
1222 * thermal) decide to lower performance if it so chooses, but for now
1223 * we'll crank up the speed.
1224 */
1225
1226 result = acpi_processor_get_throttling(pr);
1227 if (result)
1228 goto end;
1229
1230 if (pr->throttling.state) {
1231 acpi_handle_debug(pr->handle,
1232 "Disabling throttling (was T%d)\n",
1233 pr->throttling.state);
1234 result = acpi_processor_set_throttling(pr, state: 0, force: false);
1235 if (result)
1236 goto end;
1237 }
1238
1239end:
1240 if (result)
1241 pr->flags.throttling = 0;
1242
1243 return result;
1244}
1245
1246