1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * itmt.c: Support Intel Turbo Boost Max Technology 3.0
4 *
5 * (C) Copyright 2016 Intel Corporation
6 * Author: Tim Chen <tim.c.chen@linux.intel.com>
7 *
8 * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
9 * the maximum turbo frequencies of some cores in a CPU package may be
10 * higher than for the other cores in the same package. In that case,
11 * better performance can be achieved by making the scheduler prefer
12 * to run tasks on the CPUs with higher max turbo frequencies.
13 *
14 * This file provides functions and data structures for enabling the
15 * scheduler to favor scheduling on cores can be boosted to a higher
16 * frequency under ITMT.
17 */
18
19#include <linux/sched.h>
20#include <linux/cpumask.h>
21#include <linux/cpuset.h>
22#include <linux/debugfs.h>
23#include <linux/mutex.h>
24#include <linux/sysctl.h>
25#include <linux/nodemask.h>
26
27static DEFINE_MUTEX(itmt_update_mutex);
28DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
29
30/* Boolean to track if system has ITMT capabilities */
31static bool __read_mostly sched_itmt_capable;
32
33/*
34 * Boolean to control whether we want to move processes to cpu capable
35 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
36 * Technology 3.0.
37 *
38 * It can be set via /sys/kernel/debug/x86/sched_itmt_enabled
39 */
40bool __read_mostly sysctl_sched_itmt_enabled;
41
42static ssize_t sched_itmt_enabled_write(struct file *filp,
43 const char __user *ubuf,
44 size_t cnt, loff_t *ppos)
45{
46 ssize_t result;
47 bool orig;
48
49 guard(mutex)(T: &itmt_update_mutex);
50
51 orig = sysctl_sched_itmt_enabled;
52 result = debugfs_write_file_bool(file: filp, user_buf: ubuf, count: cnt, ppos);
53
54 if (sysctl_sched_itmt_enabled != orig) {
55 x86_topology_update = true;
56 rebuild_sched_domains();
57 }
58
59 return result;
60}
61
62static int sched_core_priority_show(struct seq_file *s, void *unused)
63{
64 int cpu;
65
66 seq_puts(m: s, s: "CPU #\tPriority\n");
67 for_each_possible_cpu(cpu)
68 seq_printf(m: s, fmt: "%d\t%d\n", cpu, arch_asym_cpu_priority(cpu));
69
70 return 0;
71}
72DEFINE_SHOW_ATTRIBUTE(sched_core_priority);
73
74static const struct file_operations dfs_sched_itmt_fops = {
75 .read = debugfs_read_file_bool,
76 .write = sched_itmt_enabled_write,
77 .open = simple_open,
78 .llseek = default_llseek,
79};
80
81static struct dentry *dfs_sched_itmt;
82static struct dentry *dfs_sched_core_prio;
83
84/**
85 * sched_set_itmt_support() - Indicate platform supports ITMT
86 *
87 * This function is used by the OS to indicate to scheduler that the platform
88 * is capable of supporting the ITMT feature.
89 *
90 * The current scheme has the pstate driver detects if the system
91 * is ITMT capable and call sched_set_itmt_support.
92 *
93 * This must be done only after sched_set_itmt_core_prio
94 * has been called to set the cpus' priorities.
95 * It must not be called with cpu hot plug lock
96 * held as we need to acquire the lock to rebuild sched domains
97 * later.
98 *
99 * Return: 0 on success
100 */
101int sched_set_itmt_support(void)
102{
103 guard(mutex)(T: &itmt_update_mutex);
104
105 if (sched_itmt_capable)
106 return 0;
107
108 dfs_sched_itmt = debugfs_create_file_unsafe(name: "sched_itmt_enabled",
109 mode: 0644,
110 parent: arch_debugfs_dir,
111 data: &sysctl_sched_itmt_enabled,
112 fops: &dfs_sched_itmt_fops);
113 if (IS_ERR_OR_NULL(ptr: dfs_sched_itmt)) {
114 dfs_sched_itmt = NULL;
115 return -ENOMEM;
116 }
117
118 dfs_sched_core_prio = debugfs_create_file("sched_core_priority", 0644,
119 arch_debugfs_dir, NULL,
120 &sched_core_priority_fops);
121 if (IS_ERR_OR_NULL(ptr: dfs_sched_core_prio)) {
122 dfs_sched_core_prio = NULL;
123 return -ENOMEM;
124 }
125
126 sched_itmt_capable = true;
127
128 sysctl_sched_itmt_enabled = 1;
129
130 x86_topology_update = true;
131 rebuild_sched_domains();
132
133 return 0;
134}
135
136/**
137 * sched_clear_itmt_support() - Revoke platform's support of ITMT
138 *
139 * This function is used by the OS to indicate that it has
140 * revoked the platform's support of ITMT feature.
141 *
142 * It must not be called with cpu hot plug lock
143 * held as we need to acquire the lock to rebuild sched domains
144 * later.
145 */
146void sched_clear_itmt_support(void)
147{
148 guard(mutex)(T: &itmt_update_mutex);
149
150 if (!sched_itmt_capable)
151 return;
152
153 sched_itmt_capable = false;
154
155 debugfs_remove(dentry: dfs_sched_itmt);
156 dfs_sched_itmt = NULL;
157 debugfs_remove(dentry: dfs_sched_core_prio);
158 dfs_sched_core_prio = NULL;
159
160 if (sysctl_sched_itmt_enabled) {
161 /* disable sched_itmt if we are no longer ITMT capable */
162 sysctl_sched_itmt_enabled = 0;
163 x86_topology_update = true;
164 rebuild_sched_domains();
165 }
166}
167
168int arch_asym_cpu_priority(int cpu)
169{
170 return per_cpu(sched_core_priority, cpu);
171}
172
173/**
174 * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
175 * @prio: Priority of @cpu
176 * @cpu: The CPU number
177 *
178 * The pstate driver will find out the max boost frequency
179 * and call this function to set a priority proportional
180 * to the max boost frequency. CPUs with higher boost
181 * frequency will receive higher priority.
182 *
183 * No need to rebuild sched domain after updating
184 * the CPU priorities. The sched domains have no
185 * dependency on CPU priorities.
186 */
187void sched_set_itmt_core_prio(int prio, int cpu)
188{
189 per_cpu(sched_core_priority, cpu) = prio;
190}
191