1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Intel Transactional Synchronization Extensions (TSX) control.
4 *
5 * Copyright (C) 2019-2021 Intel Corporation
6 *
7 * Author:
8 * Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
9 */
10
11#include <linux/cpufeature.h>
12
13#include <asm/cmdline.h>
14#include <asm/cpu.h>
15#include <asm/msr.h>
16
17#include "cpu.h"
18
19#undef pr_fmt
20#define pr_fmt(fmt) "tsx: " fmt
21
22enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED;
23
24static void tsx_disable(void)
25{
26 u64 tsx;
27
28 rdmsrq(MSR_IA32_TSX_CTRL, tsx);
29
30 /* Force all transactions to immediately abort */
31 tsx |= TSX_CTRL_RTM_DISABLE;
32
33 /*
34 * Ensure TSX support is not enumerated in CPUID.
35 * This is visible to userspace and will ensure they
36 * do not waste resources trying TSX transactions that
37 * will always abort.
38 */
39 tsx |= TSX_CTRL_CPUID_CLEAR;
40
41 wrmsrq(MSR_IA32_TSX_CTRL, val: tsx);
42}
43
44static void tsx_enable(void)
45{
46 u64 tsx;
47
48 rdmsrq(MSR_IA32_TSX_CTRL, tsx);
49
50 /* Enable the RTM feature in the cpu */
51 tsx &= ~TSX_CTRL_RTM_DISABLE;
52
53 /*
54 * Ensure TSX support is enumerated in CPUID.
55 * This is visible to userspace and will ensure they
56 * can enumerate and use the TSX feature.
57 */
58 tsx &= ~TSX_CTRL_CPUID_CLEAR;
59
60 wrmsrq(MSR_IA32_TSX_CTRL, val: tsx);
61}
62
63static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
64{
65 if (boot_cpu_has_bug(X86_BUG_TAA))
66 return TSX_CTRL_DISABLE;
67
68 return TSX_CTRL_ENABLE;
69}
70
71/*
72 * Disabling TSX is not a trivial business.
73 *
74 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
75 * which says that TSX is practically disabled (all transactions are
76 * aborted by default). When that bit is set, the kernel unconditionally
77 * disables TSX.
78 *
79 * In order to do that, however, it needs to dance a bit:
80 *
81 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
82 * the MSR is present only when *two* CPUID bits are set:
83 *
84 * - X86_FEATURE_RTM_ALWAYS_ABORT
85 * - X86_FEATURE_TSX_FORCE_ABORT
86 *
87 * 2. The second method is for CPUs which do not have the above-mentioned
88 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
89 * through that one. Those CPUs can also have the initially mentioned
90 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
91 * applies: TSX gets disabled unconditionally.
92 *
93 * When either of the two methods are present, the kernel disables TSX and
94 * clears the respective RTM and HLE feature flags.
95 *
96 * An additional twist in the whole thing presents late microcode loading
97 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
98 * bit to be set after the update.
99 *
100 * A subsequent hotplug operation on any logical CPU except the BSP will
101 * cause for the supported CPUID feature bits to get re-detected and, if
102 * RTM and HLE get cleared all of a sudden, but, userspace did consult
103 * them before the update, then funny explosions will happen. Long story
104 * short: the kernel doesn't modify CPUID feature bits after booting.
105 *
106 * That's why, this function's call in init_intel() doesn't clear the
107 * feature flags.
108 */
109static void tsx_clear_cpuid(void)
110{
111 u64 msr;
112
113 /*
114 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID
115 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present.
116 */
117 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
118 boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
119 rdmsrq(MSR_TSX_FORCE_ABORT, msr);
120 msr |= MSR_TFA_TSX_CPUID_CLEAR;
121 wrmsrq(MSR_TSX_FORCE_ABORT, val: msr);
122 } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) {
123 rdmsrq(MSR_IA32_TSX_CTRL, msr);
124 msr |= TSX_CTRL_CPUID_CLEAR;
125 wrmsrq(MSR_IA32_TSX_CTRL, val: msr);
126 }
127}
128
129/*
130 * Disable TSX development mode
131 *
132 * When the microcode released in Feb 2022 is applied, TSX will be disabled by
133 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123
134 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is
135 * not recommended for production deployments. In particular, applying MD_CLEAR
136 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient
137 * execution attack may not be effective on these processors when Intel TSX is
138 * enabled with updated microcode.
139 */
140static void tsx_dev_mode_disable(void)
141{
142 u64 mcu_opt_ctrl;
143
144 /* Check if RTM_ALLOW exists */
145 if (!boot_cpu_has_bug(X86_BUG_TAA) ||
146 !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) ||
147 !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL))
148 return;
149
150 rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
151
152 if (mcu_opt_ctrl & RTM_ALLOW) {
153 mcu_opt_ctrl &= ~RTM_ALLOW;
154 wrmsrq(MSR_IA32_MCU_OPT_CTRL, val: mcu_opt_ctrl);
155 setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
156 }
157}
158
159void __init tsx_init(void)
160{
161 char arg[5] = {};
162 int ret;
163
164 tsx_dev_mode_disable();
165
166 /*
167 * Hardware will always abort a TSX transaction when the CPUID bit
168 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
169 * CPUID.RTM and CPUID.HLE bits. Clear them here.
170 */
171 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
172 tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
173 tsx_clear_cpuid();
174 setup_clear_cpu_cap(X86_FEATURE_RTM);
175 setup_clear_cpu_cap(X86_FEATURE_HLE);
176 return;
177 }
178
179 /*
180 * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this
181 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES.
182 *
183 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a
184 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES
185 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get
186 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus,
187 * tsx= cmdline requests will do nothing on CPUs without
188 * MSR_IA32_TSX_CTRL support.
189 */
190 if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) {
191 setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL);
192 } else {
193 tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED;
194 return;
195 }
196
197 ret = cmdline_find_option(cmdline_ptr: boot_command_line, option: "tsx", buffer: arg, bufsize: sizeof(arg));
198 if (ret >= 0) {
199 if (!strcmp(arg, "on")) {
200 tsx_ctrl_state = TSX_CTRL_ENABLE;
201 } else if (!strcmp(arg, "off")) {
202 tsx_ctrl_state = TSX_CTRL_DISABLE;
203 } else if (!strcmp(arg, "auto")) {
204 tsx_ctrl_state = x86_get_tsx_auto_mode();
205 } else {
206 tsx_ctrl_state = TSX_CTRL_DISABLE;
207 pr_err("invalid option, defaulting to off\n");
208 }
209 } else {
210 /* tsx= not provided */
211 if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO))
212 tsx_ctrl_state = x86_get_tsx_auto_mode();
213 else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF))
214 tsx_ctrl_state = TSX_CTRL_DISABLE;
215 else
216 tsx_ctrl_state = TSX_CTRL_ENABLE;
217 }
218
219 if (tsx_ctrl_state == TSX_CTRL_DISABLE) {
220 tsx_disable();
221
222 /*
223 * tsx_disable() will change the state of the RTM and HLE CPUID
224 * bits. Clear them here since they are now expected to be not
225 * set.
226 */
227 setup_clear_cpu_cap(X86_FEATURE_RTM);
228 setup_clear_cpu_cap(X86_FEATURE_HLE);
229 } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) {
230
231 /*
232 * HW defaults TSX to be enabled at bootup.
233 * We may still need the TSX enable support
234 * during init for special cases like
235 * kexec after TSX is disabled.
236 */
237 tsx_enable();
238
239 /*
240 * tsx_enable() will change the state of the RTM and HLE CPUID
241 * bits. Force them here since they are now expected to be set.
242 */
243 setup_force_cpu_cap(X86_FEATURE_RTM);
244 setup_force_cpu_cap(X86_FEATURE_HLE);
245 }
246}
247
248void tsx_ap_init(void)
249{
250 tsx_dev_mode_disable();
251
252 if (tsx_ctrl_state == TSX_CTRL_ENABLE)
253 tsx_enable();
254 else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
255 tsx_disable();
256 else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
257 /* See comment over that function for more details. */
258 tsx_clear_cpuid();
259}
260