1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "i915_wait_util.h"
8#include "intel_gt.h"
9#include "intel_gt_mcr.h"
10#include "intel_gt_print.h"
11#include "intel_gt_regs.h"
12
13/**
14 * DOC: GT Multicast/Replicated (MCR) Register Support
15 *
16 * Some GT registers are designed as "multicast" or "replicated" registers:
17 * multiple instances of the same register share a single MMIO offset. MCR
18 * registers are generally used when the hardware needs to potentially track
19 * independent values of a register per hardware unit (e.g., per-subslice,
20 * per-L3bank, etc.). The specific types of replication that exist vary
21 * per-platform.
22 *
23 * MMIO accesses to MCR registers are controlled according to the settings
24 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
25 * registers can be done in either a (i.e., a single write updates all
26 * instances of the register to the same value) or unicast (a write updates only
27 * one specific instance). Reads of MCR registers always operate in a unicast
28 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
29 * Selection of a specific MCR instance for unicast operations is referred to
30 * as "steering."
31 *
32 * If MCR register operations are steered toward a hardware unit that is
33 * fused off or currently powered down due to power gating, the MMIO operation
34 * is "terminated" by the hardware. Terminated read operations will return a
35 * value of zero and terminated unicast write operations will be silently
36 * ignored.
37 */
38
39#define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering)
40
41static const char * const intel_steering_types[] = {
42 "L3BANK",
43 "MSLICE",
44 "LNCF",
45 "GAM",
46 "DSS",
47 "OADDRM",
48 "INSTANCE 0",
49};
50
51static const struct intel_mmio_range icl_l3bank_steering_table[] = {
52 { 0x00B100, 0x00B3FF },
53 {},
54};
55
56/*
57 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
58 * are of a "GAM" subclass that has special rules. Thus we use a separate
59 * GAM table farther down for those.
60 */
61static const struct intel_mmio_range dg2_mslice_steering_table[] = {
62 { 0x00DD00, 0x00DDFF },
63 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
64 {},
65};
66
67static const struct intel_mmio_range dg2_lncf_steering_table[] = {
68 { 0x00B000, 0x00B0FF },
69 { 0x00D880, 0x00D8FF },
70 {},
71};
72
73static const struct intel_mmio_range xelpg_instance0_steering_table[] = {
74 { 0x000B00, 0x000BFF }, /* SQIDI */
75 { 0x001000, 0x001FFF }, /* SQIDI */
76 { 0x004000, 0x0048FF }, /* GAM */
77 { 0x008700, 0x0087FF }, /* SQIDI */
78 { 0x00B000, 0x00B0FF }, /* NODE */
79 { 0x00C800, 0x00CFFF }, /* GAM */
80 { 0x00D880, 0x00D8FF }, /* NODE */
81 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
82 {},
83};
84
85static const struct intel_mmio_range xelpg_l3bank_steering_table[] = {
86 { 0x00B100, 0x00B3FF },
87 {},
88};
89
90/* DSS steering is used for SLICE ranges as well */
91static const struct intel_mmio_range xelpg_dss_steering_table[] = {
92 { 0x005200, 0x0052FF }, /* SLICE */
93 { 0x005500, 0x007FFF }, /* SLICE */
94 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
95 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
96 { 0x009680, 0x0096FF }, /* DSS */
97 { 0x00D800, 0x00D87F }, /* SLICE */
98 { 0x00DC00, 0x00DCFF }, /* SLICE */
99 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
100 {},
101};
102
103static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = {
104 { 0x393200, 0x39323F },
105 { 0x393400, 0x3934FF },
106 {},
107};
108
109void intel_gt_mcr_init(struct intel_gt *gt)
110{
111 struct drm_i915_private *i915 = gt->i915;
112 unsigned long fuse;
113 int i;
114
115 spin_lock_init(&gt->mcr_lock);
116
117 /*
118 * An mslice is unavailable only if both the meml3 for the slice is
119 * disabled *and* all of the DSS in the slice (quadrant) are disabled.
120 */
121 if (HAS_MSLICE_STEERING(i915)) {
122 gt->info.mslice_mask =
123 intel_slicemask_from_xehp_dssmask(dss_mask: gt->info.sseu.subslice_mask,
124 GEN_DSS_PER_MSLICE);
125 gt->info.mslice_mask |= REG_FIELD_GET(GEN12_MEML3_EN_MASK,
126 intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3));
127
128 if (!gt->info.mslice_mask) /* should be impossible! */
129 gt_warn(gt, "mslice mask all zero!\n");
130 }
131
132 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {
133 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table;
134 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
135 /* Wa_14016747170 */
136 if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
137 IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))
138 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK,
139 intel_uncore_read(gt->uncore,
140 MTL_GT_ACTIVITY_FACTOR));
141 else
142 fuse = REG_FIELD_GET(GT_L3_EXC_MASK,
143 intel_uncore_read(gt->uncore, XEHP_FUSE4));
144
145 /*
146 * Despite the register field being named "exclude mask" the
147 * bits actually represent enabled banks (two banks per bit).
148 */
149 for_each_set_bit(i, &fuse, 3)
150 gt->info.l3bank_mask |= 0x3 << 2 * i;
151
152 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;
153 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;
154 gt->steering_table[DSS] = xelpg_dss_steering_table;
155 } else if (IS_DG2(i915)) {
156 gt->steering_table[MSLICE] = dg2_mslice_steering_table;
157 gt->steering_table[LNCF] = dg2_lncf_steering_table;
158 /*
159 * No need to hook up the GAM table since it has a dedicated
160 * steering control register on DG2 and can use implicit
161 * steering.
162 */
163 } else if (GRAPHICS_VER(i915) >= 11 &&
164 GRAPHICS_VER_FULL(i915) < IP_VER(12, 55)) {
165 gt->steering_table[L3BANK] = icl_l3bank_steering_table;
166 gt->info.l3bank_mask =
167 ~intel_uncore_read(uncore: gt->uncore, GEN10_MIRROR_FUSE3) &
168 GEN10_L3BANK_MASK;
169 if (!gt->info.l3bank_mask) /* should be impossible! */
170 gt_warn(gt, "L3 bank mask is all zero!\n");
171 } else if (GRAPHICS_VER(i915) >= 11) {
172 /*
173 * We expect all modern platforms to have at least some
174 * type of steering that needs to be initialized.
175 */
176 MISSING_CASE(INTEL_INFO(i915)->platform);
177 }
178}
179
180/*
181 * Although the rest of the driver should use MCR-specific functions to
182 * read/write MCR registers, we still use the regular intel_uncore_* functions
183 * internally to implement those, so we need a way for the functions in this
184 * file to "cast" an i915_mcr_reg_t into an i915_reg_t.
185 */
186static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr)
187{
188 i915_reg_t r = { .reg = mcr.reg };
189
190 return r;
191}
192
193/*
194 * rw_with_mcr_steering_fw - Access a register with specific MCR steering
195 * @gt: GT to read register from
196 * @reg: register being accessed
197 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access
198 * @group: group number (documented as "sliceid" on older platforms)
199 * @instance: instance number (documented as "subsliceid" on older platforms)
200 * @value: register value to be written (ignored for read)
201 *
202 * Context: The caller must hold the MCR lock
203 * Return: 0 for write access. register value for read access.
204 *
205 * Caller needs to make sure the relevant forcewake wells are up.
206 */
207static u32 rw_with_mcr_steering_fw(struct intel_gt *gt,
208 i915_mcr_reg_t reg, u8 rw_flag,
209 int group, int instance, u32 value)
210{
211 struct intel_uncore *uncore = gt->uncore;
212 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;
213
214 lockdep_assert_held(&gt->mcr_lock);
215
216 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) {
217 /*
218 * Always leave the hardware in multicast mode when doing reads
219 * (see comment about Wa_22013088509 below) and only change it
220 * to unicast mode when doing writes of a specific instance.
221 *
222 * No need to save old steering reg value.
223 */
224 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR,
225 REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
226 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) |
227 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0));
228 } else if (GRAPHICS_VER(uncore->i915) >= 11) {
229 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
230 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);
231
232 /*
233 * Wa_22013088509
234 *
235 * The setting of the multicast/unicast bit usually wouldn't
236 * matter for read operations (which always return the value
237 * from a single register instance regardless of how that bit
238 * is set), but some platforms have a workaround requiring us
239 * to remain in multicast mode for reads. There's no real
240 * downside to this, so we'll just go ahead and do so on all
241 * platforms; we'll only clear the multicast bit from the mask
242 * when explicitly doing a write operation.
243 */
244 if (rw_flag == FW_REG_WRITE)
245 mcr_mask |= GEN11_MCR_MULTICAST;
246
247 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
248 old_mcr = mcr;
249
250 mcr &= ~mcr_mask;
251 mcr |= mcr_ss;
252 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
253 } else {
254 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
255 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);
256
257 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
258 old_mcr = mcr;
259
260 mcr &= ~mcr_mask;
261 mcr |= mcr_ss;
262 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
263 }
264
265 if (rw_flag == FW_REG_READ)
266 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg));
267 else
268 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value);
269
270 /*
271 * For pre-MTL platforms, we need to restore the old value of the
272 * steering control register to ensure that implicit steering continues
273 * to behave as expected. For MTL and beyond, we need only reinstate
274 * the 'multicast' bit (and only if we did a write that cleared it).
275 */
276 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE)
277 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
278 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70))
279 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr);
280
281 return val;
282}
283
284static u32 rw_with_mcr_steering(struct intel_gt *gt,
285 i915_mcr_reg_t reg, u8 rw_flag,
286 int group, int instance,
287 u32 value)
288{
289 struct intel_uncore *uncore = gt->uncore;
290 enum forcewake_domains fw_domains;
291 unsigned long flags;
292 u32 val;
293
294 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg: mcr_reg_cast(mcr: reg),
295 op: rw_flag);
296 fw_domains |= intel_uncore_forcewake_for_reg(uncore,
297 GEN8_MCR_SELECTOR,
298 FW_REG_READ | FW_REG_WRITE);
299
300 intel_gt_mcr_lock(gt, flags: &flags);
301 spin_lock(lock: &uncore->lock);
302 intel_uncore_forcewake_get__locked(uncore, domains: fw_domains);
303
304 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value);
305
306 intel_uncore_forcewake_put__locked(uncore, domains: fw_domains);
307 spin_unlock(lock: &uncore->lock);
308 intel_gt_mcr_unlock(gt, flags);
309
310 return val;
311}
312
313/**
314 * intel_gt_mcr_lock - Acquire MCR steering lock
315 * @gt: GT structure
316 * @flags: storage to save IRQ flags to
317 *
318 * Performs locking to protect the steering for the duration of an MCR
319 * operation. On MTL and beyond, a hardware lock will also be taken to
320 * serialize access not only for the driver, but also for external hardware and
321 * firmware agents.
322 *
323 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this
324 * function is called, although it may be acquired after this
325 * function call.
326 */
327void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
328 __acquires(&gt->mcr_lock)
329{
330 unsigned long __flags;
331 int err = 0;
332
333 lockdep_assert_not_held(&gt->uncore->lock);
334
335 /*
336 * Starting with MTL, we need to coordinate not only with other
337 * driver threads, but also with hardware/firmware agents. A dedicated
338 * locking register is used.
339 */
340 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
341 /*
342 * The steering control and semaphore registers are inside an
343 * "always on" power domain with respect to RC6. However there
344 * are some issues if higher-level platform sleep states are
345 * entering/exiting at the same time these registers are
346 * accessed. Grabbing GT forcewake and holding it over the
347 * entire lock/steer/unlock cycle ensures that those sleep
348 * states have been fully exited before we access these
349 * registers. This wakeref will be released in the unlock
350 * routine.
351 *
352 * Wa_22018931422
353 */
354 intel_uncore_forcewake_get(uncore: gt->uncore, domains: FORCEWAKE_GT);
355
356 err = wait_for(intel_uncore_read_fw(gt->uncore,
357 MTL_STEER_SEMAPHORE) == 0x1, 100);
358 }
359
360 /*
361 * Even on platforms with a hardware lock, we'll continue to grab
362 * a software spinlock too for lockdep purposes. If the hardware lock
363 * was already acquired, there should never be contention on the
364 * software lock.
365 */
366 spin_lock_irqsave(&gt->mcr_lock, __flags);
367
368 *flags = __flags;
369
370 /*
371 * In theory we should never fail to acquire the HW semaphore; this
372 * would indicate some hardware/firmware is misbehaving and not
373 * releasing it properly.
374 */
375 if (err == -ETIMEDOUT) {
376 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out");
377 add_taint_for_CI(i915: gt->i915, TAINT_WARN); /* CI is now unreliable */
378 }
379}
380
381/**
382 * intel_gt_mcr_unlock - Release MCR steering lock
383 * @gt: GT structure
384 * @flags: IRQ flags to restore
385 *
386 * Releases the lock acquired by intel_gt_mcr_lock().
387 *
388 * Context: Releases gt->mcr_lock
389 */
390void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)
391 __releases(&gt->mcr_lock)
392{
393 spin_unlock_irqrestore(lock: &gt->mcr_lock, flags);
394
395 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
396 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
397
398 intel_uncore_forcewake_put(uncore: gt->uncore, domains: FORCEWAKE_GT);
399 }
400}
401
402/**
403 * intel_gt_mcr_lock_sanitize - Sanitize MCR steering lock
404 * @gt: GT structure
405 *
406 * This will be used to sanitize the initial status of the hardware lock
407 * during driver load and resume since there won't be any concurrent access
408 * from other agents at those times, but it's possible that boot firmware
409 * may have left the lock in a bad state.
410 *
411 */
412void intel_gt_mcr_lock_sanitize(struct intel_gt *gt)
413{
414 /*
415 * This gets called at load/resume time, so we shouldn't be
416 * racing with other driver threads grabbing the mcr lock.
417 */
418 lockdep_assert_not_held(&gt->mcr_lock);
419
420 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
421 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
422}
423
424/**
425 * intel_gt_mcr_read - read a specific instance of an MCR register
426 * @gt: GT structure
427 * @reg: the MCR register to read
428 * @group: the MCR group
429 * @instance: the MCR instance
430 *
431 * Context: Takes and releases gt->mcr_lock
432 *
433 * Returns the value read from an MCR register after steering toward a specific
434 * group/instance.
435 */
436u32 intel_gt_mcr_read(struct intel_gt *gt,
437 i915_mcr_reg_t reg,
438 int group, int instance)
439{
440 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, value: 0);
441}
442
443/**
444 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register
445 * @gt: GT structure
446 * @reg: the MCR register to write
447 * @value: value to write
448 * @group: the MCR group
449 * @instance: the MCR instance
450 *
451 * Write an MCR register in unicast mode after steering toward a specific
452 * group/instance.
453 *
454 * Context: Calls a function that takes and releases gt->mcr_lock
455 */
456void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value,
457 int group, int instance)
458{
459 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value);
460}
461
462/**
463 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register
464 * @gt: GT structure
465 * @reg: the MCR register to write
466 * @value: value to write
467 *
468 * Write an MCR register in multicast mode to update all instances.
469 *
470 * Context: Takes and releases gt->mcr_lock
471 */
472void intel_gt_mcr_multicast_write(struct intel_gt *gt,
473 i915_mcr_reg_t reg, u32 value)
474{
475 unsigned long flags;
476
477 intel_gt_mcr_lock(gt, flags: &flags);
478
479 /*
480 * Ensure we have multicast behavior, just in case some non-i915 agent
481 * left the hardware in unicast mode.
482 */
483 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
484 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
485
486 intel_uncore_write(uncore: gt->uncore, reg: mcr_reg_cast(mcr: reg), val: value);
487
488 intel_gt_mcr_unlock(gt, flags);
489}
490
491/**
492 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register
493 * @gt: GT structure
494 * @reg: the MCR register to write
495 * @value: value to write
496 *
497 * Write an MCR register in multicast mode to update all instances. This
498 * function assumes the caller is already holding any necessary forcewake
499 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should
500 * be obtained automatically.
501 *
502 * Context: The caller must hold gt->mcr_lock.
503 */
504void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value)
505{
506 lockdep_assert_held(&gt->mcr_lock);
507
508 /*
509 * Ensure we have multicast behavior, just in case some non-i915 agent
510 * left the hardware in unicast mode.
511 */
512 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
513 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
514
515 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value);
516}
517
518/**
519 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations
520 * @gt: GT structure
521 * @reg: the MCR register to read and write
522 * @clear: bits to clear during RMW
523 * @set: bits to set during RMW
524 *
525 * Performs a read-modify-write on an MCR register in a multicast manner.
526 * This operation only makes sense on MCR registers where all instances are
527 * expected to have the same value. The read will target any non-terminated
528 * instance and the write will be applied to all instances.
529 *
530 * This function assumes the caller is already holding any necessary forcewake
531 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should
532 * be obtained automatically.
533 *
534 * Context: Calls functions that take and release gt->mcr_lock
535 *
536 * Returns the old (unmodified) value read.
537 */
538u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg,
539 u32 clear, u32 set)
540{
541 u32 val = intel_gt_mcr_read_any(gt, reg);
542
543 intel_gt_mcr_multicast_write(gt, reg, value: (val & ~clear) | set);
544
545 return val;
546}
547
548/*
549 * reg_needs_read_steering - determine whether a register read requires
550 * explicit steering
551 * @gt: GT structure
552 * @reg: the register to check steering requirements for
553 * @type: type of multicast steering to check
554 *
555 * Determines whether @reg needs explicit steering of a specific type for
556 * reads.
557 *
558 * Returns false if @reg does not belong to a register range of the given
559 * steering type, or if the default (subslice-based) steering IDs are suitable
560 * for @type steering too.
561 */
562static bool reg_needs_read_steering(struct intel_gt *gt,
563 i915_mcr_reg_t reg,
564 enum intel_steering_type type)
565{
566 u32 offset = i915_mmio_reg_offset(reg);
567 const struct intel_mmio_range *entry;
568
569 if (likely(!gt->steering_table[type]))
570 return false;
571
572 if (IS_GSI_REG(offset))
573 offset += gt->uncore->gsi_offset;
574
575 for (entry = gt->steering_table[type]; entry->end; entry++) {
576 if (offset >= entry->start && offset <= entry->end)
577 return true;
578 }
579
580 return false;
581}
582
583/*
584 * get_nonterminated_steering - determines valid IDs for a class of MCR steering
585 * @gt: GT structure
586 * @type: multicast register type
587 * @group: Group ID returned
588 * @instance: Instance ID returned
589 *
590 * Determines group and instance values that will steer reads of the specified
591 * MCR class to a non-terminated instance.
592 */
593static void get_nonterminated_steering(struct intel_gt *gt,
594 enum intel_steering_type type,
595 u8 *group, u8 *instance)
596{
597 u32 dss;
598
599 switch (type) {
600 case L3BANK:
601 *group = 0; /* unused */
602 *instance = __ffs(gt->info.l3bank_mask);
603 break;
604 case MSLICE:
605 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
606 *group = __ffs(gt->info.mslice_mask);
607 *instance = 0; /* unused */
608 break;
609 case LNCF:
610 /*
611 * An LNCF is always present if its mslice is present, so we
612 * can safely just steer to LNCF 0 in all cases.
613 */
614 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
615 *group = __ffs(gt->info.mslice_mask) << 1;
616 *instance = 0; /* unused */
617 break;
618 case GAM:
619 *group = IS_DG2(gt->i915) ? 1 : 0;
620 *instance = 0;
621 break;
622 case DSS:
623 dss = intel_sseu_find_first_xehp_dss(sseu: &gt->info.sseu, groupsize: 0, groupnum: 0);
624 *group = dss / GEN_DSS_PER_GSLICE;
625 *instance = dss % GEN_DSS_PER_GSLICE;
626 break;
627 case INSTANCE0:
628 /*
629 * There are a lot of MCR types for which instance (0, 0)
630 * will always provide a non-terminated value.
631 */
632 *group = 0;
633 *instance = 0;
634 break;
635 case OADDRM:
636 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0))
637 *group = 0;
638 else
639 *group = 1;
640 *instance = 0;
641 break;
642 default:
643 MISSING_CASE(type);
644 *group = 0;
645 *instance = 0;
646 }
647}
648
649/**
650 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that
651 * will steer a register to a non-terminated instance
652 * @gt: GT structure
653 * @reg: register for which the steering is required
654 * @group: return variable for group steering
655 * @instance: return variable for instance steering
656 *
657 * This function returns a group/instance pair that is guaranteed to work for
658 * read steering of the given register. Note that a value will be returned even
659 * if the register is not replicated and therefore does not actually require
660 * steering.
661 */
662void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,
663 i915_mcr_reg_t reg,
664 u8 *group, u8 *instance)
665{
666 int type;
667
668 for (type = 0; type < NUM_STEERING_TYPES; type++) {
669 if (reg_needs_read_steering(gt, reg, type)) {
670 get_nonterminated_steering(gt, type, group, instance);
671 return;
672 }
673 }
674
675 *group = gt->default_steering.groupid;
676 *instance = gt->default_steering.instanceid;
677}
678
679/**
680 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register
681 * @gt: GT structure
682 * @reg: register to read
683 *
684 * Reads a GT MCR register. The read will be steered to a non-terminated
685 * instance (i.e., one that isn't fused off or powered down by power gating).
686 * This function assumes the caller is already holding any necessary forcewake
687 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be
688 * obtained automatically.
689 *
690 * Context: The caller must hold gt->mcr_lock.
691 *
692 * Returns the value from a non-terminated instance of @reg.
693 */
694u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg)
695{
696 int type;
697 u8 group, instance;
698
699 lockdep_assert_held(&gt->mcr_lock);
700
701 for (type = 0; type < NUM_STEERING_TYPES; type++) {
702 if (reg_needs_read_steering(gt, reg, type)) {
703 get_nonterminated_steering(gt, type, group: &group, instance: &instance);
704 return rw_with_mcr_steering_fw(gt, reg,
705 FW_REG_READ,
706 group, instance, value: 0);
707 }
708 }
709
710 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg));
711}
712
713/**
714 * intel_gt_mcr_read_any - reads one instance of an MCR register
715 * @gt: GT structure
716 * @reg: register to read
717 *
718 * Reads a GT MCR register. The read will be steered to a non-terminated
719 * instance (i.e., one that isn't fused off or powered down by power gating).
720 *
721 * Context: Calls a function that takes and releases gt->mcr_lock.
722 *
723 * Returns the value from a non-terminated instance of @reg.
724 */
725u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg)
726{
727 int type;
728 u8 group, instance;
729
730 for (type = 0; type < NUM_STEERING_TYPES; type++) {
731 if (reg_needs_read_steering(gt, reg, type)) {
732 get_nonterminated_steering(gt, type, group: &group, instance: &instance);
733 return rw_with_mcr_steering(gt, reg,
734 FW_REG_READ,
735 group, instance, value: 0);
736 }
737 }
738
739 return intel_uncore_read(uncore: gt->uncore, reg: mcr_reg_cast(mcr: reg));
740}
741
742static void report_steering_type(struct drm_printer *p,
743 struct intel_gt *gt,
744 enum intel_steering_type type,
745 bool dump_table)
746{
747 const struct intel_mmio_range *entry;
748 u8 group, instance;
749
750 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
751
752 if (!gt->steering_table[type]) {
753 drm_printf(p, f: "%s steering: uses default steering\n",
754 intel_steering_types[type]);
755 return;
756 }
757
758 get_nonterminated_steering(gt, type, group: &group, instance: &instance);
759 drm_printf(p, f: "%s steering: group=0x%x, instance=0x%x\n",
760 intel_steering_types[type], group, instance);
761
762 if (!dump_table)
763 return;
764
765 for (entry = gt->steering_table[type]; entry->end; entry++)
766 drm_printf(p, f: "\t0x%06x - 0x%06x\n", entry->start, entry->end);
767}
768
769void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
770 bool dump_table)
771{
772 /*
773 * Starting with MTL we no longer have default steering;
774 * all ranges are explicitly steered.
775 */
776 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70))
777 drm_printf(p, f: "Default steering: group=0x%x, instance=0x%x\n",
778 gt->default_steering.groupid,
779 gt->default_steering.instanceid);
780
781 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
782 for (int i = 0; i < NUM_STEERING_TYPES; i++)
783 if (gt->steering_table[i])
784 report_steering_type(p, gt, type: i, dump_table);
785 } else if (HAS_MSLICE_STEERING(gt->i915)) {
786 report_steering_type(p, gt, type: MSLICE, dump_table);
787 report_steering_type(p, gt, type: LNCF, dump_table);
788 }
789}
790
791/**
792 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS
793 * @gt: GT structure
794 * @dss: DSS ID to obtain steering for
795 * @group: pointer to storage for steering group ID
796 * @instance: pointer to storage for steering instance ID
797 *
798 * Returns the steering IDs (via the @group and @instance parameters) that
799 * correspond to a specific subslice/DSS ID.
800 */
801void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
802 unsigned int *group, unsigned int *instance)
803{
804 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) {
805 *group = dss / GEN_DSS_PER_GSLICE;
806 *instance = dss % GEN_DSS_PER_GSLICE;
807 } else {
808 *group = dss / GEN_MAX_SS_PER_HSW_SLICE;
809 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE;
810 return;
811 }
812}
813
814/**
815 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state
816 * @gt: GT structure
817 * @reg: the register to read
818 * @mask: mask to apply to register value
819 * @value: value to wait for
820 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
821 * @slow_timeout_ms: slow timeout in millisecond
822 *
823 * This routine waits until the target register @reg contains the expected
824 * @value after applying the @mask, i.e. it waits until ::
825 *
826 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value
827 *
828 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
829 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
830 * must be not larger than 20,0000 microseconds.
831 *
832 * This function is basically an MCR-friendly version of
833 * __intel_wait_for_register_fw(). Generally this function will only be used
834 * on GAM registers which are a bit special --- although they're MCR registers,
835 * reads (e.g., waiting for status updates) are always directed to the primary
836 * instance.
837 *
838 * Note that this routine assumes the caller holds forcewake asserted, it is
839 * not suitable for very long waits.
840 *
841 * Context: Calls a function that takes and releases gt->mcr_lock
842 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
843 */
844int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,
845 i915_mcr_reg_t reg,
846 u32 mask,
847 u32 value,
848 unsigned int fast_timeout_us,
849 unsigned int slow_timeout_ms)
850{
851 int ret;
852
853 lockdep_assert_not_held(&gt->mcr_lock);
854
855#define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value)
856
857 /* Catch any overuse of this function */
858 might_sleep_if(slow_timeout_ms);
859 GEM_BUG_ON(fast_timeout_us > 20000);
860 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
861
862 ret = -ETIMEDOUT;
863 if (fast_timeout_us && fast_timeout_us <= 20000)
864 ret = _wait_for_atomic(done, fast_timeout_us, 0);
865 if (ret && slow_timeout_ms)
866 ret = wait_for(done, slow_timeout_ms);
867
868 return ret;
869#undef done
870}
871