1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <drm/drm_managed.h>
7#include <drm/intel/intel-gtt.h>
8
9#include "gem/i915_gem_internal.h"
10#include "gem/i915_gem_lmem.h"
11
12#include "i915_drv.h"
13#include "i915_perf_oa_regs.h"
14#include "i915_reg.h"
15#include "intel_context.h"
16#include "intel_engine_pm.h"
17#include "intel_engine_regs.h"
18#include "intel_ggtt_gmch.h"
19#include "intel_gt.h"
20#include "intel_gt_buffer_pool.h"
21#include "intel_gt_clock_utils.h"
22#include "intel_gt_debugfs.h"
23#include "intel_gt_mcr.h"
24#include "intel_gt_pm.h"
25#include "intel_gt_print.h"
26#include "intel_gt_regs.h"
27#include "intel_gt_requests.h"
28#include "intel_migrate.h"
29#include "intel_mocs.h"
30#include "intel_pci_config.h"
31#include "intel_rc6.h"
32#include "intel_renderstate.h"
33#include "intel_rps.h"
34#include "intel_sa_media.h"
35#include "intel_gt_sysfs.h"
36#include "intel_tlb.h"
37#include "intel_uncore.h"
38#include "shmem_utils.h"
39
40void intel_gt_common_init_early(struct intel_gt *gt)
41{
42 spin_lock_init(gt->irq_lock);
43
44 INIT_LIST_HEAD(list: &gt->closed_vma);
45 spin_lock_init(&gt->closed_lock);
46
47 init_llist_head(list: &gt->watchdog.list);
48 INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work);
49
50 intel_gt_init_buffer_pool(gt);
51 intel_gt_init_reset(gt);
52 intel_gt_init_requests(gt);
53 intel_gt_init_timelines(gt);
54 intel_gt_init_tlb(gt);
55 intel_gt_pm_init_early(gt);
56
57 intel_wopcm_init_early(wopcm: &gt->wopcm);
58 intel_uc_init_early(uc: &gt->uc);
59 intel_rps_init_early(rps: &gt->rps);
60}
61
62/* Preliminary initialization of Tile 0 */
63int intel_root_gt_init_early(struct drm_i915_private *i915)
64{
65 struct intel_gt *gt;
66
67 gt = drmm_kzalloc(dev: &i915->drm, size: sizeof(*gt), GFP_KERNEL);
68 if (!gt)
69 return -ENOMEM;
70
71 i915->gt[0] = gt;
72
73 gt->i915 = i915;
74 gt->uncore = &i915->uncore;
75 gt->irq_lock = drmm_kzalloc(dev: &i915->drm, size: sizeof(*gt->irq_lock), GFP_KERNEL);
76 if (!gt->irq_lock)
77 return -ENOMEM;
78
79 intel_gt_common_init_early(gt);
80
81 return 0;
82}
83
84static int intel_gt_probe_lmem(struct intel_gt *gt)
85{
86 struct drm_i915_private *i915 = gt->i915;
87 unsigned int instance = gt->info.id;
88 int id = INTEL_REGION_LMEM_0 + instance;
89 struct intel_memory_region *mem;
90 int err;
91
92 mem = intel_gt_setup_lmem(gt);
93 if (IS_ERR(ptr: mem)) {
94 err = PTR_ERR(ptr: mem);
95 if (err == -ENODEV)
96 return 0;
97
98 gt_err(gt, "Failed to setup region(%d) type=%d\n",
99 err, INTEL_MEMORY_LOCAL);
100 return err;
101 }
102
103 mem->id = id;
104 mem->instance = instance;
105
106 intel_memory_region_set_name(mem, fmt: "local%u", mem->instance);
107
108 GEM_BUG_ON(!HAS_REGION(i915, id));
109 GEM_BUG_ON(i915->mm.regions[id]);
110 i915->mm.regions[id] = mem;
111
112 return 0;
113}
114
115int intel_gt_assign_ggtt(struct intel_gt *gt)
116{
117 /* Media GT shares primary GT's GGTT */
118 if (gt->type == GT_MEDIA) {
119 gt->ggtt = to_gt(i915: gt->i915)->ggtt;
120 } else {
121 gt->ggtt = i915_ggtt_create(i915: gt->i915);
122 if (IS_ERR(ptr: gt->ggtt))
123 return PTR_ERR(ptr: gt->ggtt);
124 }
125
126 list_add_tail(new: &gt->ggtt_link, head: &gt->ggtt->gt_list);
127
128 return 0;
129}
130
131int intel_gt_init_mmio(struct intel_gt *gt)
132{
133 intel_gt_init_clock_frequency(gt);
134
135 intel_uc_init_mmio(uc: &gt->uc);
136 intel_sseu_info_init(gt);
137 intel_gt_mcr_init(gt);
138
139 return intel_engines_init_mmio(gt);
140}
141
142static void init_unused_ring(struct intel_gt *gt, u32 base)
143{
144 struct intel_uncore *uncore = gt->uncore;
145
146 intel_uncore_write(uncore, RING_CTL(base), val: 0);
147 intel_uncore_write(uncore, RING_HEAD(base), val: 0);
148 intel_uncore_write(uncore, RING_TAIL(base), val: 0);
149 intel_uncore_write(uncore, RING_START(base), val: 0);
150}
151
152static void init_unused_rings(struct intel_gt *gt)
153{
154 struct drm_i915_private *i915 = gt->i915;
155
156 if (IS_I830(i915)) {
157 init_unused_ring(gt, PRB1_BASE);
158 init_unused_ring(gt, SRB0_BASE);
159 init_unused_ring(gt, SRB1_BASE);
160 init_unused_ring(gt, SRB2_BASE);
161 init_unused_ring(gt, SRB3_BASE);
162 } else if (GRAPHICS_VER(i915) == 2) {
163 init_unused_ring(gt, SRB0_BASE);
164 init_unused_ring(gt, SRB1_BASE);
165 } else if (GRAPHICS_VER(i915) == 3) {
166 init_unused_ring(gt, PRB1_BASE);
167 init_unused_ring(gt, PRB2_BASE);
168 }
169}
170
171int intel_gt_init_hw(struct intel_gt *gt)
172{
173 struct drm_i915_private *i915 = gt->i915;
174 struct intel_uncore *uncore = gt->uncore;
175 int ret;
176
177 gt->last_init_time = ktime_get();
178
179 /* Double layer security blanket, see i915_gem_init() */
180 intel_uncore_forcewake_get(uncore, domains: FORCEWAKE_ALL);
181
182 if (HAS_EDRAM(i915) && GRAPHICS_VER(i915) < 9)
183 intel_uncore_rmw(uncore, HSW_IDICR, clear: 0, IDIHASHMSK(0xf));
184
185 if (IS_HASWELL(i915))
186 intel_uncore_write(uncore,
187 HSW_MI_PREDICATE_RESULT_2,
188 INTEL_INFO(i915)->gt == 3 ?
189 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
190
191 /* Apply the GT workarounds... */
192 intel_gt_apply_workarounds(gt);
193 /* ...and determine whether they are sticking. */
194 intel_gt_verify_workarounds(gt, from: "init");
195
196 intel_gt_init_swizzling(gt);
197
198 /*
199 * At least 830 can leave some of the unused rings
200 * "active" (ie. head != tail) after resume which
201 * will prevent c3 entry. Makes sure all unused rings
202 * are totally idle.
203 */
204 init_unused_rings(gt);
205
206 ret = i915_ppgtt_init_hw(gt);
207 if (ret) {
208 gt_err(gt, "Enabling PPGTT failed (%d)\n", ret);
209 goto out;
210 }
211
212 /* We can't enable contexts until all firmware is loaded */
213 ret = intel_uc_init_hw(uc: &gt->uc);
214 if (ret) {
215 gt_probe_error(gt, "Enabling uc failed (%d)\n", ret);
216 goto out;
217 }
218
219 intel_mocs_init(gt);
220
221out:
222 intel_uncore_forcewake_put(uncore, domains: FORCEWAKE_ALL);
223 return ret;
224}
225
226static void gen6_clear_engine_error_register(struct intel_engine_cs *engine)
227{
228 GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
229 GEN6_RING_FAULT_REG_POSTING_READ(engine);
230}
231
232i915_reg_t intel_gt_perf_limit_reasons_reg(struct intel_gt *gt)
233{
234 /* GT0_PERF_LIMIT_REASONS is available only for Gen11+ */
235 if (GRAPHICS_VER(gt->i915) < 11)
236 return INVALID_MMIO_REG;
237
238 return gt->type == GT_MEDIA ?
239 MTL_MEDIA_PERF_LIMIT_REASONS : GT0_PERF_LIMIT_REASONS;
240}
241
242void
243intel_gt_clear_error_registers(struct intel_gt *gt,
244 intel_engine_mask_t engine_mask)
245{
246 struct drm_i915_private *i915 = gt->i915;
247 struct intel_uncore *uncore = gt->uncore;
248 u32 eir;
249
250 if (GRAPHICS_VER(i915) != 2)
251 intel_uncore_write(uncore, PGTBL_ER, val: 0);
252
253 if (GRAPHICS_VER(i915) < 4)
254 intel_uncore_write(uncore, IPEIR(RENDER_RING_BASE), val: 0);
255 else
256 intel_uncore_write(uncore, IPEIR_I965, val: 0);
257
258 intel_uncore_write(uncore, EIR, val: 0);
259 eir = intel_uncore_read(uncore, EIR);
260 if (eir) {
261 /*
262 * some errors might have become stuck,
263 * mask them.
264 */
265 gt_dbg(gt, "EIR stuck: 0x%08x, masking\n", eir);
266 intel_uncore_rmw(uncore, EMR, clear: 0, set: eir);
267 intel_uncore_write(uncore, GEN2_IIR,
268 I915_MASTER_ERROR_INTERRUPT);
269 }
270
271 /*
272 * For the media GT, this ring fault register is not replicated,
273 * so don't do multicast/replicated register read/write operation on it.
274 */
275 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {
276 intel_uncore_rmw(uncore, XELPMP_RING_FAULT_REG,
277 RING_FAULT_VALID, set: 0);
278 intel_uncore_posting_read(uncore,
279 XELPMP_RING_FAULT_REG);
280
281 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
282 intel_gt_mcr_multicast_rmw(gt, XEHP_RING_FAULT_REG,
283 RING_FAULT_VALID, set: 0);
284 intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG);
285
286 } else if (GRAPHICS_VER(i915) >= 12) {
287 intel_uncore_rmw(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID, set: 0);
288 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
289 } else if (GRAPHICS_VER(i915) >= 8) {
290 intel_uncore_rmw(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID, set: 0);
291 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
292 } else if (GRAPHICS_VER(i915) >= 6) {
293 struct intel_engine_cs *engine;
294 enum intel_engine_id id;
295
296 for_each_engine_masked(engine, gt, engine_mask, id)
297 gen6_clear_engine_error_register(engine);
298 }
299}
300
301static void gen6_check_faults(struct intel_gt *gt)
302{
303 struct intel_engine_cs *engine;
304 enum intel_engine_id id;
305
306 for_each_engine(engine, gt, id) {
307 u32 fault;
308
309 fault = GEN6_RING_FAULT_REG_READ(engine);
310
311 if (fault & RING_FAULT_VALID) {
312 gt_dbg(gt, "Unexpected fault\n"
313 "\tAddr: 0x%08x\n"
314 "\tAddress space: %s\n"
315 "\tSource ID: %d\n"
316 "\tType: %d\n",
317 fault & RING_FAULT_VADDR_MASK,
318 fault & RING_FAULT_GTTSEL_MASK ?
319 "GGTT" : "PPGTT",
320 REG_FIELD_GET(RING_FAULT_SRCID_MASK, fault),
321 REG_FIELD_GET(RING_FAULT_FAULT_TYPE_MASK, fault));
322 }
323 }
324}
325
326static void gen8_report_fault(struct intel_gt *gt, u32 fault,
327 u32 fault_data0, u32 fault_data1)
328{
329 u64 fault_addr;
330
331 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
332 ((u64)fault_data0 << 12);
333
334 gt_dbg(gt, "Unexpected fault\n"
335 "\tAddr: 0x%08x_%08x\n"
336 "\tAddress space: %s\n"
337 "\tEngine ID: %d\n"
338 "\tSource ID: %d\n"
339 "\tType: %d\n",
340 upper_32_bits(fault_addr), lower_32_bits(fault_addr),
341 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
342 REG_FIELD_GET(RING_FAULT_ENGINE_ID_MASK, fault),
343 REG_FIELD_GET(RING_FAULT_SRCID_MASK, fault),
344 REG_FIELD_GET(RING_FAULT_FAULT_TYPE_MASK, fault));
345}
346
347static void xehp_check_faults(struct intel_gt *gt)
348{
349 u32 fault;
350
351 /*
352 * Although the fault register now lives in an MCR register range,
353 * the GAM registers are special and we only truly need to read
354 * the "primary" GAM instance rather than handling each instance
355 * individually. intel_gt_mcr_read_any() will automatically steer
356 * toward the primary instance.
357 */
358 fault = intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG);
359 if (fault & RING_FAULT_VALID)
360 gen8_report_fault(gt, fault,
361 fault_data0: intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA0),
362 fault_data1: intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA1));
363}
364
365static void gen8_check_faults(struct intel_gt *gt)
366{
367 struct intel_uncore *uncore = gt->uncore;
368 i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
369 u32 fault;
370
371 if (GRAPHICS_VER(gt->i915) >= 12) {
372 fault_reg = GEN12_RING_FAULT_REG;
373 fault_data0_reg = GEN12_FAULT_TLB_DATA0;
374 fault_data1_reg = GEN12_FAULT_TLB_DATA1;
375 } else {
376 fault_reg = GEN8_RING_FAULT_REG;
377 fault_data0_reg = GEN8_FAULT_TLB_DATA0;
378 fault_data1_reg = GEN8_FAULT_TLB_DATA1;
379 }
380
381 fault = intel_uncore_read(uncore, reg: fault_reg);
382 if (fault & RING_FAULT_VALID)
383 gen8_report_fault(gt, fault,
384 fault_data0: intel_uncore_read(uncore, reg: fault_data0_reg),
385 fault_data1: intel_uncore_read(uncore, reg: fault_data1_reg));
386}
387
388void intel_gt_check_and_clear_faults(struct intel_gt *gt)
389{
390 struct drm_i915_private *i915 = gt->i915;
391
392 /* From GEN8 onwards we only have one 'All Engine Fault Register' */
393 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
394 xehp_check_faults(gt);
395 else if (GRAPHICS_VER(i915) >= 8)
396 gen8_check_faults(gt);
397 else if (GRAPHICS_VER(i915) >= 6)
398 gen6_check_faults(gt);
399 else
400 return;
401
402 intel_gt_clear_error_registers(gt, ALL_ENGINES);
403}
404
405void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
406{
407 struct intel_uncore *uncore = gt->uncore;
408 intel_wakeref_t wakeref;
409
410 /*
411 * No actual flushing is required for the GTT write domain for reads
412 * from the GTT domain. Writes to it "immediately" go to main memory
413 * as far as we know, so there's no chipset flush. It also doesn't
414 * land in the GPU render cache.
415 *
416 * However, we do have to enforce the order so that all writes through
417 * the GTT land before any writes to the device, such as updates to
418 * the GATT itself.
419 *
420 * We also have to wait a bit for the writes to land from the GTT.
421 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
422 * timing. This issue has only been observed when switching quickly
423 * between GTT writes and CPU reads from inside the kernel on recent hw,
424 * and it appears to only affect discrete GTT blocks (i.e. on LLC
425 * system agents we cannot reproduce this behaviour, until Cannonlake
426 * that was!).
427 */
428
429 wmb();
430
431 if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
432 return;
433
434 intel_gt_chipset_flush(gt);
435
436 with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
437 unsigned long flags;
438
439 spin_lock_irqsave(&uncore->lock, flags);
440 intel_uncore_posting_read_fw(uncore,
441 RING_TAIL(RENDER_RING_BASE));
442 spin_unlock_irqrestore(lock: &uncore->lock, flags);
443 }
444}
445
446void intel_gt_chipset_flush(struct intel_gt *gt)
447{
448 wmb();
449 if (GRAPHICS_VER(gt->i915) < 6)
450 intel_ggtt_gmch_flush();
451}
452
453void intel_gt_driver_register(struct intel_gt *gt)
454{
455 intel_gsc_init(gsc: &gt->gsc, i915: gt->i915);
456
457 intel_rps_driver_register(rps: &gt->rps);
458
459 intel_gt_debugfs_register(gt);
460 intel_gt_sysfs_register(gt);
461}
462
463static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
464{
465 struct drm_i915_private *i915 = gt->i915;
466 struct drm_i915_gem_object *obj;
467 struct i915_vma *vma;
468 int ret;
469
470 obj = i915_gem_object_create_lmem(i915, size,
471 I915_BO_ALLOC_VOLATILE |
472 I915_BO_ALLOC_GPU_ONLY);
473 if (IS_ERR(ptr: obj) && !IS_METEORLAKE(i915)) /* Wa_22018444074 */
474 obj = i915_gem_object_create_stolen(i915, size);
475 if (IS_ERR(ptr: obj))
476 obj = i915_gem_object_create_internal(i915, size);
477 if (IS_ERR(ptr: obj)) {
478 gt_err(gt, "Failed to allocate scratch page\n");
479 return PTR_ERR(ptr: obj);
480 }
481
482 vma = i915_vma_instance(obj, vm: &gt->ggtt->vm, NULL);
483 if (IS_ERR(ptr: vma)) {
484 ret = PTR_ERR(ptr: vma);
485 goto err_unref;
486 }
487
488 ret = i915_ggtt_pin(vma, NULL, align: 0, PIN_HIGH);
489 if (ret)
490 goto err_unref;
491
492 gt->scratch = i915_vma_make_unshrinkable(vma);
493
494 return 0;
495
496err_unref:
497 i915_gem_object_put(obj);
498 return ret;
499}
500
501static void intel_gt_fini_scratch(struct intel_gt *gt)
502{
503 i915_vma_unpin_and_release(p_vma: &gt->scratch, flags: 0);
504}
505
506static struct i915_address_space *kernel_vm(struct intel_gt *gt)
507{
508 if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
509 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm;
510 else
511 return i915_vm_get(vm: &gt->ggtt->vm);
512}
513
514static int __engines_record_defaults(struct intel_gt *gt)
515{
516 struct i915_request *requests[I915_NUM_ENGINES] = {};
517 struct intel_engine_cs *engine;
518 enum intel_engine_id id;
519 int err = 0;
520
521 /*
522 * As we reset the gpu during very early sanitisation, the current
523 * register state on the GPU should reflect its defaults values.
524 * We load a context onto the hw (with restore-inhibit), then switch
525 * over to a second context to save that default register state. We
526 * can then prime every new context with that state so they all start
527 * from the same default HW values.
528 */
529
530 for_each_engine(engine, gt, id) {
531 struct intel_renderstate so;
532 struct intel_context *ce;
533 struct i915_request *rq;
534
535 /* We must be able to switch to something! */
536 GEM_BUG_ON(!engine->kernel_context);
537
538 ce = intel_context_create(engine);
539 if (IS_ERR(ptr: ce)) {
540 err = PTR_ERR(ptr: ce);
541 goto out;
542 }
543
544 err = intel_renderstate_init(so: &so, ce);
545 if (err)
546 goto err;
547
548 rq = i915_request_create(ce);
549 if (IS_ERR(ptr: rq)) {
550 err = PTR_ERR(ptr: rq);
551 goto err_fini;
552 }
553
554 err = intel_engine_emit_ctx_wa(rq);
555 if (err)
556 goto err_rq;
557
558 err = intel_renderstate_emit(so: &so, rq);
559 if (err)
560 goto err_rq;
561
562err_rq:
563 requests[id] = i915_request_get(rq);
564 i915_request_add(rq);
565err_fini:
566 intel_renderstate_fini(so: &so, ce);
567err:
568 if (err) {
569 intel_context_put(ce);
570 goto out;
571 }
572 }
573
574 /* Flush the default context image to memory, and enable powersaving. */
575 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
576 err = -EIO;
577 goto out;
578 }
579
580 for (id = 0; id < ARRAY_SIZE(requests); id++) {
581 struct i915_request *rq;
582 struct file *state;
583
584 rq = requests[id];
585 if (!rq)
586 continue;
587
588 if (rq->fence.error) {
589 err = -EIO;
590 goto out;
591 }
592
593 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
594 if (!rq->context->state)
595 continue;
596
597 /* Keep a copy of the state's backing pages; free the obj */
598 state = shmem_create_from_object(obj: rq->context->state->obj);
599 if (IS_ERR(ptr: state)) {
600 err = PTR_ERR(ptr: state);
601 goto out;
602 }
603 rq->engine->default_state = state;
604 }
605
606out:
607 /*
608 * If we have to abandon now, we expect the engines to be idle
609 * and ready to be torn-down. The quickest way we can accomplish
610 * this is by declaring ourselves wedged.
611 */
612 if (err)
613 intel_gt_set_wedged(gt);
614
615 for (id = 0; id < ARRAY_SIZE(requests); id++) {
616 struct intel_context *ce;
617 struct i915_request *rq;
618
619 rq = requests[id];
620 if (!rq)
621 continue;
622
623 ce = rq->context;
624 i915_request_put(rq);
625 intel_context_put(ce);
626 }
627 return err;
628}
629
630static int __engines_verify_workarounds(struct intel_gt *gt)
631{
632 struct intel_engine_cs *engine;
633 enum intel_engine_id id;
634 int err = 0;
635
636 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
637 return 0;
638
639 for_each_engine(engine, gt, id) {
640 if (intel_engine_verify_workarounds(engine, from: "load"))
641 err = -EIO;
642 }
643
644 /* Flush and restore the kernel context for safety */
645 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
646 err = -EIO;
647
648 return err;
649}
650
651static void __intel_gt_disable(struct intel_gt *gt)
652{
653 intel_gt_set_wedged_on_fini(gt);
654
655 intel_gt_suspend_prepare(gt);
656 intel_gt_suspend_late(gt);
657
658 GEM_BUG_ON(intel_gt_pm_is_awake(gt));
659}
660
661int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
662{
663 long remaining_timeout;
664
665 /* If the device is asleep, we have no requests outstanding */
666 if (!intel_gt_pm_is_awake(gt))
667 return 0;
668
669 while ((timeout = intel_gt_retire_requests_timeout(gt, timeout,
670 remaining_timeout: &remaining_timeout)) > 0) {
671 cond_resched();
672 if (signal_pending(current))
673 return -EINTR;
674 }
675
676 if (timeout)
677 return timeout;
678
679 if (remaining_timeout < 0)
680 remaining_timeout = 0;
681
682 return intel_uc_wait_for_idle(uc: &gt->uc, timeout: remaining_timeout);
683}
684
685int intel_gt_init(struct intel_gt *gt)
686{
687 int err;
688
689 err = i915_inject_probe_error(gt->i915, -ENODEV);
690 if (err)
691 return err;
692
693 intel_gt_init_workarounds(gt);
694
695 /*
696 * This is just a security blanket to placate dragons.
697 * On some systems, we very sporadically observe that the first TLBs
698 * used by the CS may be stale, despite us poking the TLB reset. If
699 * we hold the forcewake during initialisation these problems
700 * just magically go away.
701 */
702 intel_uncore_forcewake_get(uncore: gt->uncore, domains: FORCEWAKE_ALL);
703
704 err = intel_gt_init_scratch(gt,
705 GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K);
706 if (err)
707 goto out_fw;
708
709 intel_gt_pm_init(gt);
710
711 gt->vm = kernel_vm(gt);
712 if (!gt->vm) {
713 err = -ENOMEM;
714 goto err_pm;
715 }
716
717 intel_set_mocs_index(gt);
718
719 err = intel_engines_init(gt);
720 if (err)
721 goto err_engines;
722
723 err = intel_uc_init(uc: &gt->uc);
724 if (err)
725 goto err_engines;
726
727 err = intel_gt_resume(gt);
728 if (err)
729 goto err_uc_init;
730
731 err = intel_gt_init_hwconfig(gt);
732 if (err)
733 gt_err(gt, "Failed to retrieve hwconfig table: %pe\n", ERR_PTR(err));
734
735 err = __engines_record_defaults(gt);
736 if (err)
737 goto err_gt;
738
739 err = __engines_verify_workarounds(gt);
740 if (err)
741 goto err_gt;
742
743 err = i915_inject_probe_error(gt->i915, -EIO);
744 if (err)
745 goto err_gt;
746
747 intel_uc_init_late(uc: &gt->uc);
748
749 intel_migrate_init(m: &gt->migrate, gt);
750
751 goto out_fw;
752err_gt:
753 __intel_gt_disable(gt);
754 intel_uc_fini_hw(uc: &gt->uc);
755err_uc_init:
756 intel_uc_fini(uc: &gt->uc);
757err_engines:
758 intel_engines_release(gt);
759 i915_vm_put(fetch_and_zero(&gt->vm));
760err_pm:
761 intel_gt_pm_fini(gt);
762 intel_gt_fini_scratch(gt);
763out_fw:
764 if (err)
765 intel_gt_set_wedged_on_init(gt);
766 intel_uncore_forcewake_put(uncore: gt->uncore, domains: FORCEWAKE_ALL);
767 return err;
768}
769
770void intel_gt_driver_remove(struct intel_gt *gt)
771{
772 __intel_gt_disable(gt);
773
774 intel_migrate_fini(m: &gt->migrate);
775 intel_uc_driver_remove(uc: &gt->uc);
776
777 intel_engines_release(gt);
778
779 intel_gt_flush_buffer_pool(gt);
780}
781
782void intel_gt_driver_unregister(struct intel_gt *gt)
783{
784 intel_wakeref_t wakeref;
785
786 intel_gt_sysfs_unregister(gt);
787 intel_rps_driver_unregister(rps: &gt->rps);
788 intel_gsc_fini(gsc: &gt->gsc);
789
790 /*
791 * If we unload the driver and wedge before the GSC worker is complete,
792 * the worker will hit an error on its submission to the GSC engine and
793 * then exit. This is hard to hit for a user, but it is reproducible
794 * with skipping selftests. The error is handled gracefully by the
795 * worker, so there are no functional issues, but we still end up with
796 * an error message in dmesg, which is something we want to avoid as
797 * this is a supported scenario. We could modify the worker to better
798 * handle a wedging occurring during its execution, but that gets
799 * complicated for a couple of reasons:
800 * - We do want the error on runtime wedging, because there are
801 * implications for subsystems outside of GT (i.e., PXP, HDCP), it's
802 * only the error on driver unload that we want to silence.
803 * - The worker is responsible for multiple submissions (GSC FW load,
804 * HuC auth, SW proxy), so all of those will have to be adapted to
805 * handle the wedged_on_fini scenario.
806 * Therefore, it's much simpler to just wait for the worker to be done
807 * before wedging on driver removal, also considering that the worker
808 * will likely already be idle in the great majority of non-selftest
809 * scenarios.
810 */
811 intel_gsc_uc_flush_work(gsc: &gt->uc.gsc);
812
813 /*
814 * Upon unregistering the device to prevent any new users, cancel
815 * all in-flight requests so that we can quickly unbind the active
816 * resources.
817 */
818 intel_gt_set_wedged_on_fini(gt);
819
820 /* Scrub all HW state upon release */
821 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
822 intel_gt_reset_all_engines(gt);
823}
824
825void intel_gt_driver_release(struct intel_gt *gt)
826{
827 struct i915_address_space *vm;
828
829 vm = fetch_and_zero(&gt->vm);
830 if (vm) /* FIXME being called twice on error paths :( */
831 i915_vm_put(vm);
832
833 intel_wa_list_free(wal: &gt->wa_list);
834 intel_gt_pm_fini(gt);
835 intel_gt_fini_scratch(gt);
836 intel_gt_fini_buffer_pool(gt);
837 intel_gt_fini_hwconfig(gt);
838}
839
840void intel_gt_driver_late_release_all(struct drm_i915_private *i915)
841{
842 struct intel_gt *gt;
843 unsigned int id;
844
845 /* We need to wait for inflight RCU frees to release their grip */
846 rcu_barrier();
847
848 for_each_gt(gt, i915, id) {
849 intel_uc_driver_late_release(uc: &gt->uc);
850 intel_gt_fini_requests(gt);
851 intel_gt_fini_reset(gt);
852 intel_gt_fini_timelines(gt);
853 intel_gt_fini_tlb(gt);
854 intel_engines_free(gt);
855 }
856}
857
858static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)
859{
860 int ret;
861
862 if (!gt_is_root(gt)) {
863 struct intel_uncore *uncore;
864 spinlock_t *irq_lock;
865
866 uncore = drmm_kzalloc(dev: &gt->i915->drm, size: sizeof(*uncore), GFP_KERNEL);
867 if (!uncore)
868 return -ENOMEM;
869
870 irq_lock = drmm_kzalloc(dev: &gt->i915->drm, size: sizeof(*irq_lock), GFP_KERNEL);
871 if (!irq_lock)
872 return -ENOMEM;
873
874 gt->uncore = uncore;
875 gt->irq_lock = irq_lock;
876
877 intel_gt_common_init_early(gt);
878 }
879
880 intel_uncore_init_early(uncore: gt->uncore, gt);
881
882 ret = intel_uncore_setup_mmio(uncore: gt->uncore, phys_addr);
883 if (ret)
884 return ret;
885
886 gt->phys_addr = phys_addr;
887
888 return 0;
889}
890
891int intel_gt_probe_all(struct drm_i915_private *i915)
892{
893 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
894 struct intel_gt *gt = to_gt(i915);
895 const struct intel_gt_definition *gtdef;
896 phys_addr_t phys_addr;
897 unsigned int mmio_bar;
898 unsigned int i;
899 int ret;
900
901 mmio_bar = intel_mmio_bar(GRAPHICS_VER(i915));
902 phys_addr = pci_resource_start(pdev, mmio_bar);
903
904 /*
905 * We always have at least one primary GT on any device
906 * and it has been already initialized early during probe
907 * in i915_driver_probe()
908 */
909 gt->i915 = i915;
910 gt->name = "Primary GT";
911 gt->info.engine_mask = INTEL_INFO(i915)->platform_engine_mask;
912
913 gt_dbg(gt, "Setting up %s\n", gt->name);
914 ret = intel_gt_tile_setup(gt, phys_addr);
915 if (ret)
916 return ret;
917
918 if (!HAS_EXTRA_GT_LIST(i915))
919 return 0;
920
921 for (i = 1, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1];
922 gtdef->name != NULL;
923 i++, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]) {
924 gt = drmm_kzalloc(dev: &i915->drm, size: sizeof(*gt), GFP_KERNEL);
925 if (!gt) {
926 ret = -ENOMEM;
927 goto err;
928 }
929
930 gt->i915 = i915;
931 gt->name = gtdef->name;
932 gt->type = gtdef->type;
933 gt->info.engine_mask = gtdef->engine_mask;
934 gt->info.id = i;
935
936 gt_dbg(gt, "Setting up %s\n", gt->name);
937 if (GEM_WARN_ON(range_overflows_t(resource_size_t,
938 gtdef->mapping_base,
939 SZ_16M,
940 pci_resource_len(pdev, mmio_bar)))) {
941 ret = -ENODEV;
942 goto err;
943 }
944
945 switch (gtdef->type) {
946 case GT_TILE:
947 ret = intel_gt_tile_setup(gt, phys_addr: phys_addr + gtdef->mapping_base);
948 break;
949
950 case GT_MEDIA:
951 ret = intel_sa_mediagt_setup(gt, phys_addr: phys_addr + gtdef->mapping_base,
952 gsi_offset: gtdef->gsi_offset);
953 break;
954
955 case GT_PRIMARY:
956 /* Primary GT should not appear in extra GT list */
957 default:
958 MISSING_CASE(gtdef->type);
959 ret = -ENODEV;
960 }
961
962 if (ret)
963 goto err;
964
965 i915->gt[i] = gt;
966 }
967
968 return 0;
969
970err:
971 i915_probe_error(i915, "Failed to initialize %s! (%d)\n", gtdef->name, ret);
972 return ret;
973}
974
975int intel_gt_tiles_init(struct drm_i915_private *i915)
976{
977 struct intel_gt *gt;
978 unsigned int id;
979 int ret;
980
981 for_each_gt(gt, i915, id) {
982 ret = intel_gt_probe_lmem(gt);
983 if (ret)
984 return ret;
985 }
986
987 return 0;
988}
989
990void intel_gt_info_print(const struct intel_gt_info *info,
991 struct drm_printer *p)
992{
993 drm_printf(p, f: "available engines: %x\n", info->engine_mask);
994
995 intel_sseu_dump(sseu: &info->sseu, p);
996}
997
998enum i915_map_type intel_gt_coherent_map_type(struct intel_gt *gt,
999 struct drm_i915_gem_object *obj,
1000 bool always_coherent)
1001{
1002 /*
1003 * Wa_22016122933: always return I915_MAP_WC for Media
1004 * version 13.0 when the object is on the Media GT
1005 */
1006 if (i915_gem_object_is_lmem(obj) || intel_gt_needs_wa_22016122933(gt))
1007 return I915_MAP_WC;
1008 if (HAS_LLC(gt->i915) || always_coherent)
1009 return I915_MAP_WB;
1010 else
1011 return I915_MAP_WC;
1012}
1013
1014bool intel_gt_needs_wa_16018031267(struct intel_gt *gt)
1015{
1016 /* Wa_16018031267, Wa_16018063123 */
1017 return IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 55), IP_VER(12, 71));
1018}
1019
1020bool intel_gt_needs_wa_22016122933(struct intel_gt *gt)
1021{
1022 return MEDIA_VER_FULL(gt->i915) == IP_VER(13, 0) && gt->type == GT_MEDIA;
1023}
1024
1025static void __intel_gt_bind_context_set_ready(struct intel_gt *gt, bool ready)
1026{
1027 struct intel_engine_cs *engine = gt->engine[BCS0];
1028
1029 if (engine && engine->bind_context)
1030 engine->bind_context_ready = ready;
1031}
1032
1033/**
1034 * intel_gt_bind_context_set_ready - Set the context binding as ready
1035 *
1036 * @gt: GT structure
1037 *
1038 * This function marks the binder context as ready.
1039 */
1040void intel_gt_bind_context_set_ready(struct intel_gt *gt)
1041{
1042 __intel_gt_bind_context_set_ready(gt, ready: true);
1043}
1044
1045/**
1046 * intel_gt_bind_context_set_unready - Set the context binding as ready
1047 * @gt: GT structure
1048 *
1049 * This function marks the binder context as not ready.
1050 */
1051
1052void intel_gt_bind_context_set_unready(struct intel_gt *gt)
1053{
1054 __intel_gt_bind_context_set_ready(gt, ready: false);
1055}
1056
1057/**
1058 * intel_gt_is_bind_context_ready - Check if context binding is ready
1059 *
1060 * @gt: GT structure
1061 *
1062 * This function returns binder context's ready status.
1063 */
1064bool intel_gt_is_bind_context_ready(struct intel_gt *gt)
1065{
1066 struct intel_engine_cs *engine = gt->engine[BCS0];
1067
1068 if (engine)
1069 return engine->bind_context_ready;
1070
1071 return false;
1072}
1073