1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2014 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7
8#include "gem/i915_gem_context.h"
9#include "gem/i915_gem_lmem.h"
10#include "gt/gen8_engine_cs.h"
11#include "gt/intel_breadcrumbs.h"
12#include "gt/intel_context.h"
13#include "gt/intel_engine_heartbeat.h"
14#include "gt/intel_engine_pm.h"
15#include "gt/intel_engine_regs.h"
16#include "gt/intel_gpu_commands.h"
17#include "gt/intel_gt.h"
18#include "gt/intel_gt_clock_utils.h"
19#include "gt/intel_gt_irq.h"
20#include "gt/intel_gt_pm.h"
21#include "gt/intel_gt_regs.h"
22#include "gt/intel_gt_requests.h"
23#include "gt/intel_lrc.h"
24#include "gt/intel_lrc_reg.h"
25#include "gt/intel_mocs.h"
26#include "gt/intel_ring.h"
27
28#include "i915_drv.h"
29#include "i915_irq.h"
30#include "i915_reg.h"
31#include "i915_trace.h"
32#include "i915_wait_util.h"
33#include "intel_guc_ads.h"
34#include "intel_guc_capture.h"
35#include "intel_guc_print.h"
36#include "intel_guc_submission.h"
37
38/**
39 * DOC: GuC-based command submission
40 *
41 * The Scratch registers:
42 * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
43 * a value to the action register (SOFT_SCRATCH_0) along with any data. It then
44 * triggers an interrupt on the GuC via another register write (0xC4C8).
45 * Firmware writes a success/fail code back to the action register after
46 * processes the request. The kernel driver polls waiting for this update and
47 * then proceeds.
48 *
49 * Command Transport buffers (CTBs):
50 * Covered in detail in other sections but CTBs (Host to GuC - H2G, GuC to Host
51 * - G2H) are a message interface between the i915 and GuC.
52 *
53 * Context registration:
54 * Before a context can be submitted it must be registered with the GuC via a
55 * H2G. A unique guc_id is associated with each context. The context is either
56 * registered at request creation time (normal operation) or at submission time
57 * (abnormal operation, e.g. after a reset).
58 *
59 * Context submission:
60 * The i915 updates the LRC tail value in memory. The i915 must enable the
61 * scheduling of the context within the GuC for the GuC to actually consider it.
62 * Therefore, the first time a disabled context is submitted we use a schedule
63 * enable H2G, while follow up submissions are done via the context submit H2G,
64 * which informs the GuC that a previously enabled context has new work
65 * available.
66 *
67 * Context unpin:
68 * To unpin a context a H2G is used to disable scheduling. When the
69 * corresponding G2H returns indicating the scheduling disable operation has
70 * completed it is safe to unpin the context. While a disable is in flight it
71 * isn't safe to resubmit the context so a fence is used to stall all future
72 * requests of that context until the G2H is returned. Because this interaction
73 * with the GuC takes a non-zero amount of time we delay the disabling of
74 * scheduling after the pin count goes to zero by a configurable period of time
75 * (see SCHED_DISABLE_DELAY_MS). The thought is this gives the user a window of
76 * time to resubmit something on the context before doing this costly operation.
77 * This delay is only done if the context isn't closed and the guc_id usage is
78 * less than a threshold (see NUM_SCHED_DISABLE_GUC_IDS_THRESHOLD).
79 *
80 * Context deregistration:
81 * Before a context can be destroyed or if we steal its guc_id we must
82 * deregister the context with the GuC via H2G. If stealing the guc_id it isn't
83 * safe to submit anything to this guc_id until the deregister completes so a
84 * fence is used to stall all requests associated with this guc_id until the
85 * corresponding G2H returns indicating the guc_id has been deregistered.
86 *
87 * submission_state.guc_ids:
88 * Unique number associated with private GuC context data passed in during
89 * context registration / submission / deregistration. 64k available. Simple ida
90 * is used for allocation.
91 *
92 * Stealing guc_ids:
93 * If no guc_ids are available they can be stolen from another context at
94 * request creation time if that context is unpinned. If a guc_id can't be found
95 * we punt this problem to the user as we believe this is near impossible to hit
96 * during normal use cases.
97 *
98 * Locking:
99 * In the GuC submission code we have 3 basic spin locks which protect
100 * everything. Details about each below.
101 *
102 * sched_engine->lock
103 * This is the submission lock for all contexts that share an i915 schedule
104 * engine (sched_engine), thus only one of the contexts which share a
105 * sched_engine can be submitting at a time. Currently only one sched_engine is
106 * used for all of GuC submission but that could change in the future.
107 *
108 * guc->submission_state.lock
109 * Global lock for GuC submission state. Protects guc_ids and destroyed contexts
110 * list.
111 *
112 * ce->guc_state.lock
113 * Protects everything under ce->guc_state. Ensures that a context is in the
114 * correct state before issuing a H2G. e.g. We don't issue a schedule disable
115 * on a disabled context (bad idea), we don't issue a schedule enable when a
116 * schedule disable is in flight, etc... Also protects list of inflight requests
117 * on the context and the priority management state. Lock is individual to each
118 * context.
119 *
120 * Lock ordering rules:
121 * sched_engine->lock -> ce->guc_state.lock
122 * guc->submission_state.lock -> ce->guc_state.lock
123 *
124 * Reset races:
125 * When a full GT reset is triggered it is assumed that some G2H responses to
126 * H2Gs can be lost as the GuC is also reset. Losing these G2H can prove to be
127 * fatal as we do certain operations upon receiving a G2H (e.g. destroy
128 * contexts, release guc_ids, etc...). When this occurs we can scrub the
129 * context state and cleanup appropriately, however this is quite racey.
130 * To avoid races, the reset code must disable submission before scrubbing for
131 * the missing G2H, while the submission code must check for submission being
132 * disabled and skip sending H2Gs and updating context states when it is. Both
133 * sides must also make sure to hold the relevant locks.
134 */
135
136/* GuC Virtual Engine */
137struct guc_virtual_engine {
138 struct intel_engine_cs base;
139 struct intel_context context;
140};
141
142static struct intel_context *
143guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
144 unsigned long flags);
145
146static struct intel_context *
147guc_create_parallel(struct intel_engine_cs **engines,
148 unsigned int num_siblings,
149 unsigned int width);
150
151#define GUC_REQUEST_SIZE 64 /* bytes */
152
153/*
154 * We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous
155 * per the GuC submission interface. A different allocation algorithm is used
156 * (bitmap vs. ida) between multi-lrc and single-lrc hence the reason to
157 * partition the guc_id space. We believe the number of multi-lrc contexts in
158 * use should be low and 1/16 should be sufficient. Minimum of 32 guc_ids for
159 * multi-lrc.
160 */
161#define NUMBER_MULTI_LRC_GUC_ID(guc) \
162 ((guc)->submission_state.num_guc_ids / 16)
163
164/*
165 * Below is a set of functions which control the GuC scheduling state which
166 * require a lock.
167 */
168#define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER BIT(0)
169#define SCHED_STATE_DESTROYED BIT(1)
170#define SCHED_STATE_PENDING_DISABLE BIT(2)
171#define SCHED_STATE_BANNED BIT(3)
172#define SCHED_STATE_ENABLED BIT(4)
173#define SCHED_STATE_PENDING_ENABLE BIT(5)
174#define SCHED_STATE_REGISTERED BIT(6)
175#define SCHED_STATE_POLICY_REQUIRED BIT(7)
176#define SCHED_STATE_CLOSED BIT(8)
177#define SCHED_STATE_BLOCKED_SHIFT 9
178#define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT)
179#define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT)
180
181static inline void init_sched_state(struct intel_context *ce)
182{
183 lockdep_assert_held(&ce->guc_state.lock);
184 ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK;
185}
186
187/*
188 * Kernel contexts can have SCHED_STATE_REGISTERED after suspend.
189 * A context close can race with the submission path, so SCHED_STATE_CLOSED
190 * can be set immediately before we try to register.
191 */
192#define SCHED_STATE_VALID_INIT \
193 (SCHED_STATE_BLOCKED_MASK | \
194 SCHED_STATE_CLOSED | \
195 SCHED_STATE_REGISTERED)
196
197__maybe_unused
198static bool sched_state_is_init(struct intel_context *ce)
199{
200 return !(ce->guc_state.sched_state & ~SCHED_STATE_VALID_INIT);
201}
202
203static inline bool
204context_wait_for_deregister_to_register(struct intel_context *ce)
205{
206 return ce->guc_state.sched_state &
207 SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
208}
209
210static inline void
211set_context_wait_for_deregister_to_register(struct intel_context *ce)
212{
213 lockdep_assert_held(&ce->guc_state.lock);
214 ce->guc_state.sched_state |=
215 SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
216}
217
218static inline void
219clr_context_wait_for_deregister_to_register(struct intel_context *ce)
220{
221 lockdep_assert_held(&ce->guc_state.lock);
222 ce->guc_state.sched_state &=
223 ~SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
224}
225
226static inline bool
227context_destroyed(struct intel_context *ce)
228{
229 return ce->guc_state.sched_state & SCHED_STATE_DESTROYED;
230}
231
232static inline void
233set_context_destroyed(struct intel_context *ce)
234{
235 lockdep_assert_held(&ce->guc_state.lock);
236 ce->guc_state.sched_state |= SCHED_STATE_DESTROYED;
237}
238
239static inline void
240clr_context_destroyed(struct intel_context *ce)
241{
242 lockdep_assert_held(&ce->guc_state.lock);
243 ce->guc_state.sched_state &= ~SCHED_STATE_DESTROYED;
244}
245
246static inline bool context_pending_disable(struct intel_context *ce)
247{
248 return ce->guc_state.sched_state & SCHED_STATE_PENDING_DISABLE;
249}
250
251static inline void set_context_pending_disable(struct intel_context *ce)
252{
253 lockdep_assert_held(&ce->guc_state.lock);
254 ce->guc_state.sched_state |= SCHED_STATE_PENDING_DISABLE;
255}
256
257static inline void clr_context_pending_disable(struct intel_context *ce)
258{
259 lockdep_assert_held(&ce->guc_state.lock);
260 ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_DISABLE;
261}
262
263static inline bool context_banned(struct intel_context *ce)
264{
265 return ce->guc_state.sched_state & SCHED_STATE_BANNED;
266}
267
268static inline void set_context_banned(struct intel_context *ce)
269{
270 lockdep_assert_held(&ce->guc_state.lock);
271 ce->guc_state.sched_state |= SCHED_STATE_BANNED;
272}
273
274static inline void clr_context_banned(struct intel_context *ce)
275{
276 lockdep_assert_held(&ce->guc_state.lock);
277 ce->guc_state.sched_state &= ~SCHED_STATE_BANNED;
278}
279
280static inline bool context_enabled(struct intel_context *ce)
281{
282 return ce->guc_state.sched_state & SCHED_STATE_ENABLED;
283}
284
285static inline void set_context_enabled(struct intel_context *ce)
286{
287 lockdep_assert_held(&ce->guc_state.lock);
288 ce->guc_state.sched_state |= SCHED_STATE_ENABLED;
289}
290
291static inline void clr_context_enabled(struct intel_context *ce)
292{
293 lockdep_assert_held(&ce->guc_state.lock);
294 ce->guc_state.sched_state &= ~SCHED_STATE_ENABLED;
295}
296
297static inline bool context_pending_enable(struct intel_context *ce)
298{
299 return ce->guc_state.sched_state & SCHED_STATE_PENDING_ENABLE;
300}
301
302static inline void set_context_pending_enable(struct intel_context *ce)
303{
304 lockdep_assert_held(&ce->guc_state.lock);
305 ce->guc_state.sched_state |= SCHED_STATE_PENDING_ENABLE;
306}
307
308static inline void clr_context_pending_enable(struct intel_context *ce)
309{
310 lockdep_assert_held(&ce->guc_state.lock);
311 ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_ENABLE;
312}
313
314static inline bool context_registered(struct intel_context *ce)
315{
316 return ce->guc_state.sched_state & SCHED_STATE_REGISTERED;
317}
318
319static inline void set_context_registered(struct intel_context *ce)
320{
321 lockdep_assert_held(&ce->guc_state.lock);
322 ce->guc_state.sched_state |= SCHED_STATE_REGISTERED;
323}
324
325static inline void clr_context_registered(struct intel_context *ce)
326{
327 lockdep_assert_held(&ce->guc_state.lock);
328 ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED;
329}
330
331static inline bool context_policy_required(struct intel_context *ce)
332{
333 return ce->guc_state.sched_state & SCHED_STATE_POLICY_REQUIRED;
334}
335
336static inline void set_context_policy_required(struct intel_context *ce)
337{
338 lockdep_assert_held(&ce->guc_state.lock);
339 ce->guc_state.sched_state |= SCHED_STATE_POLICY_REQUIRED;
340}
341
342static inline void clr_context_policy_required(struct intel_context *ce)
343{
344 lockdep_assert_held(&ce->guc_state.lock);
345 ce->guc_state.sched_state &= ~SCHED_STATE_POLICY_REQUIRED;
346}
347
348static inline bool context_close_done(struct intel_context *ce)
349{
350 return ce->guc_state.sched_state & SCHED_STATE_CLOSED;
351}
352
353static inline void set_context_close_done(struct intel_context *ce)
354{
355 lockdep_assert_held(&ce->guc_state.lock);
356 ce->guc_state.sched_state |= SCHED_STATE_CLOSED;
357}
358
359static inline u32 context_blocked(struct intel_context *ce)
360{
361 return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >>
362 SCHED_STATE_BLOCKED_SHIFT;
363}
364
365static inline void incr_context_blocked(struct intel_context *ce)
366{
367 lockdep_assert_held(&ce->guc_state.lock);
368
369 ce->guc_state.sched_state += SCHED_STATE_BLOCKED;
370
371 GEM_BUG_ON(!context_blocked(ce)); /* Overflow check */
372}
373
374static inline void decr_context_blocked(struct intel_context *ce)
375{
376 lockdep_assert_held(&ce->guc_state.lock);
377
378 GEM_BUG_ON(!context_blocked(ce)); /* Underflow check */
379
380 ce->guc_state.sched_state -= SCHED_STATE_BLOCKED;
381}
382
383static struct intel_context *
384request_to_scheduling_context(struct i915_request *rq)
385{
386 return intel_context_to_parent(ce: rq->context);
387}
388
389static inline bool context_guc_id_invalid(struct intel_context *ce)
390{
391 return ce->guc_id.id == GUC_INVALID_CONTEXT_ID;
392}
393
394static inline void set_context_guc_id_invalid(struct intel_context *ce)
395{
396 ce->guc_id.id = GUC_INVALID_CONTEXT_ID;
397}
398
399static inline struct intel_guc *ce_to_guc(struct intel_context *ce)
400{
401 return gt_to_guc(gt: ce->engine->gt);
402}
403
404static inline struct i915_priolist *to_priolist(struct rb_node *rb)
405{
406 return rb_entry(rb, struct i915_priolist, node);
407}
408
409/*
410 * When using multi-lrc submission a scratch memory area is reserved in the
411 * parent's context state for the process descriptor, work queue, and handshake
412 * between the parent + children contexts to insert safe preemption points
413 * between each of the BBs. Currently the scratch area is sized to a page.
414 *
415 * The layout of this scratch area is below:
416 * 0 guc_process_desc
417 * + sizeof(struct guc_process_desc) child go
418 * + CACHELINE_BYTES child join[0]
419 * ...
420 * + CACHELINE_BYTES child join[n - 1]
421 * ... unused
422 * PARENT_SCRATCH_SIZE / 2 work queue start
423 * ... work queue
424 * PARENT_SCRATCH_SIZE - 1 work queue end
425 */
426#define WQ_SIZE (PARENT_SCRATCH_SIZE / 2)
427#define WQ_OFFSET (PARENT_SCRATCH_SIZE - WQ_SIZE)
428
429struct sync_semaphore {
430 u32 semaphore;
431 u8 unused[CACHELINE_BYTES - sizeof(u32)];
432};
433
434struct parent_scratch {
435 union guc_descs {
436 struct guc_sched_wq_desc wq_desc;
437 struct guc_process_desc_v69 pdesc;
438 } descs;
439
440 struct sync_semaphore go;
441 struct sync_semaphore join[MAX_ENGINE_INSTANCE + 1];
442
443 u8 unused[WQ_OFFSET - sizeof(union guc_descs) -
444 sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE + 2)];
445
446 u32 wq[WQ_SIZE / sizeof(u32)];
447};
448
449static u32 __get_parent_scratch_offset(struct intel_context *ce)
450{
451 GEM_BUG_ON(!ce->parallel.guc.parent_page);
452
453 return ce->parallel.guc.parent_page * PAGE_SIZE;
454}
455
456static u32 __get_wq_offset(struct intel_context *ce)
457{
458 BUILD_BUG_ON(offsetof(struct parent_scratch, wq) != WQ_OFFSET);
459
460 return __get_parent_scratch_offset(ce) + WQ_OFFSET;
461}
462
463static struct parent_scratch *
464__get_parent_scratch(struct intel_context *ce)
465{
466 BUILD_BUG_ON(sizeof(struct parent_scratch) != PARENT_SCRATCH_SIZE);
467 BUILD_BUG_ON(sizeof(struct sync_semaphore) != CACHELINE_BYTES);
468
469 /*
470 * Need to subtract LRC_STATE_OFFSET here as the
471 * parallel.guc.parent_page is the offset into ce->state while
472 * ce->lrc_reg_reg is ce->state + LRC_STATE_OFFSET.
473 */
474 return (struct parent_scratch *)
475 (ce->lrc_reg_state +
476 ((__get_parent_scratch_offset(ce) -
477 LRC_STATE_OFFSET) / sizeof(u32)));
478}
479
480static struct guc_process_desc_v69 *
481__get_process_desc_v69(struct intel_context *ce)
482{
483 struct parent_scratch *ps = __get_parent_scratch(ce);
484
485 return &ps->descs.pdesc;
486}
487
488static struct guc_sched_wq_desc *
489__get_wq_desc_v70(struct intel_context *ce)
490{
491 struct parent_scratch *ps = __get_parent_scratch(ce);
492
493 return &ps->descs.wq_desc;
494}
495
496static u32 *get_wq_pointer(struct intel_context *ce, u32 wqi_size)
497{
498 /*
499 * Check for space in work queue. Caching a value of head pointer in
500 * intel_context structure in order reduce the number accesses to shared
501 * GPU memory which may be across a PCIe bus.
502 */
503#define AVAILABLE_SPACE \
504 CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE)
505 if (wqi_size > AVAILABLE_SPACE) {
506 ce->parallel.guc.wqi_head = READ_ONCE(*ce->parallel.guc.wq_head);
507
508 if (wqi_size > AVAILABLE_SPACE)
509 return NULL;
510 }
511#undef AVAILABLE_SPACE
512
513 return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)];
514}
515
516static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id)
517{
518 struct intel_context *ce = xa_load(&guc->context_lookup, index: id);
519
520 GEM_BUG_ON(id >= GUC_MAX_CONTEXT_ID);
521
522 return ce;
523}
524
525static struct guc_lrc_desc_v69 *__get_lrc_desc_v69(struct intel_guc *guc, u32 index)
526{
527 struct guc_lrc_desc_v69 *base = guc->lrc_desc_pool_vaddr_v69;
528
529 if (!base)
530 return NULL;
531
532 GEM_BUG_ON(index >= GUC_MAX_CONTEXT_ID);
533
534 return &base[index];
535}
536
537static int guc_lrc_desc_pool_create_v69(struct intel_guc *guc)
538{
539 u32 size;
540 int ret;
541
542 size = PAGE_ALIGN(sizeof(struct guc_lrc_desc_v69) *
543 GUC_MAX_CONTEXT_ID);
544 ret = intel_guc_allocate_and_map_vma(guc, size, out_vma: &guc->lrc_desc_pool_v69,
545 out_vaddr: (void **)&guc->lrc_desc_pool_vaddr_v69);
546 if (ret)
547 return ret;
548
549 return 0;
550}
551
552static void guc_lrc_desc_pool_destroy_v69(struct intel_guc *guc)
553{
554 if (!guc->lrc_desc_pool_vaddr_v69)
555 return;
556
557 guc->lrc_desc_pool_vaddr_v69 = NULL;
558 i915_vma_unpin_and_release(p_vma: &guc->lrc_desc_pool_v69, I915_VMA_RELEASE_MAP);
559}
560
561static inline bool guc_submission_initialized(struct intel_guc *guc)
562{
563 return guc->submission_initialized;
564}
565
566static inline void _reset_lrc_desc_v69(struct intel_guc *guc, u32 id)
567{
568 struct guc_lrc_desc_v69 *desc = __get_lrc_desc_v69(guc, index: id);
569
570 if (desc)
571 memset(s: desc, c: 0, n: sizeof(*desc));
572}
573
574static inline bool ctx_id_mapped(struct intel_guc *guc, u32 id)
575{
576 return __get_context(guc, id);
577}
578
579static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id,
580 struct intel_context *ce)
581{
582 unsigned long flags;
583
584 /*
585 * xarray API doesn't have xa_save_irqsave wrapper, so calling the
586 * lower level functions directly.
587 */
588 xa_lock_irqsave(&guc->context_lookup, flags);
589 __xa_store(&guc->context_lookup, index: id, entry: ce, GFP_ATOMIC);
590 xa_unlock_irqrestore(&guc->context_lookup, flags);
591}
592
593static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id)
594{
595 unsigned long flags;
596
597 if (unlikely(!guc_submission_initialized(guc)))
598 return;
599
600 _reset_lrc_desc_v69(guc, id);
601
602 /*
603 * xarray API doesn't have xa_erase_irqsave wrapper, so calling
604 * the lower level functions directly.
605 */
606 xa_lock_irqsave(&guc->context_lookup, flags);
607 __xa_erase(&guc->context_lookup, index: id);
608 xa_unlock_irqrestore(&guc->context_lookup, flags);
609}
610
611static void decr_outstanding_submission_g2h(struct intel_guc *guc)
612{
613 if (atomic_dec_and_test(v: &guc->outstanding_submission_g2h))
614 wake_up_all(&guc->ct.wq);
615}
616
617static int guc_submission_send_busy_loop(struct intel_guc *guc,
618 const u32 *action,
619 u32 len,
620 u32 g2h_len_dw,
621 bool loop)
622{
623 int ret;
624
625 /*
626 * We always loop when a send requires a reply (i.e. g2h_len_dw > 0),
627 * so we don't handle the case where we don't get a reply because we
628 * aborted the send due to the channel being busy.
629 */
630 GEM_BUG_ON(g2h_len_dw && !loop);
631
632 if (g2h_len_dw)
633 atomic_inc(v: &guc->outstanding_submission_g2h);
634
635 ret = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
636 if (ret && g2h_len_dw)
637 atomic_dec(v: &guc->outstanding_submission_g2h);
638
639 return ret;
640}
641
642int intel_guc_wait_for_pending_msg(struct intel_guc *guc,
643 atomic_t *wait_var,
644 bool interruptible,
645 long timeout)
646{
647 const int state = interruptible ?
648 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
649 DEFINE_WAIT(wait);
650
651 might_sleep();
652 GEM_BUG_ON(timeout < 0);
653
654 if (!atomic_read(v: wait_var))
655 return 0;
656
657 if (!timeout)
658 return -ETIME;
659
660 for (;;) {
661 prepare_to_wait(wq_head: &guc->ct.wq, wq_entry: &wait, state);
662
663 if (!atomic_read(v: wait_var))
664 break;
665
666 if (signal_pending_state(state, current)) {
667 timeout = -EINTR;
668 break;
669 }
670
671 if (!timeout) {
672 timeout = -ETIME;
673 break;
674 }
675
676 timeout = io_schedule_timeout(timeout);
677 }
678 finish_wait(wq_head: &guc->ct.wq, wq_entry: &wait);
679
680 return (timeout < 0) ? timeout : 0;
681}
682
683int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout)
684{
685 if (!intel_uc_uses_guc_submission(uc: &guc_to_gt(guc)->uc))
686 return 0;
687
688 return intel_guc_wait_for_pending_msg(guc,
689 wait_var: &guc->outstanding_submission_g2h,
690 interruptible: true, timeout);
691}
692
693static int guc_context_policy_init_v70(struct intel_context *ce, bool loop);
694static int try_context_registration(struct intel_context *ce, bool loop);
695
696static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)
697{
698 int err = 0;
699 struct intel_context *ce = request_to_scheduling_context(rq);
700 u32 action[3];
701 int len = 0;
702 u32 g2h_len_dw = 0;
703 bool enabled;
704
705 lockdep_assert_held(&rq->engine->sched_engine->lock);
706
707 /*
708 * Corner case where requests were sitting in the priority list or a
709 * request resubmitted after the context was banned.
710 */
711 if (unlikely(!intel_context_is_schedulable(ce))) {
712 i915_request_put(rq: i915_request_mark_eio(rq));
713 intel_engine_signal_breadcrumbs(engine: ce->engine);
714 return 0;
715 }
716
717 GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));
718 GEM_BUG_ON(context_guc_id_invalid(ce));
719
720 if (context_policy_required(ce)) {
721 err = guc_context_policy_init_v70(ce, loop: false);
722 if (err)
723 return err;
724 }
725
726 spin_lock(lock: &ce->guc_state.lock);
727
728 /*
729 * The request / context will be run on the hardware when scheduling
730 * gets enabled in the unblock. For multi-lrc we still submit the
731 * context to move the LRC tails.
732 */
733 if (unlikely(context_blocked(ce) && !intel_context_is_parent(ce)))
734 goto out;
735
736 enabled = context_enabled(ce) || context_blocked(ce);
737
738 if (!enabled) {
739 action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
740 action[len++] = ce->guc_id.id;
741 action[len++] = GUC_CONTEXT_ENABLE;
742 set_context_pending_enable(ce);
743 intel_context_get(ce);
744 g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;
745 } else {
746 action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT;
747 action[len++] = ce->guc_id.id;
748 }
749
750 err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
751 if (!enabled && !err) {
752 trace_intel_context_sched_enable(ce);
753 atomic_inc(v: &guc->outstanding_submission_g2h);
754 set_context_enabled(ce);
755
756 /*
757 * Without multi-lrc KMD does the submission step (moving the
758 * lrc tail) so enabling scheduling is sufficient to submit the
759 * context. This isn't the case in multi-lrc submission as the
760 * GuC needs to move the tails, hence the need for another H2G
761 * to submit a multi-lrc context after enabling scheduling.
762 */
763 if (intel_context_is_parent(ce)) {
764 action[0] = INTEL_GUC_ACTION_SCHED_CONTEXT;
765 err = intel_guc_send_nb(guc, action, len: len - 1, g2h_len_dw: 0);
766 }
767 } else if (!enabled) {
768 clr_context_pending_enable(ce);
769 intel_context_put(ce);
770 }
771 if (likely(!err))
772 trace_i915_request_guc_submit(rq);
773
774out:
775 spin_unlock(lock: &ce->guc_state.lock);
776 return err;
777}
778
779static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)
780{
781 int ret = __guc_add_request(guc, rq);
782
783 if (unlikely(ret == -EBUSY)) {
784 guc->stalled_request = rq;
785 guc->submission_stall_reason = STALL_ADD_REQUEST;
786 }
787
788 return ret;
789}
790
791static inline void guc_set_lrc_tail(struct i915_request *rq)
792{
793 rq->context->lrc_reg_state[CTX_RING_TAIL] =
794 intel_ring_set_tail(ring: rq->ring, tail: rq->tail);
795}
796
797static inline int rq_prio(const struct i915_request *rq)
798{
799 return rq->sched.attr.priority;
800}
801
802static bool is_multi_lrc_rq(struct i915_request *rq)
803{
804 return intel_context_is_parallel(ce: rq->context);
805}
806
807static bool can_merge_rq(struct i915_request *rq,
808 struct i915_request *last)
809{
810 return request_to_scheduling_context(rq) ==
811 request_to_scheduling_context(rq: last);
812}
813
814static u32 wq_space_until_wrap(struct intel_context *ce)
815{
816 return (WQ_SIZE - ce->parallel.guc.wqi_tail);
817}
818
819static void write_wqi(struct intel_context *ce, u32 wqi_size)
820{
821 BUILD_BUG_ON(!is_power_of_2(WQ_SIZE));
822
823 /*
824 * Ensure WQI are visible before updating tail
825 */
826 intel_guc_write_barrier(guc: ce_to_guc(ce));
827
828 ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) &
829 (WQ_SIZE - 1);
830 WRITE_ONCE(*ce->parallel.guc.wq_tail, ce->parallel.guc.wqi_tail);
831}
832
833static int guc_wq_noop_append(struct intel_context *ce)
834{
835 u32 *wqi = get_wq_pointer(ce, wqi_size: wq_space_until_wrap(ce));
836 u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1;
837
838 if (!wqi)
839 return -EBUSY;
840
841 GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));
842
843 *wqi = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
844 FIELD_PREP(WQ_LEN_MASK, len_dw);
845 ce->parallel.guc.wqi_tail = 0;
846
847 return 0;
848}
849
850static int __guc_wq_item_append(struct i915_request *rq)
851{
852 struct intel_context *ce = request_to_scheduling_context(rq);
853 struct intel_context *child;
854 unsigned int wqi_size = (ce->parallel.number_children + 4) *
855 sizeof(u32);
856 u32 *wqi;
857 u32 len_dw = (wqi_size / sizeof(u32)) - 1;
858 int ret;
859
860 /* Ensure context is in correct state updating work queue */
861 GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));
862 GEM_BUG_ON(context_guc_id_invalid(ce));
863 GEM_BUG_ON(context_wait_for_deregister_to_register(ce));
864 GEM_BUG_ON(!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id));
865
866 /* Insert NOOP if this work queue item will wrap the tail pointer. */
867 if (wqi_size > wq_space_until_wrap(ce)) {
868 ret = guc_wq_noop_append(ce);
869 if (ret)
870 return ret;
871 }
872
873 wqi = get_wq_pointer(ce, wqi_size);
874 if (!wqi)
875 return -EBUSY;
876
877 GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));
878
879 *wqi++ = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |
880 FIELD_PREP(WQ_LEN_MASK, len_dw);
881 *wqi++ = ce->lrc.lrca;
882 *wqi++ = FIELD_PREP(WQ_GUC_ID_MASK, ce->guc_id.id) |
883 FIELD_PREP(WQ_RING_TAIL_MASK, ce->ring->tail / sizeof(u64));
884 *wqi++ = 0; /* fence_id */
885 for_each_child(ce, child)
886 *wqi++ = child->ring->tail / sizeof(u64);
887
888 write_wqi(ce, wqi_size);
889
890 return 0;
891}
892
893static int guc_wq_item_append(struct intel_guc *guc,
894 struct i915_request *rq)
895{
896 struct intel_context *ce = request_to_scheduling_context(rq);
897 int ret;
898
899 if (unlikely(!intel_context_is_schedulable(ce)))
900 return 0;
901
902 ret = __guc_wq_item_append(rq);
903 if (unlikely(ret == -EBUSY)) {
904 guc->stalled_request = rq;
905 guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
906 }
907
908 return ret;
909}
910
911static bool multi_lrc_submit(struct i915_request *rq)
912{
913 struct intel_context *ce = request_to_scheduling_context(rq);
914
915 intel_ring_set_tail(ring: rq->ring, tail: rq->tail);
916
917 /*
918 * We expect the front end (execbuf IOCTL) to set this flag on the last
919 * request generated from a multi-BB submission. This indicates to the
920 * backend (GuC interface) that we should submit this context thus
921 * submitting all the requests generated in parallel.
922 */
923 return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) ||
924 !intel_context_is_schedulable(ce);
925}
926
927static int guc_dequeue_one_context(struct intel_guc *guc)
928{
929 struct i915_sched_engine * const sched_engine = guc->sched_engine;
930 struct i915_request *last = NULL;
931 bool submit = false;
932 struct rb_node *rb;
933 int ret;
934
935 lockdep_assert_held(&sched_engine->lock);
936
937 if (guc->stalled_request) {
938 submit = true;
939 last = guc->stalled_request;
940
941 switch (guc->submission_stall_reason) {
942 case STALL_REGISTER_CONTEXT:
943 goto register_context;
944 case STALL_MOVE_LRC_TAIL:
945 goto move_lrc_tail;
946 case STALL_ADD_REQUEST:
947 goto add_request;
948 default:
949 MISSING_CASE(guc->submission_stall_reason);
950 }
951 }
952
953 while ((rb = rb_first_cached(&sched_engine->queue))) {
954 struct i915_priolist *p = to_priolist(rb);
955 struct i915_request *rq, *rn;
956
957 priolist_for_each_request_consume(rq, rn, p) {
958 if (last && !can_merge_rq(rq, last))
959 goto register_context;
960
961 list_del_init(entry: &rq->sched.link);
962
963 __i915_request_submit(request: rq);
964
965 trace_i915_request_in(rq, port: 0);
966 last = rq;
967
968 if (is_multi_lrc_rq(rq)) {
969 /*
970 * We need to coalesce all multi-lrc requests in
971 * a relationship into a single H2G. We are
972 * guaranteed that all of these requests will be
973 * submitted sequentially.
974 */
975 if (multi_lrc_submit(rq)) {
976 submit = true;
977 goto register_context;
978 }
979 } else {
980 submit = true;
981 }
982 }
983
984 rb_erase_cached(node: &p->node, root: &sched_engine->queue);
985 i915_priolist_free(p);
986 }
987
988register_context:
989 if (submit) {
990 struct intel_context *ce = request_to_scheduling_context(rq: last);
991
992 if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&
993 intel_context_is_schedulable(ce))) {
994 ret = try_context_registration(ce, loop: false);
995 if (unlikely(ret == -EPIPE)) {
996 goto deadlk;
997 } else if (ret == -EBUSY) {
998 guc->stalled_request = last;
999 guc->submission_stall_reason =
1000 STALL_REGISTER_CONTEXT;
1001 goto schedule_tasklet;
1002 } else if (ret != 0) {
1003 GEM_WARN_ON(ret); /* Unexpected */
1004 goto deadlk;
1005 }
1006 }
1007
1008move_lrc_tail:
1009 if (is_multi_lrc_rq(rq: last)) {
1010 ret = guc_wq_item_append(guc, rq: last);
1011 if (ret == -EBUSY) {
1012 goto schedule_tasklet;
1013 } else if (ret != 0) {
1014 GEM_WARN_ON(ret); /* Unexpected */
1015 goto deadlk;
1016 }
1017 } else {
1018 guc_set_lrc_tail(rq: last);
1019 }
1020
1021add_request:
1022 ret = guc_add_request(guc, rq: last);
1023 if (unlikely(ret == -EPIPE)) {
1024 goto deadlk;
1025 } else if (ret == -EBUSY) {
1026 goto schedule_tasklet;
1027 } else if (ret != 0) {
1028 GEM_WARN_ON(ret); /* Unexpected */
1029 goto deadlk;
1030 }
1031 }
1032
1033 guc->stalled_request = NULL;
1034 guc->submission_stall_reason = STALL_NONE;
1035 return submit;
1036
1037deadlk:
1038 sched_engine->tasklet.callback = NULL;
1039 tasklet_disable_nosync(t: &sched_engine->tasklet);
1040 return false;
1041
1042schedule_tasklet:
1043 tasklet_schedule(t: &sched_engine->tasklet);
1044 return false;
1045}
1046
1047static void guc_submission_tasklet(struct tasklet_struct *t)
1048{
1049 struct i915_sched_engine *sched_engine =
1050 from_tasklet(sched_engine, t, tasklet);
1051 unsigned long flags;
1052 bool loop;
1053
1054 spin_lock_irqsave(&sched_engine->lock, flags);
1055
1056 do {
1057 loop = guc_dequeue_one_context(guc: sched_engine->private_data);
1058 } while (loop);
1059
1060 i915_sched_engine_reset_on_empty(sched_engine);
1061
1062 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
1063}
1064
1065static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir)
1066{
1067 if (iir & GT_RENDER_USER_INTERRUPT)
1068 intel_engine_signal_breadcrumbs(engine);
1069}
1070
1071static void __guc_context_destroy(struct intel_context *ce);
1072static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);
1073static void guc_signal_context_fence(struct intel_context *ce);
1074static void guc_cancel_context_requests(struct intel_context *ce);
1075static void guc_blocked_fence_complete(struct intel_context *ce);
1076
1077static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
1078{
1079 struct intel_context *ce;
1080 unsigned long index, flags;
1081 bool pending_disable, pending_enable, deregister, destroyed, banned;
1082
1083 xa_lock_irqsave(&guc->context_lookup, flags);
1084 xa_for_each(&guc->context_lookup, index, ce) {
1085 /*
1086 * Corner case where the ref count on the object is zero but and
1087 * deregister G2H was lost. In this case we don't touch the ref
1088 * count and finish the destroy of the context.
1089 */
1090 bool do_put = kref_get_unless_zero(kref: &ce->ref);
1091
1092 xa_unlock(&guc->context_lookup);
1093
1094 if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&
1095 (cancel_delayed_work(dwork: &ce->guc_state.sched_disable_delay_work))) {
1096 /* successful cancel so jump straight to close it */
1097 intel_context_sched_disable_unpin(ce);
1098 }
1099
1100 spin_lock(lock: &ce->guc_state.lock);
1101
1102 /*
1103 * Once we are at this point submission_disabled() is guaranteed
1104 * to be visible to all callers who set the below flags (see above
1105 * flush and flushes in reset_prepare). If submission_disabled()
1106 * is set, the caller shouldn't set these flags.
1107 */
1108
1109 destroyed = context_destroyed(ce);
1110 pending_enable = context_pending_enable(ce);
1111 pending_disable = context_pending_disable(ce);
1112 deregister = context_wait_for_deregister_to_register(ce);
1113 banned = context_banned(ce);
1114 init_sched_state(ce);
1115
1116 spin_unlock(lock: &ce->guc_state.lock);
1117
1118 if (pending_enable || destroyed || deregister) {
1119 decr_outstanding_submission_g2h(guc);
1120 if (deregister)
1121 guc_signal_context_fence(ce);
1122 if (destroyed) {
1123 intel_gt_pm_put_async_untracked(gt: guc_to_gt(guc));
1124 release_guc_id(guc, ce);
1125 __guc_context_destroy(ce);
1126 }
1127 if (pending_enable || deregister)
1128 intel_context_put(ce);
1129 }
1130
1131 /* Not mutualy exclusive with above if statement. */
1132 if (pending_disable) {
1133 guc_signal_context_fence(ce);
1134 if (banned) {
1135 guc_cancel_context_requests(ce);
1136 intel_engine_signal_breadcrumbs(engine: ce->engine);
1137 }
1138 intel_context_sched_disable_unpin(ce);
1139 decr_outstanding_submission_g2h(guc);
1140
1141 spin_lock(lock: &ce->guc_state.lock);
1142 guc_blocked_fence_complete(ce);
1143 spin_unlock(lock: &ce->guc_state.lock);
1144
1145 intel_context_put(ce);
1146 }
1147
1148 if (do_put)
1149 intel_context_put(ce);
1150 xa_lock(&guc->context_lookup);
1151 }
1152 xa_unlock_irqrestore(&guc->context_lookup, flags);
1153}
1154
1155/*
1156 * GuC stores busyness stats for each engine at context in/out boundaries. A
1157 * context 'in' logs execution start time, 'out' adds in -> out delta to total.
1158 * i915/kmd accesses 'start', 'total' and 'context id' from memory shared with
1159 * GuC.
1160 *
1161 * __i915_pmu_event_read samples engine busyness. When sampling, if context id
1162 * is valid (!= ~0) and start is non-zero, the engine is considered to be
1163 * active. For an active engine total busyness = total + (now - start), where
1164 * 'now' is the time at which the busyness is sampled. For inactive engine,
1165 * total busyness = total.
1166 *
1167 * All times are captured from GUCPMTIMESTAMP reg and are in gt clock domain.
1168 *
1169 * The start and total values provided by GuC are 32 bits and wrap around in a
1170 * few minutes. Since perf pmu provides busyness as 64 bit monotonically
1171 * increasing ns values, there is a need for this implementation to account for
1172 * overflows and extend the GuC provided values to 64 bits before returning
1173 * busyness to the user. In order to do that, a worker runs periodically at
1174 * frequency = 1/8th the time it takes for the timestamp to wrap (i.e. once in
1175 * 27 seconds for a gt clock frequency of 19.2 MHz).
1176 */
1177
1178#define WRAP_TIME_CLKS U32_MAX
1179#define POLL_TIME_CLKS (WRAP_TIME_CLKS >> 3)
1180
1181static void
1182__extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start)
1183{
1184 u32 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
1185 u32 gt_stamp_last = lower_32_bits(guc->timestamp.gt_stamp);
1186
1187 if (new_start == lower_32_bits(*prev_start))
1188 return;
1189
1190 /*
1191 * When gt is unparked, we update the gt timestamp and start the ping
1192 * worker that updates the gt_stamp every POLL_TIME_CLKS. As long as gt
1193 * is unparked, all switched in contexts will have a start time that is
1194 * within +/- POLL_TIME_CLKS of the most recent gt_stamp.
1195 *
1196 * If neither gt_stamp nor new_start has rolled over, then the
1197 * gt_stamp_hi does not need to be adjusted, however if one of them has
1198 * rolled over, we need to adjust gt_stamp_hi accordingly.
1199 *
1200 * The below conditions address the cases of new_start rollover and
1201 * gt_stamp_last rollover respectively.
1202 */
1203 if (new_start < gt_stamp_last &&
1204 (new_start - gt_stamp_last) <= POLL_TIME_CLKS)
1205 gt_stamp_hi++;
1206
1207 if (new_start > gt_stamp_last &&
1208 (gt_stamp_last - new_start) <= POLL_TIME_CLKS && gt_stamp_hi)
1209 gt_stamp_hi--;
1210
1211 *prev_start = ((u64)gt_stamp_hi << 32) | new_start;
1212}
1213
1214#define record_read(map_, field_) \
1215 iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_)
1216
1217/*
1218 * GuC updates shared memory and KMD reads it. Since this is not synchronized,
1219 * we run into a race where the value read is inconsistent. Sometimes the
1220 * inconsistency is in reading the upper MSB bytes of the last_in value when
1221 * this race occurs. 2 types of cases are seen - upper 8 bits are zero and upper
1222 * 24 bits are zero. Since these are non-zero values, it is non-trivial to
1223 * determine validity of these values. Instead we read the values multiple times
1224 * until they are consistent. In test runs, 3 attempts results in consistent
1225 * values. The upper bound is set to 6 attempts and may need to be tuned as per
1226 * any new occurrences.
1227 */
1228static void __get_engine_usage_record(struct intel_engine_cs *engine,
1229 u32 *last_in, u32 *id, u32 *total)
1230{
1231 struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine);
1232 int i = 0;
1233
1234 do {
1235 *last_in = record_read(&rec_map, last_switch_in_stamp);
1236 *id = record_read(&rec_map, current_context_index);
1237 *total = record_read(&rec_map, total_runtime);
1238
1239 if (record_read(&rec_map, last_switch_in_stamp) == *last_in &&
1240 record_read(&rec_map, current_context_index) == *id &&
1241 record_read(&rec_map, total_runtime) == *total)
1242 break;
1243 } while (++i < 6);
1244}
1245
1246static void __set_engine_usage_record(struct intel_engine_cs *engine,
1247 u32 last_in, u32 id, u32 total)
1248{
1249 struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine);
1250
1251#define record_write(map_, field_, val_) \
1252 iosys_map_wr_field(map_, 0, struct guc_engine_usage_record, field_, val_)
1253
1254 record_write(&rec_map, last_switch_in_stamp, last_in);
1255 record_write(&rec_map, current_context_index, id);
1256 record_write(&rec_map, total_runtime, total);
1257
1258#undef record_write
1259}
1260
1261static void guc_update_engine_gt_clks(struct intel_engine_cs *engine)
1262{
1263 struct intel_engine_guc_stats *stats = &engine->stats.guc;
1264 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
1265 u32 last_switch, ctx_id, total;
1266
1267 lockdep_assert_held(&guc->timestamp.lock);
1268
1269 __get_engine_usage_record(engine, last_in: &last_switch, id: &ctx_id, total: &total);
1270
1271 stats->running = ctx_id != ~0U && last_switch;
1272 if (stats->running)
1273 __extend_last_switch(guc, prev_start: &stats->start_gt_clk, new_start: last_switch);
1274
1275 /*
1276 * Instead of adjusting the total for overflow, just add the
1277 * difference from previous sample stats->total_gt_clks
1278 */
1279 if (total && total != ~0U) {
1280 stats->total_gt_clks += (u32)(total - stats->prev_total);
1281 stats->prev_total = total;
1282 }
1283}
1284
1285static u32 gpm_timestamp_shift(struct intel_gt *gt)
1286{
1287 intel_wakeref_t wakeref;
1288 u32 reg;
1289
1290 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
1291 reg = intel_uncore_read(uncore: gt->uncore, RPM_CONFIG0);
1292
1293 return 3 - REG_FIELD_GET(GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg);
1294}
1295
1296static void guc_update_pm_timestamp(struct intel_guc *guc, ktime_t *now)
1297{
1298 struct intel_gt *gt = guc_to_gt(guc);
1299 u32 gt_stamp_lo, gt_stamp_hi;
1300 u64 gpm_ts;
1301
1302 lockdep_assert_held(&guc->timestamp.lock);
1303
1304 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
1305 gpm_ts = intel_uncore_read64_2x32(uncore: gt->uncore, MISC_STATUS0,
1306 MISC_STATUS1) >> guc->timestamp.shift;
1307 gt_stamp_lo = lower_32_bits(gpm_ts);
1308 *now = ktime_get();
1309
1310 if (gt_stamp_lo < lower_32_bits(guc->timestamp.gt_stamp))
1311 gt_stamp_hi++;
1312
1313 guc->timestamp.gt_stamp = ((u64)gt_stamp_hi << 32) | gt_stamp_lo;
1314}
1315
1316/*
1317 * Unlike the execlist mode of submission total and active times are in terms of
1318 * gt clocks. The *now parameter is retained to return the cpu time at which the
1319 * busyness was sampled.
1320 */
1321static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)
1322{
1323 struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;
1324 struct i915_gpu_error *gpu_error = &engine->i915->gpu_error;
1325 struct intel_gt *gt = engine->gt;
1326 struct intel_guc *guc = gt_to_guc(gt);
1327 u64 total, gt_stamp_saved;
1328 unsigned long flags;
1329 u32 reset_count;
1330 bool in_reset;
1331 intel_wakeref_t wakeref;
1332
1333 spin_lock_irqsave(&guc->timestamp.lock, flags);
1334
1335 /*
1336 * If a reset happened, we risk reading partially updated engine
1337 * busyness from GuC, so we just use the driver stored copy of busyness.
1338 * Synchronize with gt reset using reset_count and the
1339 * I915_RESET_BACKOFF flag. Note that reset flow updates the reset_count
1340 * after I915_RESET_BACKOFF flag, so ensure that the reset_count is
1341 * usable by checking the flag afterwards.
1342 */
1343 reset_count = i915_reset_count(error: gpu_error);
1344 in_reset = test_bit(I915_RESET_BACKOFF, &gt->reset.flags);
1345
1346 *now = ktime_get();
1347
1348 /*
1349 * The active busyness depends on start_gt_clk and gt_stamp.
1350 * gt_stamp is updated by i915 only when gt is awake and the
1351 * start_gt_clk is derived from GuC state. To get a consistent
1352 * view of activity, we query the GuC state only if gt is awake.
1353 */
1354 wakeref = in_reset ? NULL : intel_gt_pm_get_if_awake(gt);
1355 if (wakeref) {
1356 stats_saved = *stats;
1357 gt_stamp_saved = guc->timestamp.gt_stamp;
1358 /*
1359 * Update gt_clks, then gt timestamp to simplify the 'gt_stamp -
1360 * start_gt_clk' calculation below for active engines.
1361 */
1362 guc_update_engine_gt_clks(engine);
1363 guc_update_pm_timestamp(guc, now);
1364 intel_gt_pm_put_async(gt, handle: wakeref);
1365 if (i915_reset_count(error: gpu_error) != reset_count) {
1366 *stats = stats_saved;
1367 guc->timestamp.gt_stamp = gt_stamp_saved;
1368 }
1369 }
1370
1371 total = intel_gt_clock_interval_to_ns(gt, count: stats->total_gt_clks);
1372 if (stats->running) {
1373 u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;
1374
1375 total += intel_gt_clock_interval_to_ns(gt, count: clk);
1376 }
1377
1378 if (total > stats->total)
1379 stats->total = total;
1380
1381 spin_unlock_irqrestore(lock: &guc->timestamp.lock, flags);
1382
1383 return ns_to_ktime(ns: stats->total);
1384}
1385
1386static void guc_enable_busyness_worker(struct intel_guc *guc)
1387{
1388 mod_delayed_work(wq: system_highpri_wq, dwork: &guc->timestamp.work, delay: guc->timestamp.ping_delay);
1389}
1390
1391static void guc_cancel_busyness_worker(struct intel_guc *guc)
1392{
1393 /*
1394 * There are many different call stacks that can get here. Some of them
1395 * hold the reset mutex. The busyness worker also attempts to acquire the
1396 * reset mutex. Synchronously flushing a worker thread requires acquiring
1397 * the worker mutex. Lockdep sees this as a conflict. It thinks that the
1398 * flush can deadlock because it holds the worker mutex while waiting for
1399 * the reset mutex, but another thread is holding the reset mutex and might
1400 * attempt to use other worker functions.
1401 *
1402 * In practice, this scenario does not exist because the busyness worker
1403 * does not block waiting for the reset mutex. It does a try-lock on it and
1404 * immediately exits if the lock is already held. Unfortunately, the mutex
1405 * in question (I915_RESET_BACKOFF) is an i915 implementation which has lockdep
1406 * annotation but not to the extent of explaining the 'might lock' is also a
1407 * 'does not need to lock'. So one option would be to add more complex lockdep
1408 * annotations to ignore the issue (if at all possible). A simpler option is to
1409 * just not flush synchronously when a rest in progress. Given that the worker
1410 * will just early exit and re-schedule itself anyway, there is no advantage
1411 * to running it immediately.
1412 *
1413 * If a reset is not in progress, then the synchronous flush may be required.
1414 * As noted many call stacks lead here, some during suspend and driver unload
1415 * which do require a synchronous flush to make sure the worker is stopped
1416 * before memory is freed.
1417 *
1418 * Trying to pass a 'need_sync' or 'in_reset' flag all the way down through
1419 * every possible call stack is unfeasible. It would be too intrusive to many
1420 * areas that really don't care about the GuC backend. However, there is the
1421 * I915_RESET_BACKOFF flag and the gt->reset.mutex can be tested for is_locked.
1422 * So just use those. Note that testing both is required due to the hideously
1423 * complex nature of the i915 driver's reset code paths.
1424 *
1425 * And note that in the case of a reset occurring during driver unload
1426 * (wedged_on_fini), skipping the cancel in reset_prepare/reset_fini (when the
1427 * reset flag/mutex are set) is fine because there is another explicit cancel in
1428 * intel_guc_submission_fini (when the reset flag/mutex are not).
1429 */
1430 if (mutex_is_locked(lock: &guc_to_gt(guc)->reset.mutex) ||
1431 test_bit(I915_RESET_BACKOFF, &guc_to_gt(guc)->reset.flags))
1432 cancel_delayed_work(dwork: &guc->timestamp.work);
1433 else
1434 cancel_delayed_work_sync(dwork: &guc->timestamp.work);
1435}
1436
1437static void __reset_guc_busyness_stats(struct intel_guc *guc)
1438{
1439 struct intel_gt *gt = guc_to_gt(guc);
1440 struct intel_engine_cs *engine;
1441 enum intel_engine_id id;
1442 unsigned long flags;
1443 ktime_t unused;
1444
1445 spin_lock_irqsave(&guc->timestamp.lock, flags);
1446
1447 guc_update_pm_timestamp(guc, now: &unused);
1448 for_each_engine(engine, gt, id) {
1449 struct intel_engine_guc_stats *stats = &engine->stats.guc;
1450
1451 guc_update_engine_gt_clks(engine);
1452
1453 /*
1454 * If resetting a running context, accumulate the active
1455 * time as well since there will be no context switch.
1456 */
1457 if (stats->running) {
1458 u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;
1459
1460 stats->total_gt_clks += clk;
1461 }
1462 stats->prev_total = 0;
1463 stats->running = 0;
1464 }
1465
1466 spin_unlock_irqrestore(lock: &guc->timestamp.lock, flags);
1467}
1468
1469static void __update_guc_busyness_running_state(struct intel_guc *guc)
1470{
1471 struct intel_gt *gt = guc_to_gt(guc);
1472 struct intel_engine_cs *engine;
1473 enum intel_engine_id id;
1474 unsigned long flags;
1475
1476 spin_lock_irqsave(&guc->timestamp.lock, flags);
1477 for_each_engine(engine, gt, id)
1478 engine->stats.guc.running = false;
1479 spin_unlock_irqrestore(lock: &guc->timestamp.lock, flags);
1480}
1481
1482static void __update_guc_busyness_stats(struct intel_guc *guc)
1483{
1484 struct intel_gt *gt = guc_to_gt(guc);
1485 struct intel_engine_cs *engine;
1486 enum intel_engine_id id;
1487 unsigned long flags;
1488 ktime_t unused;
1489
1490 guc->timestamp.last_stat_jiffies = jiffies;
1491
1492 spin_lock_irqsave(&guc->timestamp.lock, flags);
1493
1494 guc_update_pm_timestamp(guc, now: &unused);
1495 for_each_engine(engine, gt, id)
1496 guc_update_engine_gt_clks(engine);
1497
1498 spin_unlock_irqrestore(lock: &guc->timestamp.lock, flags);
1499}
1500
1501static void __guc_context_update_stats(struct intel_context *ce)
1502{
1503 struct intel_guc *guc = ce_to_guc(ce);
1504 unsigned long flags;
1505
1506 spin_lock_irqsave(&guc->timestamp.lock, flags);
1507 lrc_update_runtime(ce);
1508 spin_unlock_irqrestore(lock: &guc->timestamp.lock, flags);
1509}
1510
1511static void guc_context_update_stats(struct intel_context *ce)
1512{
1513 if (!intel_context_pin_if_active(ce))
1514 return;
1515
1516 __guc_context_update_stats(ce);
1517 intel_context_unpin(ce);
1518}
1519
1520static void guc_timestamp_ping(struct work_struct *wrk)
1521{
1522 struct intel_guc *guc = container_of(wrk, typeof(*guc),
1523 timestamp.work.work);
1524 struct intel_uc *uc = container_of(guc, typeof(*uc), guc);
1525 struct intel_gt *gt = guc_to_gt(guc);
1526 struct intel_context *ce;
1527 intel_wakeref_t wakeref;
1528 unsigned long index;
1529 int srcu, ret;
1530
1531 /*
1532 * Ideally the busyness worker should take a gt pm wakeref because the
1533 * worker only needs to be active while gt is awake. However, the
1534 * gt_park path cancels the worker synchronously and this complicates
1535 * the flow if the worker is also running at the same time. The cancel
1536 * waits for the worker and when the worker releases the wakeref, that
1537 * would call gt_park and would lead to a deadlock.
1538 *
1539 * The resolution is to take the global pm wakeref if runtime pm is
1540 * already active. If not, we don't need to update the busyness stats as
1541 * the stats would already be updated when the gt was parked.
1542 *
1543 * Note:
1544 * - We do not requeue the worker if we cannot take a reference to runtime
1545 * pm since intel_guc_busyness_unpark would requeue the worker in the
1546 * resume path.
1547 *
1548 * - If the gt was parked longer than time taken for GT timestamp to roll
1549 * over, we ignore those rollovers since we don't care about tracking
1550 * the exact GT time. We only care about roll overs when the gt is
1551 * active and running workloads.
1552 *
1553 * - There is a window of time between gt_park and runtime suspend,
1554 * where the worker may run. This is acceptable since the worker will
1555 * not find any new data to update busyness.
1556 */
1557 wakeref = intel_runtime_pm_get_if_active(rpm: &gt->i915->runtime_pm);
1558 if (!wakeref)
1559 return;
1560
1561 /*
1562 * Synchronize with gt reset to make sure the worker does not
1563 * corrupt the engine/guc stats. NB: can't actually block waiting
1564 * for a reset to complete as the reset requires flushing out
1565 * this worker thread if started. So waiting would deadlock.
1566 */
1567 ret = intel_gt_reset_trylock(gt, srcu: &srcu);
1568 if (ret)
1569 goto err_trylock;
1570
1571 __update_guc_busyness_stats(guc);
1572
1573 /* adjust context stats for overflow */
1574 xa_for_each(&guc->context_lookup, index, ce)
1575 guc_context_update_stats(ce);
1576
1577 intel_gt_reset_unlock(gt, tag: srcu);
1578
1579 guc_enable_busyness_worker(guc);
1580
1581err_trylock:
1582 intel_runtime_pm_put(rpm: &gt->i915->runtime_pm, wref: wakeref);
1583}
1584
1585static int guc_action_enable_usage_stats(struct intel_guc *guc)
1586{
1587 struct intel_gt *gt = guc_to_gt(guc);
1588 struct intel_engine_cs *engine;
1589 enum intel_engine_id id;
1590 u32 offset = intel_guc_engine_usage_offset(guc);
1591 u32 action[] = {
1592 INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF,
1593 offset,
1594 0,
1595 };
1596
1597 for_each_engine(engine, gt, id)
1598 __set_engine_usage_record(engine, last_in: 0, id: 0xffffffff, total: 0);
1599
1600 return intel_guc_send(guc, action, ARRAY_SIZE(action));
1601}
1602
1603static int guc_init_engine_stats(struct intel_guc *guc)
1604{
1605 struct intel_gt *gt = guc_to_gt(guc);
1606 intel_wakeref_t wakeref;
1607 int ret;
1608
1609 with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
1610 ret = guc_action_enable_usage_stats(guc);
1611
1612 if (ret)
1613 guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));
1614 else
1615 guc_enable_busyness_worker(guc);
1616
1617 return ret;
1618}
1619
1620static void guc_fini_engine_stats(struct intel_guc *guc)
1621{
1622 guc_cancel_busyness_worker(guc);
1623}
1624
1625void intel_guc_busyness_park(struct intel_gt *gt)
1626{
1627 struct intel_guc *guc = gt_to_guc(gt);
1628
1629 if (!guc_submission_initialized(guc))
1630 return;
1631
1632 /* Assume no engines are running and set running state to false */
1633 __update_guc_busyness_running_state(guc);
1634
1635 /*
1636 * There is a race with suspend flow where the worker runs after suspend
1637 * and causes an unclaimed register access warning. Cancel the worker
1638 * synchronously here.
1639 */
1640 guc_cancel_busyness_worker(guc);
1641
1642 /*
1643 * Before parking, we should sample engine busyness stats if we need to.
1644 * We can skip it if we are less than half a ping from the last time we
1645 * sampled the busyness stats.
1646 */
1647 if (guc->timestamp.last_stat_jiffies &&
1648 !time_after(jiffies, guc->timestamp.last_stat_jiffies +
1649 (guc->timestamp.ping_delay / 2)))
1650 return;
1651
1652 __update_guc_busyness_stats(guc);
1653}
1654
1655void intel_guc_busyness_unpark(struct intel_gt *gt)
1656{
1657 struct intel_guc *guc = gt_to_guc(gt);
1658 unsigned long flags;
1659 ktime_t unused;
1660
1661 if (!guc_submission_initialized(guc))
1662 return;
1663
1664 spin_lock_irqsave(&guc->timestamp.lock, flags);
1665 guc_update_pm_timestamp(guc, now: &unused);
1666 spin_unlock_irqrestore(lock: &guc->timestamp.lock, flags);
1667 guc_enable_busyness_worker(guc);
1668}
1669
1670static inline bool
1671submission_disabled(struct intel_guc *guc)
1672{
1673 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1674
1675 return unlikely(!sched_engine ||
1676 !__tasklet_is_enabled(&sched_engine->tasklet) ||
1677 intel_gt_is_wedged(guc_to_gt(guc)));
1678}
1679
1680static void disable_submission(struct intel_guc *guc)
1681{
1682 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1683
1684 if (__tasklet_is_enabled(t: &sched_engine->tasklet)) {
1685 GEM_BUG_ON(!guc->ct.enabled);
1686 __tasklet_disable_sync_once(t: &sched_engine->tasklet);
1687 sched_engine->tasklet.callback = NULL;
1688 }
1689}
1690
1691static void enable_submission(struct intel_guc *guc)
1692{
1693 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1694 unsigned long flags;
1695
1696 spin_lock_irqsave(&guc->sched_engine->lock, flags);
1697 sched_engine->tasklet.callback = guc_submission_tasklet;
1698 wmb(); /* Make sure callback visible */
1699 if (!__tasklet_is_enabled(t: &sched_engine->tasklet) &&
1700 __tasklet_enable(t: &sched_engine->tasklet)) {
1701 GEM_BUG_ON(!guc->ct.enabled);
1702
1703 /* And kick in case we missed a new request submission. */
1704 tasklet_hi_schedule(t: &sched_engine->tasklet);
1705 }
1706 spin_unlock_irqrestore(lock: &guc->sched_engine->lock, flags);
1707}
1708
1709static void guc_flush_submissions(struct intel_guc *guc)
1710{
1711 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1712 unsigned long flags;
1713
1714 spin_lock_irqsave(&sched_engine->lock, flags);
1715 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
1716}
1717
1718void intel_guc_submission_flush_work(struct intel_guc *guc)
1719{
1720 flush_work(work: &guc->submission_state.destroyed_worker);
1721}
1722
1723static void guc_flush_destroyed_contexts(struct intel_guc *guc);
1724
1725void intel_guc_submission_reset_prepare(struct intel_guc *guc)
1726{
1727 if (unlikely(!guc_submission_initialized(guc))) {
1728 /* Reset called during driver load? GuC not yet initialised! */
1729 return;
1730 }
1731
1732 intel_gt_park_heartbeats(gt: guc_to_gt(guc));
1733 disable_submission(guc);
1734 guc->interrupts.disable(guc);
1735 __reset_guc_busyness_stats(guc);
1736
1737 /* Flush IRQ handler */
1738 spin_lock_irq(lock: guc_to_gt(guc)->irq_lock);
1739 spin_unlock_irq(lock: guc_to_gt(guc)->irq_lock);
1740
1741 /* Flush tasklet */
1742 tasklet_disable(t: &guc->ct.receive_tasklet);
1743 tasklet_enable(t: &guc->ct.receive_tasklet);
1744
1745 guc_flush_submissions(guc);
1746 guc_flush_destroyed_contexts(guc);
1747 flush_work(work: &guc->ct.requests.worker);
1748
1749 scrub_guc_desc_for_outstanding_g2h(guc);
1750}
1751
1752static struct intel_engine_cs *
1753guc_virtual_get_sibling(struct intel_engine_cs *ve, unsigned int sibling)
1754{
1755 struct intel_engine_cs *engine;
1756 intel_engine_mask_t tmp, mask = ve->mask;
1757 unsigned int num_siblings = 0;
1758
1759 for_each_engine_masked(engine, ve->gt, mask, tmp)
1760 if (num_siblings++ == sibling)
1761 return engine;
1762
1763 return NULL;
1764}
1765
1766static inline struct intel_engine_cs *
1767__context_to_physical_engine(struct intel_context *ce)
1768{
1769 struct intel_engine_cs *engine = ce->engine;
1770
1771 if (intel_engine_is_virtual(engine))
1772 engine = guc_virtual_get_sibling(ve: engine, sibling: 0);
1773
1774 return engine;
1775}
1776
1777static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub)
1778{
1779 struct intel_engine_cs *engine = __context_to_physical_engine(ce);
1780
1781 if (!intel_context_is_schedulable(ce))
1782 return;
1783
1784 GEM_BUG_ON(!intel_context_is_pinned(ce));
1785
1786 /*
1787 * We want a simple context + ring to execute the breadcrumb update.
1788 * We cannot rely on the context being intact across the GPU hang,
1789 * so clear it and rebuild just what we need for the breadcrumb.
1790 * All pending requests for this context will be zapped, and any
1791 * future request will be after userspace has had the opportunity
1792 * to recreate its own state.
1793 */
1794 if (scrub)
1795 lrc_init_regs(ce, engine, clear: true);
1796
1797 /* Rerun the request; its payload has been neutered (if guilty). */
1798 lrc_update_regs(ce, engine, head);
1799}
1800
1801static void guc_engine_reset_prepare(struct intel_engine_cs *engine)
1802{
1803 /*
1804 * Wa_22011802037: In addition to stopping the cs, we need
1805 * to wait for any pending mi force wakeups
1806 */
1807 if (intel_engine_reset_needs_wa_22011802037(gt: engine->gt)) {
1808 intel_engine_stop_cs(engine);
1809 intel_engine_wait_for_pending_mi_fw(engine);
1810 }
1811}
1812
1813static void guc_reset_nop(struct intel_engine_cs *engine)
1814{
1815}
1816
1817static void guc_rewind_nop(struct intel_engine_cs *engine, bool stalled)
1818{
1819}
1820
1821static void
1822__unwind_incomplete_requests(struct intel_context *ce)
1823{
1824 struct i915_request *rq, *rn;
1825 struct list_head *pl;
1826 int prio = I915_PRIORITY_INVALID;
1827 struct i915_sched_engine * const sched_engine =
1828 ce->engine->sched_engine;
1829 unsigned long flags;
1830
1831 spin_lock_irqsave(&sched_engine->lock, flags);
1832 spin_lock(lock: &ce->guc_state.lock);
1833 list_for_each_entry_safe_reverse(rq, rn,
1834 &ce->guc_state.requests,
1835 sched.link) {
1836 if (i915_request_completed(rq))
1837 continue;
1838
1839 list_del_init(entry: &rq->sched.link);
1840 __i915_request_unsubmit(request: rq);
1841
1842 /* Push the request back into the queue for later resubmission. */
1843 GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
1844 if (rq_prio(rq) != prio) {
1845 prio = rq_prio(rq);
1846 pl = i915_sched_lookup_priolist(sched_engine, prio);
1847 }
1848 GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine));
1849
1850 list_add(new: &rq->sched.link, head: pl);
1851 set_bit(nr: I915_FENCE_FLAG_PQUEUE, addr: &rq->fence.flags);
1852 }
1853 spin_unlock(lock: &ce->guc_state.lock);
1854 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
1855}
1856
1857static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t stalled)
1858{
1859 bool guilty;
1860 struct i915_request *rq;
1861 unsigned long flags;
1862 u32 head;
1863 int i, number_children = ce->parallel.number_children;
1864 struct intel_context *parent = ce;
1865
1866 GEM_BUG_ON(intel_context_is_child(ce));
1867
1868 intel_context_get(ce);
1869
1870 /*
1871 * GuC will implicitly mark the context as non-schedulable when it sends
1872 * the reset notification. Make sure our state reflects this change. The
1873 * context will be marked enabled on resubmission.
1874 */
1875 spin_lock_irqsave(&ce->guc_state.lock, flags);
1876 clr_context_enabled(ce);
1877 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
1878
1879 /*
1880 * For each context in the relationship find the hanging request
1881 * resetting each context / request as needed
1882 */
1883 for (i = 0; i < number_children + 1; ++i) {
1884 if (!intel_context_is_pinned(ce))
1885 goto next_context;
1886
1887 guilty = false;
1888 rq = intel_context_get_active_request(ce);
1889 if (!rq) {
1890 head = ce->ring->tail;
1891 goto out_replay;
1892 }
1893
1894 if (i915_request_started(rq))
1895 guilty = stalled & ce->engine->mask;
1896
1897 GEM_BUG_ON(i915_active_is_idle(&ce->active));
1898 head = intel_ring_wrap(ring: ce->ring, pos: rq->head);
1899
1900 __i915_request_reset(rq, guilty);
1901 i915_request_put(rq);
1902out_replay:
1903 guc_reset_state(ce, head, scrub: guilty);
1904next_context:
1905 if (i != number_children)
1906 ce = list_next_entry(ce, parallel.child_link);
1907 }
1908
1909 __unwind_incomplete_requests(ce: parent);
1910 intel_context_put(ce: parent);
1911}
1912
1913void wake_up_all_tlb_invalidate(struct intel_guc *guc)
1914{
1915 struct intel_guc_tlb_wait *wait;
1916 unsigned long i;
1917
1918 if (!intel_guc_tlb_invalidation_is_available(guc))
1919 return;
1920
1921 xa_lock_irq(&guc->tlb_lookup);
1922 xa_for_each(&guc->tlb_lookup, i, wait)
1923 wake_up(&wait->wq);
1924 xa_unlock_irq(&guc->tlb_lookup);
1925}
1926
1927void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
1928{
1929 struct intel_context *ce;
1930 unsigned long index;
1931 unsigned long flags;
1932
1933 if (unlikely(!guc_submission_initialized(guc))) {
1934 /* Reset called during driver load? GuC not yet initialised! */
1935 return;
1936 }
1937
1938 xa_lock_irqsave(&guc->context_lookup, flags);
1939 xa_for_each(&guc->context_lookup, index, ce) {
1940 if (!kref_get_unless_zero(kref: &ce->ref))
1941 continue;
1942
1943 xa_unlock(&guc->context_lookup);
1944
1945 if (intel_context_is_pinned(ce) &&
1946 !intel_context_is_child(ce))
1947 __guc_reset_context(ce, stalled);
1948
1949 intel_context_put(ce);
1950
1951 xa_lock(&guc->context_lookup);
1952 }
1953 xa_unlock_irqrestore(&guc->context_lookup, flags);
1954
1955 /* GuC is blown away, drop all references to contexts */
1956 xa_destroy(&guc->context_lookup);
1957}
1958
1959static void guc_cancel_context_requests(struct intel_context *ce)
1960{
1961 struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine;
1962 struct i915_request *rq;
1963 unsigned long flags;
1964
1965 /* Mark all executing requests as skipped. */
1966 spin_lock_irqsave(&sched_engine->lock, flags);
1967 spin_lock(lock: &ce->guc_state.lock);
1968 list_for_each_entry(rq, &ce->guc_state.requests, sched.link)
1969 i915_request_put(rq: i915_request_mark_eio(rq));
1970 spin_unlock(lock: &ce->guc_state.lock);
1971 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
1972}
1973
1974static void
1975guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine)
1976{
1977 struct i915_request *rq, *rn;
1978 struct rb_node *rb;
1979 unsigned long flags;
1980
1981 /* Can be called during boot if GuC fails to load */
1982 if (!sched_engine)
1983 return;
1984
1985 /*
1986 * Before we call engine->cancel_requests(), we should have exclusive
1987 * access to the submission state. This is arranged for us by the
1988 * caller disabling the interrupt generation, the tasklet and other
1989 * threads that may then access the same state, giving us a free hand
1990 * to reset state. However, we still need to let lockdep be aware that
1991 * we know this state may be accessed in hardirq context, so we
1992 * disable the irq around this manipulation and we want to keep
1993 * the spinlock focused on its duties and not accidentally conflate
1994 * coverage to the submission's irq state. (Similarly, although we
1995 * shouldn't need to disable irq around the manipulation of the
1996 * submission's irq state, we also wish to remind ourselves that
1997 * it is irq state.)
1998 */
1999 spin_lock_irqsave(&sched_engine->lock, flags);
2000
2001 /* Flush the queued requests to the timeline list (for retiring). */
2002 while ((rb = rb_first_cached(&sched_engine->queue))) {
2003 struct i915_priolist *p = to_priolist(rb);
2004
2005 priolist_for_each_request_consume(rq, rn, p) {
2006 list_del_init(entry: &rq->sched.link);
2007
2008 __i915_request_submit(request: rq);
2009
2010 i915_request_put(rq: i915_request_mark_eio(rq));
2011 }
2012
2013 rb_erase_cached(node: &p->node, root: &sched_engine->queue);
2014 i915_priolist_free(p);
2015 }
2016
2017 /* Remaining _unready_ requests will be nop'ed when submitted */
2018
2019 sched_engine->queue_priority_hint = INT_MIN;
2020 sched_engine->queue = RB_ROOT_CACHED;
2021
2022 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
2023}
2024
2025void intel_guc_submission_cancel_requests(struct intel_guc *guc)
2026{
2027 struct intel_context *ce;
2028 unsigned long index;
2029 unsigned long flags;
2030
2031 xa_lock_irqsave(&guc->context_lookup, flags);
2032 xa_for_each(&guc->context_lookup, index, ce) {
2033 if (!kref_get_unless_zero(kref: &ce->ref))
2034 continue;
2035
2036 xa_unlock(&guc->context_lookup);
2037
2038 if (intel_context_is_pinned(ce) &&
2039 !intel_context_is_child(ce))
2040 guc_cancel_context_requests(ce);
2041
2042 intel_context_put(ce);
2043
2044 xa_lock(&guc->context_lookup);
2045 }
2046 xa_unlock_irqrestore(&guc->context_lookup, flags);
2047
2048 guc_cancel_sched_engine_requests(sched_engine: guc->sched_engine);
2049
2050 /* GuC is blown away, drop all references to contexts */
2051 xa_destroy(&guc->context_lookup);
2052
2053 /*
2054 * Wedged GT won't respond to any TLB invalidation request. Simply
2055 * release all the blocked waiters.
2056 */
2057 wake_up_all_tlb_invalidate(guc);
2058}
2059
2060void intel_guc_submission_reset_finish(struct intel_guc *guc)
2061{
2062 int outstanding;
2063
2064 /* Reset called during driver load or during wedge? */
2065 if (unlikely(!guc_submission_initialized(guc) ||
2066 !intel_guc_is_fw_running(guc) ||
2067 intel_gt_is_wedged(guc_to_gt(guc)))) {
2068 return;
2069 }
2070
2071 /*
2072 * Technically possible for either of these values to be non-zero here,
2073 * but very unlikely + harmless. Regardless let's add an error so we can
2074 * see in CI if this happens frequently / a precursor to taking down the
2075 * machine.
2076 */
2077 outstanding = atomic_read(v: &guc->outstanding_submission_g2h);
2078 if (outstanding)
2079 guc_err(guc, "Unexpected outstanding GuC to Host response(s) in reset finish: %d\n",
2080 outstanding);
2081 atomic_set(v: &guc->outstanding_submission_g2h, i: 0);
2082
2083 intel_guc_global_policies_update(guc);
2084 enable_submission(guc);
2085 intel_gt_unpark_heartbeats(gt: guc_to_gt(guc));
2086
2087 /*
2088 * The full GT reset will have cleared the TLB caches and flushed the
2089 * G2H message queue; we can release all the blocked waiters.
2090 */
2091 wake_up_all_tlb_invalidate(guc);
2092}
2093
2094static void destroyed_worker_func(struct work_struct *w);
2095static void reset_fail_worker_func(struct work_struct *w);
2096
2097bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc)
2098{
2099 return HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915) &&
2100 intel_guc_is_ready(guc);
2101}
2102
2103static int init_tlb_lookup(struct intel_guc *guc)
2104{
2105 struct intel_guc_tlb_wait *wait;
2106 int err;
2107
2108 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
2109 return 0;
2110
2111 xa_init_flags(xa: &guc->tlb_lookup, XA_FLAGS_ALLOC);
2112
2113 wait = kzalloc(sizeof(*wait), GFP_KERNEL);
2114 if (!wait)
2115 return -ENOMEM;
2116
2117 init_waitqueue_head(&wait->wq);
2118
2119 /* Preallocate a shared id for use under memory pressure. */
2120 err = xa_alloc_cyclic_irq(xa: &guc->tlb_lookup, id: &guc->serial_slot, entry: wait,
2121 xa_limit_32b, next: &guc->next_seqno, GFP_KERNEL);
2122 if (err < 0) {
2123 kfree(objp: wait);
2124 return err;
2125 }
2126
2127 return 0;
2128}
2129
2130static void fini_tlb_lookup(struct intel_guc *guc)
2131{
2132 struct intel_guc_tlb_wait *wait;
2133
2134 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
2135 return;
2136
2137 wait = xa_load(&guc->tlb_lookup, index: guc->serial_slot);
2138 if (wait && wait->busy)
2139 guc_err(guc, "Unexpected busy item in tlb_lookup on fini\n");
2140 kfree(objp: wait);
2141
2142 xa_destroy(&guc->tlb_lookup);
2143}
2144
2145/*
2146 * Set up the memory resources to be shared with the GuC (via the GGTT)
2147 * at firmware loading time.
2148 */
2149int intel_guc_submission_init(struct intel_guc *guc)
2150{
2151 struct intel_gt *gt = guc_to_gt(guc);
2152 int ret;
2153
2154 if (guc->submission_initialized)
2155 return 0;
2156
2157 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 0, 0)) {
2158 ret = guc_lrc_desc_pool_create_v69(guc);
2159 if (ret)
2160 return ret;
2161 }
2162
2163 ret = init_tlb_lookup(guc);
2164 if (ret)
2165 goto destroy_pool;
2166
2167 guc->submission_state.guc_ids_bitmap =
2168 bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL);
2169 if (!guc->submission_state.guc_ids_bitmap) {
2170 ret = -ENOMEM;
2171 goto destroy_tlb;
2172 }
2173
2174 guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ;
2175 guc->timestamp.shift = gpm_timestamp_shift(gt);
2176 guc->submission_initialized = true;
2177
2178 return 0;
2179
2180destroy_tlb:
2181 fini_tlb_lookup(guc);
2182destroy_pool:
2183 guc_lrc_desc_pool_destroy_v69(guc);
2184 return ret;
2185}
2186
2187void intel_guc_submission_fini(struct intel_guc *guc)
2188{
2189 if (!guc->submission_initialized)
2190 return;
2191
2192 guc_fini_engine_stats(guc);
2193 guc_flush_destroyed_contexts(guc);
2194 guc_lrc_desc_pool_destroy_v69(guc);
2195 i915_sched_engine_put(sched_engine: guc->sched_engine);
2196 bitmap_free(bitmap: guc->submission_state.guc_ids_bitmap);
2197 fini_tlb_lookup(guc);
2198 guc->submission_initialized = false;
2199}
2200
2201static inline void queue_request(struct i915_sched_engine *sched_engine,
2202 struct i915_request *rq,
2203 int prio)
2204{
2205 GEM_BUG_ON(!list_empty(&rq->sched.link));
2206 list_add_tail(new: &rq->sched.link,
2207 head: i915_sched_lookup_priolist(sched_engine, prio));
2208 set_bit(nr: I915_FENCE_FLAG_PQUEUE, addr: &rq->fence.flags);
2209 tasklet_hi_schedule(t: &sched_engine->tasklet);
2210}
2211
2212static int guc_bypass_tasklet_submit(struct intel_guc *guc,
2213 struct i915_request *rq)
2214{
2215 int ret = 0;
2216
2217 __i915_request_submit(request: rq);
2218
2219 trace_i915_request_in(rq, port: 0);
2220
2221 if (is_multi_lrc_rq(rq)) {
2222 if (multi_lrc_submit(rq)) {
2223 ret = guc_wq_item_append(guc, rq);
2224 if (!ret)
2225 ret = guc_add_request(guc, rq);
2226 }
2227 } else {
2228 guc_set_lrc_tail(rq);
2229 ret = guc_add_request(guc, rq);
2230 }
2231
2232 if (unlikely(ret == -EPIPE))
2233 disable_submission(guc);
2234
2235 return ret;
2236}
2237
2238static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)
2239{
2240 struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
2241 struct intel_context *ce = request_to_scheduling_context(rq);
2242
2243 return submission_disabled(guc) || guc->stalled_request ||
2244 !i915_sched_engine_is_empty(sched_engine) ||
2245 !ctx_id_mapped(guc, id: ce->guc_id.id);
2246}
2247
2248static void guc_submit_request(struct i915_request *rq)
2249{
2250 struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
2251 struct intel_guc *guc = gt_to_guc(gt: rq->engine->gt);
2252 unsigned long flags;
2253
2254 /* Will be called from irq-context when using foreign fences. */
2255 spin_lock_irqsave(&sched_engine->lock, flags);
2256
2257 if (need_tasklet(guc, rq))
2258 queue_request(sched_engine, rq, prio: rq_prio(rq));
2259 else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY)
2260 tasklet_hi_schedule(t: &sched_engine->tasklet);
2261
2262 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
2263}
2264
2265static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
2266{
2267 int ret;
2268
2269 GEM_BUG_ON(intel_context_is_child(ce));
2270
2271 if (intel_context_is_parent(ce))
2272 ret = bitmap_find_free_region(bitmap: guc->submission_state.guc_ids_bitmap,
2273 NUMBER_MULTI_LRC_GUC_ID(guc),
2274 order_base_2(ce->parallel.number_children
2275 + 1));
2276 else
2277 ret = ida_alloc_range(&guc->submission_state.guc_ids,
2278 NUMBER_MULTI_LRC_GUC_ID(guc),
2279 max: guc->submission_state.num_guc_ids - 1,
2280 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
2281 if (unlikely(ret < 0))
2282 return ret;
2283
2284 if (!intel_context_is_parent(ce))
2285 ++guc->submission_state.guc_ids_in_use;
2286
2287 ce->guc_id.id = ret;
2288 return 0;
2289}
2290
2291static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2292{
2293 GEM_BUG_ON(intel_context_is_child(ce));
2294
2295 if (!context_guc_id_invalid(ce)) {
2296 if (intel_context_is_parent(ce)) {
2297 bitmap_release_region(bitmap: guc->submission_state.guc_ids_bitmap,
2298 pos: ce->guc_id.id,
2299 order_base_2(ce->parallel.number_children
2300 + 1));
2301 } else {
2302 --guc->submission_state.guc_ids_in_use;
2303 ida_free(&guc->submission_state.guc_ids,
2304 id: ce->guc_id.id);
2305 }
2306 clr_ctx_id_mapping(guc, id: ce->guc_id.id);
2307 set_context_guc_id_invalid(ce);
2308 }
2309 if (!list_empty(head: &ce->guc_id.link))
2310 list_del_init(entry: &ce->guc_id.link);
2311}
2312
2313static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2314{
2315 unsigned long flags;
2316
2317 spin_lock_irqsave(&guc->submission_state.lock, flags);
2318 __release_guc_id(guc, ce);
2319 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
2320}
2321
2322static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce)
2323{
2324 struct intel_context *cn;
2325
2326 lockdep_assert_held(&guc->submission_state.lock);
2327 GEM_BUG_ON(intel_context_is_child(ce));
2328 GEM_BUG_ON(intel_context_is_parent(ce));
2329
2330 if (!list_empty(head: &guc->submission_state.guc_id_list)) {
2331 cn = list_first_entry(&guc->submission_state.guc_id_list,
2332 struct intel_context,
2333 guc_id.link);
2334
2335 GEM_BUG_ON(atomic_read(&cn->guc_id.ref));
2336 GEM_BUG_ON(context_guc_id_invalid(cn));
2337 GEM_BUG_ON(intel_context_is_child(cn));
2338 GEM_BUG_ON(intel_context_is_parent(cn));
2339
2340 list_del_init(entry: &cn->guc_id.link);
2341 ce->guc_id.id = cn->guc_id.id;
2342
2343 spin_lock(lock: &cn->guc_state.lock);
2344 clr_context_registered(ce: cn);
2345 spin_unlock(lock: &cn->guc_state.lock);
2346
2347 set_context_guc_id_invalid(cn);
2348
2349#ifdef CONFIG_DRM_I915_SELFTEST
2350 guc->number_guc_id_stolen++;
2351#endif
2352
2353 return 0;
2354 } else {
2355 return -EAGAIN;
2356 }
2357}
2358
2359static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce)
2360{
2361 int ret;
2362
2363 lockdep_assert_held(&guc->submission_state.lock);
2364 GEM_BUG_ON(intel_context_is_child(ce));
2365
2366 ret = new_guc_id(guc, ce);
2367 if (unlikely(ret < 0)) {
2368 if (intel_context_is_parent(ce))
2369 return -ENOSPC;
2370
2371 ret = steal_guc_id(guc, ce);
2372 if (ret < 0)
2373 return ret;
2374 }
2375
2376 if (intel_context_is_parent(ce)) {
2377 struct intel_context *child;
2378 int i = 1;
2379
2380 for_each_child(ce, child)
2381 child->guc_id.id = ce->guc_id.id + i++;
2382 }
2383
2384 return 0;
2385}
2386
2387#define PIN_GUC_ID_TRIES 4
2388static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2389{
2390 int ret = 0;
2391 unsigned long flags, tries = PIN_GUC_ID_TRIES;
2392
2393 GEM_BUG_ON(atomic_read(&ce->guc_id.ref));
2394
2395try_again:
2396 spin_lock_irqsave(&guc->submission_state.lock, flags);
2397
2398 might_lock(&ce->guc_state.lock);
2399
2400 if (context_guc_id_invalid(ce)) {
2401 ret = assign_guc_id(guc, ce);
2402 if (ret)
2403 goto out_unlock;
2404 ret = 1; /* Indidcates newly assigned guc_id */
2405 }
2406 if (!list_empty(head: &ce->guc_id.link))
2407 list_del_init(entry: &ce->guc_id.link);
2408 atomic_inc(v: &ce->guc_id.ref);
2409
2410out_unlock:
2411 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
2412
2413 /*
2414 * -EAGAIN indicates no guc_id are available, let's retire any
2415 * outstanding requests to see if that frees up a guc_id. If the first
2416 * retire didn't help, insert a sleep with the timeslice duration before
2417 * attempting to retire more requests. Double the sleep period each
2418 * subsequent pass before finally giving up. The sleep period has max of
2419 * 100ms and minimum of 1ms.
2420 */
2421 if (ret == -EAGAIN && --tries) {
2422 if (PIN_GUC_ID_TRIES - tries > 1) {
2423 unsigned int timeslice_shifted =
2424 ce->engine->props.timeslice_duration_ms <<
2425 (PIN_GUC_ID_TRIES - tries - 2);
2426 unsigned int max = min_t(unsigned int, 100,
2427 timeslice_shifted);
2428
2429 msleep(max_t(unsigned int, max, 1));
2430 }
2431 intel_gt_retire_requests(gt: guc_to_gt(guc));
2432 goto try_again;
2433 }
2434
2435 return ret;
2436}
2437
2438static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2439{
2440 unsigned long flags;
2441
2442 GEM_BUG_ON(atomic_read(&ce->guc_id.ref) < 0);
2443 GEM_BUG_ON(intel_context_is_child(ce));
2444
2445 if (unlikely(context_guc_id_invalid(ce) ||
2446 intel_context_is_parent(ce)))
2447 return;
2448
2449 spin_lock_irqsave(&guc->submission_state.lock, flags);
2450 if (!context_guc_id_invalid(ce) && list_empty(head: &ce->guc_id.link) &&
2451 !atomic_read(v: &ce->guc_id.ref))
2452 list_add_tail(new: &ce->guc_id.link,
2453 head: &guc->submission_state.guc_id_list);
2454 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
2455}
2456
2457static int __guc_action_register_multi_lrc_v69(struct intel_guc *guc,
2458 struct intel_context *ce,
2459 u32 guc_id,
2460 u32 offset,
2461 bool loop)
2462{
2463 struct intel_context *child;
2464 u32 action[4 + MAX_ENGINE_INSTANCE];
2465 int len = 0;
2466
2467 GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);
2468
2469 action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
2470 action[len++] = guc_id;
2471 action[len++] = ce->parallel.number_children + 1;
2472 action[len++] = offset;
2473 for_each_child(ce, child) {
2474 offset += sizeof(struct guc_lrc_desc_v69);
2475 action[len++] = offset;
2476 }
2477
2478 return guc_submission_send_busy_loop(guc, action, len, g2h_len_dw: 0, loop);
2479}
2480
2481static int __guc_action_register_multi_lrc_v70(struct intel_guc *guc,
2482 struct intel_context *ce,
2483 struct guc_ctxt_registration_info *info,
2484 bool loop)
2485{
2486 struct intel_context *child;
2487 u32 action[13 + (MAX_ENGINE_INSTANCE * 2)];
2488 int len = 0;
2489 u32 next_id;
2490
2491 GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);
2492
2493 action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
2494 action[len++] = info->flags;
2495 action[len++] = info->context_idx;
2496 action[len++] = info->engine_class;
2497 action[len++] = info->engine_submit_mask;
2498 action[len++] = info->wq_desc_lo;
2499 action[len++] = info->wq_desc_hi;
2500 action[len++] = info->wq_base_lo;
2501 action[len++] = info->wq_base_hi;
2502 action[len++] = info->wq_size;
2503 action[len++] = ce->parallel.number_children + 1;
2504 action[len++] = info->hwlrca_lo;
2505 action[len++] = info->hwlrca_hi;
2506
2507 next_id = info->context_idx + 1;
2508 for_each_child(ce, child) {
2509 GEM_BUG_ON(next_id++ != child->guc_id.id);
2510
2511 /*
2512 * NB: GuC interface supports 64 bit LRCA even though i915/HW
2513 * only supports 32 bit currently.
2514 */
2515 action[len++] = lower_32_bits(child->lrc.lrca);
2516 action[len++] = upper_32_bits(child->lrc.lrca);
2517 }
2518
2519 GEM_BUG_ON(len > ARRAY_SIZE(action));
2520
2521 return guc_submission_send_busy_loop(guc, action, len, g2h_len_dw: 0, loop);
2522}
2523
2524static int __guc_action_register_context_v69(struct intel_guc *guc,
2525 u32 guc_id,
2526 u32 offset,
2527 bool loop)
2528{
2529 u32 action[] = {
2530 INTEL_GUC_ACTION_REGISTER_CONTEXT,
2531 guc_id,
2532 offset,
2533 };
2534
2535 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2536 g2h_len_dw: 0, loop);
2537}
2538
2539static int __guc_action_register_context_v70(struct intel_guc *guc,
2540 struct guc_ctxt_registration_info *info,
2541 bool loop)
2542{
2543 u32 action[] = {
2544 INTEL_GUC_ACTION_REGISTER_CONTEXT,
2545 info->flags,
2546 info->context_idx,
2547 info->engine_class,
2548 info->engine_submit_mask,
2549 info->wq_desc_lo,
2550 info->wq_desc_hi,
2551 info->wq_base_lo,
2552 info->wq_base_hi,
2553 info->wq_size,
2554 info->hwlrca_lo,
2555 info->hwlrca_hi,
2556 };
2557
2558 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2559 g2h_len_dw: 0, loop);
2560}
2561
2562static void prepare_context_registration_info_v69(struct intel_context *ce);
2563static void prepare_context_registration_info_v70(struct intel_context *ce,
2564 struct guc_ctxt_registration_info *info);
2565
2566static int
2567register_context_v69(struct intel_guc *guc, struct intel_context *ce, bool loop)
2568{
2569 u32 offset = intel_guc_ggtt_offset(guc, vma: guc->lrc_desc_pool_v69) +
2570 ce->guc_id.id * sizeof(struct guc_lrc_desc_v69);
2571
2572 prepare_context_registration_info_v69(ce);
2573
2574 if (intel_context_is_parent(ce))
2575 return __guc_action_register_multi_lrc_v69(guc, ce, guc_id: ce->guc_id.id,
2576 offset, loop);
2577 else
2578 return __guc_action_register_context_v69(guc, guc_id: ce->guc_id.id,
2579 offset, loop);
2580}
2581
2582static int
2583register_context_v70(struct intel_guc *guc, struct intel_context *ce, bool loop)
2584{
2585 struct guc_ctxt_registration_info info;
2586
2587 prepare_context_registration_info_v70(ce, info: &info);
2588
2589 if (intel_context_is_parent(ce))
2590 return __guc_action_register_multi_lrc_v70(guc, ce, info: &info, loop);
2591 else
2592 return __guc_action_register_context_v70(guc, info: &info, loop);
2593}
2594
2595static int register_context(struct intel_context *ce, bool loop)
2596{
2597 struct intel_guc *guc = ce_to_guc(ce);
2598 int ret;
2599
2600 GEM_BUG_ON(intel_context_is_child(ce));
2601 trace_intel_context_register(ce);
2602
2603 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
2604 ret = register_context_v70(guc, ce, loop);
2605 else
2606 ret = register_context_v69(guc, ce, loop);
2607
2608 if (likely(!ret)) {
2609 unsigned long flags;
2610
2611 spin_lock_irqsave(&ce->guc_state.lock, flags);
2612 set_context_registered(ce);
2613 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
2614
2615 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
2616 guc_context_policy_init_v70(ce, loop);
2617 }
2618
2619 return ret;
2620}
2621
2622static int __guc_action_deregister_context(struct intel_guc *guc,
2623 u32 guc_id)
2624{
2625 u32 action[] = {
2626 INTEL_GUC_ACTION_DEREGISTER_CONTEXT,
2627 guc_id,
2628 };
2629
2630 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2631 G2H_LEN_DW_DEREGISTER_CONTEXT,
2632 loop: true);
2633}
2634
2635static int deregister_context(struct intel_context *ce, u32 guc_id)
2636{
2637 struct intel_guc *guc = ce_to_guc(ce);
2638
2639 GEM_BUG_ON(intel_context_is_child(ce));
2640 trace_intel_context_deregister(ce);
2641
2642 return __guc_action_deregister_context(guc, guc_id);
2643}
2644
2645static inline void clear_children_join_go_memory(struct intel_context *ce)
2646{
2647 struct parent_scratch *ps = __get_parent_scratch(ce);
2648 int i;
2649
2650 ps->go.semaphore = 0;
2651 for (i = 0; i < ce->parallel.number_children + 1; ++i)
2652 ps->join[i].semaphore = 0;
2653}
2654
2655static inline u32 get_children_go_value(struct intel_context *ce)
2656{
2657 return __get_parent_scratch(ce)->go.semaphore;
2658}
2659
2660static inline u32 get_children_join_value(struct intel_context *ce,
2661 u8 child_index)
2662{
2663 return __get_parent_scratch(ce)->join[child_index].semaphore;
2664}
2665
2666struct context_policy {
2667 u32 count;
2668 struct guc_update_context_policy h2g;
2669};
2670
2671static u32 __guc_context_policy_action_size(struct context_policy *policy)
2672{
2673 size_t bytes = sizeof(policy->h2g.header) +
2674 (sizeof(policy->h2g.klv[0]) * policy->count);
2675
2676 return bytes / sizeof(u32);
2677}
2678
2679static void __guc_context_policy_start_klv(struct context_policy *policy, u16 guc_id)
2680{
2681 policy->h2g.header.action = INTEL_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
2682 policy->h2g.header.ctx_id = guc_id;
2683 policy->count = 0;
2684}
2685
2686#define MAKE_CONTEXT_POLICY_ADD(func, id) \
2687static void __guc_context_policy_add_##func(struct context_policy *policy, u32 data) \
2688{ \
2689 GEM_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \
2690 policy->h2g.klv[policy->count].kl = \
2691 FIELD_PREP(GUC_KLV_0_KEY, GUC_CONTEXT_POLICIES_KLV_ID_##id) | \
2692 FIELD_PREP(GUC_KLV_0_LEN, 1); \
2693 policy->h2g.klv[policy->count].value = data; \
2694 policy->count++; \
2695}
2696
2697MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
2698MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
2699MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY)
2700MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY)
2701MAKE_CONTEXT_POLICY_ADD(slpc_ctx_freq_req, SLPM_GT_FREQUENCY)
2702
2703#undef MAKE_CONTEXT_POLICY_ADD
2704
2705static int __guc_context_set_context_policies(struct intel_guc *guc,
2706 struct context_policy *policy,
2707 bool loop)
2708{
2709 return guc_submission_send_busy_loop(guc, action: (u32 *)&policy->h2g,
2710 len: __guc_context_policy_action_size(policy),
2711 g2h_len_dw: 0, loop);
2712}
2713
2714static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
2715{
2716 struct intel_engine_cs *engine = ce->engine;
2717 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
2718 struct context_policy policy;
2719 u32 execution_quantum;
2720 u32 preemption_timeout;
2721 u32 slpc_ctx_freq_req = 0;
2722 unsigned long flags;
2723 int ret;
2724
2725 /* NB: For both of these, zero means disabled. */
2726 GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,
2727 execution_quantum));
2728 GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,
2729 preemption_timeout));
2730 execution_quantum = engine->props.timeslice_duration_ms * 1000;
2731 preemption_timeout = engine->props.preempt_timeout_ms * 1000;
2732
2733 if (ce->flags & BIT(CONTEXT_LOW_LATENCY))
2734 slpc_ctx_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
2735
2736 __guc_context_policy_start_klv(policy: &policy, guc_id: ce->guc_id.id);
2737
2738 __guc_context_policy_add_priority(policy: &policy, data: ce->guc_state.prio);
2739 __guc_context_policy_add_execution_quantum(policy: &policy, data: execution_quantum);
2740 __guc_context_policy_add_preemption_timeout(policy: &policy, data: preemption_timeout);
2741 __guc_context_policy_add_slpc_ctx_freq_req(policy: &policy, data: slpc_ctx_freq_req);
2742
2743 if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
2744 __guc_context_policy_add_preempt_to_idle(policy: &policy, data: 1);
2745
2746 ret = __guc_context_set_context_policies(guc, policy: &policy, loop);
2747
2748 spin_lock_irqsave(&ce->guc_state.lock, flags);
2749 if (ret != 0)
2750 set_context_policy_required(ce);
2751 else
2752 clr_context_policy_required(ce);
2753 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
2754
2755 return ret;
2756}
2757
2758static void guc_context_policy_init_v69(struct intel_engine_cs *engine,
2759 struct guc_lrc_desc_v69 *desc)
2760{
2761 desc->policy_flags = 0;
2762
2763 if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
2764 desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE_V69;
2765
2766 /* NB: For both of these, zero means disabled. */
2767 GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,
2768 desc->execution_quantum));
2769 GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,
2770 desc->preemption_timeout));
2771 desc->execution_quantum = engine->props.timeslice_duration_ms * 1000;
2772 desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000;
2773}
2774
2775static u32 map_guc_prio_to_lrc_desc_prio(u8 prio)
2776{
2777 /*
2778 * this matches the mapping we do in map_i915_prio_to_guc_prio()
2779 * (e.g. prio < I915_PRIORITY_NORMAL maps to GUC_CLIENT_PRIORITY_NORMAL)
2780 */
2781 switch (prio) {
2782 default:
2783 MISSING_CASE(prio);
2784 fallthrough;
2785 case GUC_CLIENT_PRIORITY_KMD_NORMAL:
2786 return GEN12_CTX_PRIORITY_NORMAL;
2787 case GUC_CLIENT_PRIORITY_NORMAL:
2788 return GEN12_CTX_PRIORITY_LOW;
2789 case GUC_CLIENT_PRIORITY_HIGH:
2790 case GUC_CLIENT_PRIORITY_KMD_HIGH:
2791 return GEN12_CTX_PRIORITY_HIGH;
2792 }
2793}
2794
2795static void prepare_context_registration_info_v69(struct intel_context *ce)
2796{
2797 struct intel_engine_cs *engine = ce->engine;
2798 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
2799 u32 ctx_id = ce->guc_id.id;
2800 struct guc_lrc_desc_v69 *desc;
2801 struct intel_context *child;
2802
2803 GEM_BUG_ON(!engine->mask);
2804
2805 /*
2806 * Ensure LRC + CT vmas are is same region as write barrier is done
2807 * based on CT vma region.
2808 */
2809 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
2810 i915_gem_object_is_lmem(ce->ring->vma->obj));
2811
2812 desc = __get_lrc_desc_v69(guc, index: ctx_id);
2813 GEM_BUG_ON(!desc);
2814 desc->engine_class = engine_class_to_guc_class(class: engine->class);
2815 desc->engine_submit_mask = engine->logical_mask;
2816 desc->hw_context_desc = ce->lrc.lrca;
2817 desc->priority = ce->guc_state.prio;
2818 desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;
2819 guc_context_policy_init_v69(engine, desc);
2820
2821 /*
2822 * If context is a parent, we need to register a process descriptor
2823 * describing a work queue and register all child contexts.
2824 */
2825 if (intel_context_is_parent(ce)) {
2826 struct guc_process_desc_v69 *pdesc;
2827
2828 ce->parallel.guc.wqi_tail = 0;
2829 ce->parallel.guc.wqi_head = 0;
2830
2831 desc->process_desc = i915_ggtt_offset(vma: ce->state) +
2832 __get_parent_scratch_offset(ce);
2833 desc->wq_addr = i915_ggtt_offset(vma: ce->state) +
2834 __get_wq_offset(ce);
2835 desc->wq_size = WQ_SIZE;
2836
2837 pdesc = __get_process_desc_v69(ce);
2838 memset(s: pdesc, c: 0, n: sizeof(*(pdesc)));
2839 pdesc->stage_id = ce->guc_id.id;
2840 pdesc->wq_base_addr = desc->wq_addr;
2841 pdesc->wq_size_bytes = desc->wq_size;
2842 pdesc->wq_status = WQ_STATUS_ACTIVE;
2843
2844 ce->parallel.guc.wq_head = &pdesc->head;
2845 ce->parallel.guc.wq_tail = &pdesc->tail;
2846 ce->parallel.guc.wq_status = &pdesc->wq_status;
2847
2848 for_each_child(ce, child) {
2849 desc = __get_lrc_desc_v69(guc, index: child->guc_id.id);
2850
2851 desc->engine_class =
2852 engine_class_to_guc_class(class: engine->class);
2853 desc->hw_context_desc = child->lrc.lrca;
2854 desc->priority = ce->guc_state.prio;
2855 desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;
2856 guc_context_policy_init_v69(engine, desc);
2857 }
2858
2859 clear_children_join_go_memory(ce);
2860 }
2861}
2862
2863static void prepare_context_registration_info_v70(struct intel_context *ce,
2864 struct guc_ctxt_registration_info *info)
2865{
2866 struct intel_engine_cs *engine = ce->engine;
2867 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
2868 u32 ctx_id = ce->guc_id.id;
2869
2870 GEM_BUG_ON(!engine->mask);
2871
2872 /*
2873 * Ensure LRC + CT vmas are is same region as write barrier is done
2874 * based on CT vma region.
2875 */
2876 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
2877 i915_gem_object_is_lmem(ce->ring->vma->obj));
2878
2879 memset(s: info, c: 0, n: sizeof(*info));
2880 info->context_idx = ctx_id;
2881 info->engine_class = engine_class_to_guc_class(class: engine->class);
2882 info->engine_submit_mask = engine->logical_mask;
2883 /*
2884 * NB: GuC interface supports 64 bit LRCA even though i915/HW
2885 * only supports 32 bit currently.
2886 */
2887 info->hwlrca_lo = lower_32_bits(ce->lrc.lrca);
2888 info->hwlrca_hi = upper_32_bits(ce->lrc.lrca);
2889 if (engine->flags & I915_ENGINE_HAS_EU_PRIORITY)
2890 info->hwlrca_lo |= map_guc_prio_to_lrc_desc_prio(prio: ce->guc_state.prio);
2891 info->flags = CONTEXT_REGISTRATION_FLAG_KMD;
2892
2893 /*
2894 * If context is a parent, we need to register a process descriptor
2895 * describing a work queue and register all child contexts.
2896 */
2897 if (intel_context_is_parent(ce)) {
2898 struct guc_sched_wq_desc *wq_desc;
2899 u64 wq_desc_offset, wq_base_offset;
2900
2901 ce->parallel.guc.wqi_tail = 0;
2902 ce->parallel.guc.wqi_head = 0;
2903
2904 wq_desc_offset = (u64)i915_ggtt_offset(vma: ce->state) +
2905 __get_parent_scratch_offset(ce);
2906 wq_base_offset = (u64)i915_ggtt_offset(vma: ce->state) +
2907 __get_wq_offset(ce);
2908 info->wq_desc_lo = lower_32_bits(wq_desc_offset);
2909 info->wq_desc_hi = upper_32_bits(wq_desc_offset);
2910 info->wq_base_lo = lower_32_bits(wq_base_offset);
2911 info->wq_base_hi = upper_32_bits(wq_base_offset);
2912 info->wq_size = WQ_SIZE;
2913
2914 wq_desc = __get_wq_desc_v70(ce);
2915 memset(s: wq_desc, c: 0, n: sizeof(*wq_desc));
2916 wq_desc->wq_status = WQ_STATUS_ACTIVE;
2917
2918 ce->parallel.guc.wq_head = &wq_desc->head;
2919 ce->parallel.guc.wq_tail = &wq_desc->tail;
2920 ce->parallel.guc.wq_status = &wq_desc->wq_status;
2921
2922 clear_children_join_go_memory(ce);
2923 }
2924}
2925
2926static int try_context_registration(struct intel_context *ce, bool loop)
2927{
2928 struct intel_engine_cs *engine = ce->engine;
2929 struct intel_runtime_pm *runtime_pm = engine->uncore->rpm;
2930 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
2931 intel_wakeref_t wakeref;
2932 u32 ctx_id = ce->guc_id.id;
2933 bool context_registered;
2934 int ret = 0;
2935
2936 GEM_BUG_ON(!sched_state_is_init(ce));
2937
2938 context_registered = ctx_id_mapped(guc, id: ctx_id);
2939
2940 clr_ctx_id_mapping(guc, id: ctx_id);
2941 set_ctx_id_mapping(guc, id: ctx_id, ce);
2942
2943 /*
2944 * The context_lookup xarray is used to determine if the hardware
2945 * context is currently registered. There are two cases in which it
2946 * could be registered either the guc_id has been stolen from another
2947 * context or the lrc descriptor address of this context has changed. In
2948 * either case the context needs to be deregistered with the GuC before
2949 * registering this context.
2950 */
2951 if (context_registered) {
2952 bool disabled;
2953 unsigned long flags;
2954
2955 trace_intel_context_steal_guc_id(ce);
2956 GEM_BUG_ON(!loop);
2957
2958 /* Seal race with Reset */
2959 spin_lock_irqsave(&ce->guc_state.lock, flags);
2960 disabled = submission_disabled(guc);
2961 if (likely(!disabled)) {
2962 set_context_wait_for_deregister_to_register(ce);
2963 intel_context_get(ce);
2964 }
2965 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
2966 if (unlikely(disabled)) {
2967 clr_ctx_id_mapping(guc, id: ctx_id);
2968 return 0; /* Will get registered later */
2969 }
2970
2971 /*
2972 * If stealing the guc_id, this ce has the same guc_id as the
2973 * context whose guc_id was stolen.
2974 */
2975 with_intel_runtime_pm(runtime_pm, wakeref)
2976 ret = deregister_context(ce, guc_id: ce->guc_id.id);
2977 if (unlikely(ret == -ENODEV))
2978 ret = 0; /* Will get registered later */
2979 } else {
2980 with_intel_runtime_pm(runtime_pm, wakeref)
2981 ret = register_context(ce, loop);
2982 if (unlikely(ret == -EBUSY)) {
2983 clr_ctx_id_mapping(guc, id: ctx_id);
2984 } else if (unlikely(ret == -ENODEV)) {
2985 clr_ctx_id_mapping(guc, id: ctx_id);
2986 ret = 0; /* Will get registered later */
2987 }
2988 }
2989
2990 return ret;
2991}
2992
2993static int __guc_context_pre_pin(struct intel_context *ce,
2994 struct intel_engine_cs *engine,
2995 struct i915_gem_ww_ctx *ww,
2996 void **vaddr)
2997{
2998 return lrc_pre_pin(ce, engine, ww, vaddr);
2999}
3000
3001static int __guc_context_pin(struct intel_context *ce,
3002 struct intel_engine_cs *engine,
3003 void *vaddr)
3004{
3005 if (i915_ggtt_offset(vma: ce->state) !=
3006 (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK))
3007 set_bit(CONTEXT_LRCA_DIRTY, addr: &ce->flags);
3008
3009 /*
3010 * GuC context gets pinned in guc_request_alloc. See that function for
3011 * explanation of why.
3012 */
3013
3014 return lrc_pin(ce, engine, vaddr);
3015}
3016
3017static int guc_context_pre_pin(struct intel_context *ce,
3018 struct i915_gem_ww_ctx *ww,
3019 void **vaddr)
3020{
3021 return __guc_context_pre_pin(ce, engine: ce->engine, ww, vaddr);
3022}
3023
3024static int guc_context_pin(struct intel_context *ce, void *vaddr)
3025{
3026 int ret = __guc_context_pin(ce, engine: ce->engine, vaddr);
3027
3028 if (likely(!ret && !intel_context_is_barrier(ce)))
3029 intel_engine_pm_get(engine: ce->engine);
3030
3031 return ret;
3032}
3033
3034static void guc_context_unpin(struct intel_context *ce)
3035{
3036 struct intel_guc *guc = ce_to_guc(ce);
3037
3038 __guc_context_update_stats(ce);
3039 unpin_guc_id(guc, ce);
3040 lrc_unpin(ce);
3041
3042 if (likely(!intel_context_is_barrier(ce)))
3043 intel_engine_pm_put_async(engine: ce->engine);
3044}
3045
3046static void guc_context_post_unpin(struct intel_context *ce)
3047{
3048 lrc_post_unpin(ce);
3049}
3050
3051static void __guc_context_sched_enable(struct intel_guc *guc,
3052 struct intel_context *ce)
3053{
3054 u32 action[] = {
3055 INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
3056 ce->guc_id.id,
3057 GUC_CONTEXT_ENABLE
3058 };
3059
3060 trace_intel_context_sched_enable(ce);
3061
3062 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
3063 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, loop: true);
3064}
3065
3066static void __guc_context_sched_disable(struct intel_guc *guc,
3067 struct intel_context *ce,
3068 u16 guc_id)
3069{
3070 u32 action[] = {
3071 INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
3072 guc_id, /* ce->guc_id.id not stable */
3073 GUC_CONTEXT_DISABLE
3074 };
3075
3076 GEM_BUG_ON(guc_id == GUC_INVALID_CONTEXT_ID);
3077
3078 GEM_BUG_ON(intel_context_is_child(ce));
3079 trace_intel_context_sched_disable(ce);
3080
3081 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
3082 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, loop: true);
3083}
3084
3085static void guc_blocked_fence_complete(struct intel_context *ce)
3086{
3087 lockdep_assert_held(&ce->guc_state.lock);
3088
3089 if (!i915_sw_fence_done(fence: &ce->guc_state.blocked))
3090 i915_sw_fence_complete(fence: &ce->guc_state.blocked);
3091}
3092
3093static void guc_blocked_fence_reinit(struct intel_context *ce)
3094{
3095 lockdep_assert_held(&ce->guc_state.lock);
3096 GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_state.blocked));
3097
3098 /*
3099 * This fence is always complete unless a pending schedule disable is
3100 * outstanding. We arm the fence here and complete it when we receive
3101 * the pending schedule disable complete message.
3102 */
3103 i915_sw_fence_fini(fence: &ce->guc_state.blocked);
3104 i915_sw_fence_reinit(fence: &ce->guc_state.blocked);
3105 i915_sw_fence_await(fence: &ce->guc_state.blocked);
3106 i915_sw_fence_commit(fence: &ce->guc_state.blocked);
3107}
3108
3109static u16 prep_context_pending_disable(struct intel_context *ce)
3110{
3111 lockdep_assert_held(&ce->guc_state.lock);
3112
3113 set_context_pending_disable(ce);
3114 clr_context_enabled(ce);
3115 guc_blocked_fence_reinit(ce);
3116 intel_context_get(ce);
3117
3118 return ce->guc_id.id;
3119}
3120
3121static struct i915_sw_fence *guc_context_block(struct intel_context *ce)
3122{
3123 struct intel_guc *guc = ce_to_guc(ce);
3124 unsigned long flags;
3125 struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
3126 intel_wakeref_t wakeref;
3127 u16 guc_id;
3128 bool enabled;
3129
3130 GEM_BUG_ON(intel_context_is_child(ce));
3131
3132 spin_lock_irqsave(&ce->guc_state.lock, flags);
3133
3134 incr_context_blocked(ce);
3135
3136 enabled = context_enabled(ce);
3137 if (unlikely(!enabled || submission_disabled(guc))) {
3138 if (enabled)
3139 clr_context_enabled(ce);
3140 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3141 return &ce->guc_state.blocked;
3142 }
3143
3144 /*
3145 * We add +2 here as the schedule disable complete CTB handler calls
3146 * intel_context_sched_disable_unpin (-2 to pin_count).
3147 */
3148 atomic_add(i: 2, v: &ce->pin_count);
3149
3150 guc_id = prep_context_pending_disable(ce);
3151
3152 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3153
3154 with_intel_runtime_pm(runtime_pm, wakeref)
3155 __guc_context_sched_disable(guc, ce, guc_id);
3156
3157 return &ce->guc_state.blocked;
3158}
3159
3160#define SCHED_STATE_MULTI_BLOCKED_MASK \
3161 (SCHED_STATE_BLOCKED_MASK & ~SCHED_STATE_BLOCKED)
3162#define SCHED_STATE_NO_UNBLOCK \
3163 (SCHED_STATE_MULTI_BLOCKED_MASK | \
3164 SCHED_STATE_PENDING_DISABLE | \
3165 SCHED_STATE_BANNED)
3166
3167static bool context_cant_unblock(struct intel_context *ce)
3168{
3169 lockdep_assert_held(&ce->guc_state.lock);
3170
3171 return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) ||
3172 context_guc_id_invalid(ce) ||
3173 !ctx_id_mapped(guc: ce_to_guc(ce), id: ce->guc_id.id) ||
3174 !intel_context_is_pinned(ce);
3175}
3176
3177static void guc_context_unblock(struct intel_context *ce)
3178{
3179 struct intel_guc *guc = ce_to_guc(ce);
3180 unsigned long flags;
3181 struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
3182 intel_wakeref_t wakeref;
3183 bool enable;
3184
3185 GEM_BUG_ON(context_enabled(ce));
3186 GEM_BUG_ON(intel_context_is_child(ce));
3187
3188 spin_lock_irqsave(&ce->guc_state.lock, flags);
3189
3190 if (unlikely(submission_disabled(guc) ||
3191 context_cant_unblock(ce))) {
3192 enable = false;
3193 } else {
3194 enable = true;
3195 set_context_pending_enable(ce);
3196 set_context_enabled(ce);
3197 intel_context_get(ce);
3198 }
3199
3200 decr_context_blocked(ce);
3201
3202 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3203
3204 if (enable) {
3205 with_intel_runtime_pm(runtime_pm, wakeref)
3206 __guc_context_sched_enable(guc, ce);
3207 }
3208}
3209
3210static void guc_context_cancel_request(struct intel_context *ce,
3211 struct i915_request *rq)
3212{
3213 struct intel_context *block_context =
3214 request_to_scheduling_context(rq);
3215
3216 if (i915_sw_fence_signaled(fence: &rq->submit)) {
3217 struct i915_sw_fence *fence;
3218
3219 intel_context_get(ce);
3220 fence = guc_context_block(ce: block_context);
3221 i915_sw_fence_wait(fence);
3222 if (!i915_request_completed(rq)) {
3223 __i915_request_skip(rq);
3224 guc_reset_state(ce, head: intel_ring_wrap(ring: ce->ring, pos: rq->head),
3225 scrub: true);
3226 }
3227
3228 guc_context_unblock(ce: block_context);
3229 intel_context_put(ce);
3230 }
3231}
3232
3233static void __guc_context_set_preemption_timeout(struct intel_guc *guc,
3234 u16 guc_id,
3235 u32 preemption_timeout)
3236{
3237 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
3238 struct context_policy policy;
3239
3240 __guc_context_policy_start_klv(policy: &policy, guc_id);
3241 __guc_context_policy_add_preemption_timeout(policy: &policy, data: preemption_timeout);
3242 __guc_context_set_context_policies(guc, policy: &policy, loop: true);
3243 } else {
3244 u32 action[] = {
3245 INTEL_GUC_ACTION_V69_SET_CONTEXT_PREEMPTION_TIMEOUT,
3246 guc_id,
3247 preemption_timeout
3248 };
3249
3250 intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), g2h_len_dw: 0, loop: true);
3251 }
3252}
3253
3254static void
3255guc_context_revoke(struct intel_context *ce, struct i915_request *rq,
3256 unsigned int preempt_timeout_ms)
3257{
3258 struct intel_guc *guc = ce_to_guc(ce);
3259 struct intel_runtime_pm *runtime_pm =
3260 &ce->engine->gt->i915->runtime_pm;
3261 intel_wakeref_t wakeref;
3262 unsigned long flags;
3263
3264 GEM_BUG_ON(intel_context_is_child(ce));
3265
3266 guc_flush_submissions(guc);
3267
3268 spin_lock_irqsave(&ce->guc_state.lock, flags);
3269 set_context_banned(ce);
3270
3271 if (submission_disabled(guc) ||
3272 (!context_enabled(ce) && !context_pending_disable(ce))) {
3273 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3274
3275 guc_cancel_context_requests(ce);
3276 intel_engine_signal_breadcrumbs(engine: ce->engine);
3277 } else if (!context_pending_disable(ce)) {
3278 u16 guc_id;
3279
3280 /*
3281 * We add +2 here as the schedule disable complete CTB handler
3282 * calls intel_context_sched_disable_unpin (-2 to pin_count).
3283 */
3284 atomic_add(i: 2, v: &ce->pin_count);
3285
3286 guc_id = prep_context_pending_disable(ce);
3287 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3288
3289 /*
3290 * In addition to disabling scheduling, set the preemption
3291 * timeout to the minimum value (1 us) so the banned context
3292 * gets kicked off the HW ASAP.
3293 */
3294 with_intel_runtime_pm(runtime_pm, wakeref) {
3295 __guc_context_set_preemption_timeout(guc, guc_id,
3296 preemption_timeout: preempt_timeout_ms);
3297 __guc_context_sched_disable(guc, ce, guc_id);
3298 }
3299 } else {
3300 if (!context_guc_id_invalid(ce))
3301 with_intel_runtime_pm(runtime_pm, wakeref)
3302 __guc_context_set_preemption_timeout(guc,
3303 guc_id: ce->guc_id.id,
3304 preemption_timeout: preempt_timeout_ms);
3305 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3306 }
3307}
3308
3309static void do_sched_disable(struct intel_guc *guc, struct intel_context *ce,
3310 unsigned long flags)
3311 __releases(ce->guc_state.lock)
3312{
3313 struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;
3314 intel_wakeref_t wakeref;
3315 u16 guc_id;
3316
3317 lockdep_assert_held(&ce->guc_state.lock);
3318 guc_id = prep_context_pending_disable(ce);
3319
3320 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3321
3322 with_intel_runtime_pm(runtime_pm, wakeref)
3323 __guc_context_sched_disable(guc, ce, guc_id);
3324}
3325
3326static bool bypass_sched_disable(struct intel_guc *guc,
3327 struct intel_context *ce)
3328{
3329 lockdep_assert_held(&ce->guc_state.lock);
3330 GEM_BUG_ON(intel_context_is_child(ce));
3331
3332 if (submission_disabled(guc) || context_guc_id_invalid(ce) ||
3333 !ctx_id_mapped(guc, id: ce->guc_id.id)) {
3334 clr_context_enabled(ce);
3335 return true;
3336 }
3337
3338 return !context_enabled(ce);
3339}
3340
3341static void __delay_sched_disable(struct work_struct *wrk)
3342{
3343 struct intel_context *ce =
3344 container_of(wrk, typeof(*ce), guc_state.sched_disable_delay_work.work);
3345 struct intel_guc *guc = ce_to_guc(ce);
3346 unsigned long flags;
3347
3348 spin_lock_irqsave(&ce->guc_state.lock, flags);
3349
3350 if (bypass_sched_disable(guc, ce)) {
3351 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3352 intel_context_sched_disable_unpin(ce);
3353 } else {
3354 do_sched_disable(guc, ce, flags);
3355 }
3356}
3357
3358static bool guc_id_pressure(struct intel_guc *guc, struct intel_context *ce)
3359{
3360 /*
3361 * parent contexts are perma-pinned, if we are unpinning do schedule
3362 * disable immediately.
3363 */
3364 if (intel_context_is_parent(ce))
3365 return true;
3366
3367 /*
3368 * If we are beyond the threshold for avail guc_ids, do schedule disable immediately.
3369 */
3370 return guc->submission_state.guc_ids_in_use >
3371 guc->submission_state.sched_disable_gucid_threshold;
3372}
3373
3374static void guc_context_sched_disable(struct intel_context *ce)
3375{
3376 struct intel_guc *guc = ce_to_guc(ce);
3377 u64 delay = guc->submission_state.sched_disable_delay_ms;
3378 unsigned long flags;
3379
3380 spin_lock_irqsave(&ce->guc_state.lock, flags);
3381
3382 if (bypass_sched_disable(guc, ce)) {
3383 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3384 intel_context_sched_disable_unpin(ce);
3385 } else if (!intel_context_is_closed(ce) && !guc_id_pressure(guc, ce) &&
3386 delay) {
3387 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3388 mod_delayed_work(wq: system_unbound_wq,
3389 dwork: &ce->guc_state.sched_disable_delay_work,
3390 delay: msecs_to_jiffies(m: delay));
3391 } else {
3392 do_sched_disable(guc, ce, flags);
3393 }
3394}
3395
3396static void guc_context_close(struct intel_context *ce)
3397{
3398 unsigned long flags;
3399
3400 if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&
3401 cancel_delayed_work(dwork: &ce->guc_state.sched_disable_delay_work))
3402 __delay_sched_disable(wrk: &ce->guc_state.sched_disable_delay_work.work);
3403
3404 spin_lock_irqsave(&ce->guc_state.lock, flags);
3405 set_context_close_done(ce);
3406 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3407}
3408
3409static inline int guc_lrc_desc_unpin(struct intel_context *ce)
3410{
3411 struct intel_guc *guc = ce_to_guc(ce);
3412 struct intel_gt *gt = guc_to_gt(guc);
3413 unsigned long flags;
3414 bool disabled;
3415 int ret;
3416
3417 GEM_BUG_ON(!intel_gt_pm_is_awake(gt));
3418 GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id));
3419 GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id));
3420 GEM_BUG_ON(context_enabled(ce));
3421
3422 /* Seal race with Reset */
3423 spin_lock_irqsave(&ce->guc_state.lock, flags);
3424 disabled = submission_disabled(guc);
3425 if (likely(!disabled)) {
3426 /*
3427 * Take a gt-pm ref and change context state to be destroyed.
3428 * NOTE: a G2H IRQ that comes after will put this gt-pm ref back
3429 */
3430 __intel_gt_pm_get(gt);
3431 set_context_destroyed(ce);
3432 clr_context_registered(ce);
3433 }
3434 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3435
3436 if (unlikely(disabled)) {
3437 release_guc_id(guc, ce);
3438 __guc_context_destroy(ce);
3439 return 0;
3440 }
3441
3442 /*
3443 * GuC is active, lets destroy this context, but at this point we can still be racing
3444 * with suspend, so we undo everything if the H2G fails in deregister_context so
3445 * that GuC reset will find this context during clean up.
3446 *
3447 * There is a race condition where the reset code could have altered
3448 * this context's state and done a wakeref put before we try to
3449 * deregister it here. So check if the context is still set to be
3450 * destroyed before undoing earlier changes, to avoid two wakeref puts
3451 * on the same context.
3452 */
3453 ret = deregister_context(ce, guc_id: ce->guc_id.id);
3454 if (ret) {
3455 bool pending_destroyed;
3456 spin_lock_irqsave(&ce->guc_state.lock, flags);
3457 pending_destroyed = context_destroyed(ce);
3458 if (pending_destroyed) {
3459 set_context_registered(ce);
3460 clr_context_destroyed(ce);
3461 }
3462 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3463 /*
3464 * As gt-pm is awake at function entry, intel_wakeref_put_async merely decrements
3465 * the wakeref immediately but per function spec usage call this after unlock.
3466 */
3467 if (pending_destroyed)
3468 intel_wakeref_put_async(wf: &gt->wakeref);
3469 }
3470
3471 return ret;
3472}
3473
3474static void __guc_context_destroy(struct intel_context *ce)
3475{
3476 GEM_BUG_ON(ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] ||
3477 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_HIGH] ||
3478 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] ||
3479 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_NORMAL]);
3480
3481 lrc_fini(ce);
3482 intel_context_fini(ce);
3483
3484 if (intel_engine_is_virtual(engine: ce->engine)) {
3485 struct guc_virtual_engine *ve =
3486 container_of(ce, typeof(*ve), context);
3487
3488 if (ve->base.breadcrumbs)
3489 intel_breadcrumbs_put(b: ve->base.breadcrumbs);
3490
3491 kfree(objp: ve);
3492 } else {
3493 intel_context_free(ce);
3494 }
3495}
3496
3497static void guc_flush_destroyed_contexts(struct intel_guc *guc)
3498{
3499 struct intel_context *ce;
3500 unsigned long flags;
3501
3502 GEM_BUG_ON(!submission_disabled(guc) &&
3503 guc_submission_initialized(guc));
3504
3505 while (!list_empty(head: &guc->submission_state.destroyed_contexts)) {
3506 spin_lock_irqsave(&guc->submission_state.lock, flags);
3507 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
3508 struct intel_context,
3509 destroyed_link);
3510 if (ce)
3511 list_del_init(entry: &ce->destroyed_link);
3512 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
3513
3514 if (!ce)
3515 break;
3516
3517 release_guc_id(guc, ce);
3518 __guc_context_destroy(ce);
3519 }
3520}
3521
3522static void deregister_destroyed_contexts(struct intel_guc *guc)
3523{
3524 struct intel_context *ce;
3525 unsigned long flags;
3526
3527 while (!list_empty(head: &guc->submission_state.destroyed_contexts)) {
3528 spin_lock_irqsave(&guc->submission_state.lock, flags);
3529 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
3530 struct intel_context,
3531 destroyed_link);
3532 if (ce)
3533 list_del_init(entry: &ce->destroyed_link);
3534 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
3535
3536 if (!ce)
3537 break;
3538
3539 if (guc_lrc_desc_unpin(ce)) {
3540 /*
3541 * This means GuC's CT link severed mid-way which could happen
3542 * in suspend-resume corner cases. In this case, put the
3543 * context back into the destroyed_contexts list which will
3544 * get picked up on the next context deregistration event or
3545 * purged in a GuC sanitization event (reset/unload/wedged/...).
3546 */
3547 spin_lock_irqsave(&guc->submission_state.lock, flags);
3548 list_add_tail(new: &ce->destroyed_link,
3549 head: &guc->submission_state.destroyed_contexts);
3550 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
3551 /* Bail now since the list might never be emptied if h2gs fail */
3552 break;
3553 }
3554
3555 }
3556}
3557
3558static void destroyed_worker_func(struct work_struct *w)
3559{
3560 struct intel_guc *guc = container_of(w, struct intel_guc,
3561 submission_state.destroyed_worker);
3562 struct intel_gt *gt = guc_to_gt(guc);
3563 intel_wakeref_t wakeref;
3564
3565 /*
3566 * In rare cases we can get here via async context-free fence-signals that
3567 * come very late in suspend flow or very early in resume flows. In these
3568 * cases, GuC won't be ready but just skipping it here is fine as these
3569 * pending-destroy-contexts get destroyed totally at GuC reset time at the
3570 * end of suspend.. OR.. this worker can be picked up later on the next
3571 * context destruction trigger after resume-completes
3572 */
3573 if (!intel_guc_is_ready(guc))
3574 return;
3575
3576 with_intel_gt_pm(gt, wakeref)
3577 deregister_destroyed_contexts(guc);
3578}
3579
3580static void guc_context_destroy(struct kref *kref)
3581{
3582 struct intel_context *ce = container_of(kref, typeof(*ce), ref);
3583 struct intel_guc *guc = ce_to_guc(ce);
3584 unsigned long flags;
3585 bool destroy;
3586
3587 /*
3588 * If the guc_id is invalid this context has been stolen and we can free
3589 * it immediately. Also can be freed immediately if the context is not
3590 * registered with the GuC or the GuC is in the middle of a reset.
3591 */
3592 spin_lock_irqsave(&guc->submission_state.lock, flags);
3593 destroy = submission_disabled(guc) || context_guc_id_invalid(ce) ||
3594 !ctx_id_mapped(guc, id: ce->guc_id.id);
3595 if (likely(!destroy)) {
3596 if (!list_empty(head: &ce->guc_id.link))
3597 list_del_init(entry: &ce->guc_id.link);
3598 list_add_tail(new: &ce->destroyed_link,
3599 head: &guc->submission_state.destroyed_contexts);
3600 } else {
3601 __release_guc_id(guc, ce);
3602 }
3603 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
3604 if (unlikely(destroy)) {
3605 __guc_context_destroy(ce);
3606 return;
3607 }
3608
3609 /*
3610 * We use a worker to issue the H2G to deregister the context as we can
3611 * take the GT PM for the first time which isn't allowed from an atomic
3612 * context.
3613 */
3614 queue_work(wq: system_unbound_wq, work: &guc->submission_state.destroyed_worker);
3615}
3616
3617static int guc_context_alloc(struct intel_context *ce)
3618{
3619 return lrc_alloc(ce, engine: ce->engine);
3620}
3621
3622static void __guc_context_set_prio(struct intel_guc *guc,
3623 struct intel_context *ce)
3624{
3625 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
3626 struct context_policy policy;
3627
3628 __guc_context_policy_start_klv(policy: &policy, guc_id: ce->guc_id.id);
3629 __guc_context_policy_add_priority(policy: &policy, data: ce->guc_state.prio);
3630 __guc_context_set_context_policies(guc, policy: &policy, loop: true);
3631 } else {
3632 u32 action[] = {
3633 INTEL_GUC_ACTION_V69_SET_CONTEXT_PRIORITY,
3634 ce->guc_id.id,
3635 ce->guc_state.prio,
3636 };
3637
3638 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), g2h_len_dw: 0, loop: true);
3639 }
3640}
3641
3642static void guc_context_set_prio(struct intel_guc *guc,
3643 struct intel_context *ce,
3644 u8 prio)
3645{
3646 GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH ||
3647 prio > GUC_CLIENT_PRIORITY_NORMAL);
3648 lockdep_assert_held(&ce->guc_state.lock);
3649
3650 if (ce->guc_state.prio == prio || submission_disabled(guc) ||
3651 !context_registered(ce)) {
3652 ce->guc_state.prio = prio;
3653 return;
3654 }
3655
3656 ce->guc_state.prio = prio;
3657 __guc_context_set_prio(guc, ce);
3658
3659 trace_intel_context_set_prio(ce);
3660}
3661
3662static inline u8 map_i915_prio_to_guc_prio(int prio)
3663{
3664 if (prio == I915_PRIORITY_NORMAL)
3665 return GUC_CLIENT_PRIORITY_KMD_NORMAL;
3666 else if (prio < I915_PRIORITY_NORMAL)
3667 return GUC_CLIENT_PRIORITY_NORMAL;
3668 else if (prio < I915_PRIORITY_DISPLAY)
3669 return GUC_CLIENT_PRIORITY_HIGH;
3670 else
3671 return GUC_CLIENT_PRIORITY_KMD_HIGH;
3672}
3673
3674static inline void add_context_inflight_prio(struct intel_context *ce,
3675 u8 guc_prio)
3676{
3677 lockdep_assert_held(&ce->guc_state.lock);
3678 GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));
3679
3680 ++ce->guc_state.prio_count[guc_prio];
3681
3682 /* Overflow protection */
3683 GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);
3684}
3685
3686static inline void sub_context_inflight_prio(struct intel_context *ce,
3687 u8 guc_prio)
3688{
3689 lockdep_assert_held(&ce->guc_state.lock);
3690 GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));
3691
3692 /* Underflow protection */
3693 GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);
3694
3695 --ce->guc_state.prio_count[guc_prio];
3696}
3697
3698static inline void update_context_prio(struct intel_context *ce)
3699{
3700 struct intel_guc *guc = &ce->engine->gt->uc.guc;
3701 int i;
3702
3703 BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0);
3704 BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL);
3705
3706 lockdep_assert_held(&ce->guc_state.lock);
3707
3708 for (i = 0; i < ARRAY_SIZE(ce->guc_state.prio_count); ++i) {
3709 if (ce->guc_state.prio_count[i]) {
3710 guc_context_set_prio(guc, ce, prio: i);
3711 break;
3712 }
3713 }
3714}
3715
3716static inline bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio)
3717{
3718 /* Lower value is higher priority */
3719 return new_guc_prio < old_guc_prio;
3720}
3721
3722static void add_to_context(struct i915_request *rq)
3723{
3724 struct intel_context *ce = request_to_scheduling_context(rq);
3725 u8 new_guc_prio = map_i915_prio_to_guc_prio(prio: rq_prio(rq));
3726
3727 GEM_BUG_ON(intel_context_is_child(ce));
3728 GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI);
3729
3730 spin_lock(lock: &ce->guc_state.lock);
3731 list_move_tail(list: &rq->sched.link, head: &ce->guc_state.requests);
3732
3733 if (rq->guc_prio == GUC_PRIO_INIT) {
3734 rq->guc_prio = new_guc_prio;
3735 add_context_inflight_prio(ce, guc_prio: rq->guc_prio);
3736 } else if (new_guc_prio_higher(old_guc_prio: rq->guc_prio, new_guc_prio)) {
3737 sub_context_inflight_prio(ce, guc_prio: rq->guc_prio);
3738 rq->guc_prio = new_guc_prio;
3739 add_context_inflight_prio(ce, guc_prio: rq->guc_prio);
3740 }
3741 update_context_prio(ce);
3742
3743 spin_unlock(lock: &ce->guc_state.lock);
3744}
3745
3746static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce)
3747{
3748 lockdep_assert_held(&ce->guc_state.lock);
3749
3750 if (rq->guc_prio != GUC_PRIO_INIT &&
3751 rq->guc_prio != GUC_PRIO_FINI) {
3752 sub_context_inflight_prio(ce, guc_prio: rq->guc_prio);
3753 update_context_prio(ce);
3754 }
3755 rq->guc_prio = GUC_PRIO_FINI;
3756}
3757
3758static void remove_from_context(struct i915_request *rq)
3759{
3760 struct intel_context *ce = request_to_scheduling_context(rq);
3761
3762 GEM_BUG_ON(intel_context_is_child(ce));
3763
3764 spin_lock_irq(lock: &ce->guc_state.lock);
3765
3766 list_del_init(entry: &rq->sched.link);
3767 clear_bit(nr: I915_FENCE_FLAG_PQUEUE, addr: &rq->fence.flags);
3768
3769 /* Prevent further __await_execution() registering a cb, then flush */
3770 set_bit(nr: I915_FENCE_FLAG_ACTIVE, addr: &rq->fence.flags);
3771
3772 guc_prio_fini(rq, ce);
3773
3774 spin_unlock_irq(lock: &ce->guc_state.lock);
3775
3776 atomic_dec(v: &ce->guc_id.ref);
3777 i915_request_notify_execute_cb_imm(rq);
3778}
3779
3780static const struct intel_context_ops guc_context_ops = {
3781 .flags = COPS_RUNTIME_CYCLES,
3782 .alloc = guc_context_alloc,
3783
3784 .close = guc_context_close,
3785
3786 .pre_pin = guc_context_pre_pin,
3787 .pin = guc_context_pin,
3788 .unpin = guc_context_unpin,
3789 .post_unpin = guc_context_post_unpin,
3790
3791 .revoke = guc_context_revoke,
3792
3793 .cancel_request = guc_context_cancel_request,
3794
3795 .enter = intel_context_enter_engine,
3796 .exit = intel_context_exit_engine,
3797
3798 .sched_disable = guc_context_sched_disable,
3799
3800 .update_stats = guc_context_update_stats,
3801
3802 .reset = lrc_reset,
3803 .destroy = guc_context_destroy,
3804
3805 .create_virtual = guc_create_virtual,
3806 .create_parallel = guc_create_parallel,
3807};
3808
3809static void submit_work_cb(struct irq_work *wrk)
3810{
3811 struct i915_request *rq = container_of(wrk, typeof(*rq), submit_work);
3812
3813 might_lock(&rq->engine->sched_engine->lock);
3814 i915_sw_fence_complete(fence: &rq->submit);
3815}
3816
3817static void __guc_signal_context_fence(struct intel_context *ce)
3818{
3819 struct i915_request *rq, *rn;
3820
3821 lockdep_assert_held(&ce->guc_state.lock);
3822
3823 if (!list_empty(head: &ce->guc_state.fences))
3824 trace_intel_context_fence_release(ce);
3825
3826 /*
3827 * Use an IRQ to ensure locking order of sched_engine->lock ->
3828 * ce->guc_state.lock is preserved.
3829 */
3830 list_for_each_entry_safe(rq, rn, &ce->guc_state.fences,
3831 guc_fence_link) {
3832 list_del(entry: &rq->guc_fence_link);
3833 irq_work_queue(work: &rq->submit_work);
3834 }
3835
3836 INIT_LIST_HEAD(list: &ce->guc_state.fences);
3837}
3838
3839static void guc_signal_context_fence(struct intel_context *ce)
3840{
3841 unsigned long flags;
3842
3843 GEM_BUG_ON(intel_context_is_child(ce));
3844
3845 spin_lock_irqsave(&ce->guc_state.lock, flags);
3846 clr_context_wait_for_deregister_to_register(ce);
3847 __guc_signal_context_fence(ce);
3848 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3849}
3850
3851static bool context_needs_register(struct intel_context *ce, bool new_guc_id)
3852{
3853 return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) ||
3854 !ctx_id_mapped(guc: ce_to_guc(ce), id: ce->guc_id.id)) &&
3855 !submission_disabled(guc: ce_to_guc(ce));
3856}
3857
3858static void guc_context_init(struct intel_context *ce)
3859{
3860 const struct i915_gem_context *ctx;
3861 int prio = I915_CONTEXT_DEFAULT_PRIORITY;
3862
3863 rcu_read_lock();
3864 ctx = rcu_dereference(ce->gem_context);
3865 if (ctx)
3866 prio = ctx->sched.priority;
3867 rcu_read_unlock();
3868
3869 ce->guc_state.prio = map_i915_prio_to_guc_prio(prio);
3870
3871 INIT_DELAYED_WORK(&ce->guc_state.sched_disable_delay_work,
3872 __delay_sched_disable);
3873
3874 set_bit(CONTEXT_GUC_INIT, addr: &ce->flags);
3875}
3876
3877static int guc_request_alloc(struct i915_request *rq)
3878{
3879 struct intel_context *ce = request_to_scheduling_context(rq);
3880 struct intel_guc *guc = ce_to_guc(ce);
3881 unsigned long flags;
3882 int ret;
3883
3884 GEM_BUG_ON(!intel_context_is_pinned(rq->context));
3885
3886 /*
3887 * Flush enough space to reduce the likelihood of waiting after
3888 * we start building the request - in which case we will just
3889 * have to repeat work.
3890 */
3891 rq->reserved_space += GUC_REQUEST_SIZE;
3892
3893 /*
3894 * Note that after this point, we have committed to using
3895 * this request as it is being used to both track the
3896 * state of engine initialisation and liveness of the
3897 * golden renderstate above. Think twice before you try
3898 * to cancel/unwind this request now.
3899 */
3900
3901 /* Unconditionally invalidate GPU caches and TLBs. */
3902 ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
3903 if (ret)
3904 return ret;
3905
3906 rq->reserved_space -= GUC_REQUEST_SIZE;
3907
3908 if (unlikely(!test_bit(CONTEXT_GUC_INIT, &ce->flags)))
3909 guc_context_init(ce);
3910
3911 /*
3912 * If the context gets closed while the execbuf is ongoing, the context
3913 * close code will race with the below code to cancel the delayed work.
3914 * If the context close wins the race and cancels the work, it will
3915 * immediately call the sched disable (see guc_context_close), so there
3916 * is a chance we can get past this check while the sched_disable code
3917 * is being executed. To make sure that code completes before we check
3918 * the status further down, we wait for the close process to complete.
3919 * Else, this code path could send a request down thinking that the
3920 * context is still in a schedule-enable mode while the GuC ends up
3921 * dropping the request completely because the disable did go from the
3922 * context_close path right to GuC just prior. In the event the CT is
3923 * full, we could potentially need to wait up to 1.5 seconds.
3924 */
3925 if (cancel_delayed_work_sync(dwork: &ce->guc_state.sched_disable_delay_work))
3926 intel_context_sched_disable_unpin(ce);
3927 else if (intel_context_is_closed(ce))
3928 if (wait_for(context_close_done(ce), 1500))
3929 guc_warn(guc, "timed out waiting on context sched close before realloc\n");
3930 /*
3931 * Call pin_guc_id here rather than in the pinning step as with
3932 * dma_resv, contexts can be repeatedly pinned / unpinned trashing the
3933 * guc_id and creating horrible race conditions. This is especially bad
3934 * when guc_id are being stolen due to over subscription. By the time
3935 * this function is reached, it is guaranteed that the guc_id will be
3936 * persistent until the generated request is retired. Thus, sealing these
3937 * race conditions. It is still safe to fail here if guc_id are
3938 * exhausted and return -EAGAIN to the user indicating that they can try
3939 * again in the future.
3940 *
3941 * There is no need for a lock here as the timeline mutex ensures at
3942 * most one context can be executing this code path at once. The
3943 * guc_id_ref is incremented once for every request in flight and
3944 * decremented on each retire. When it is zero, a lock around the
3945 * increment (in pin_guc_id) is needed to seal a race with unpin_guc_id.
3946 */
3947 if (atomic_add_unless(v: &ce->guc_id.ref, a: 1, u: 0))
3948 goto out;
3949
3950 ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */
3951 if (unlikely(ret < 0))
3952 return ret;
3953 if (context_needs_register(ce, new_guc_id: !!ret)) {
3954 ret = try_context_registration(ce, loop: true);
3955 if (unlikely(ret)) { /* unwind */
3956 if (ret == -EPIPE) {
3957 disable_submission(guc);
3958 goto out; /* GPU will be reset */
3959 }
3960 atomic_dec(v: &ce->guc_id.ref);
3961 unpin_guc_id(guc, ce);
3962 return ret;
3963 }
3964 }
3965
3966 clear_bit(CONTEXT_LRCA_DIRTY, addr: &ce->flags);
3967
3968out:
3969 /*
3970 * We block all requests on this context if a G2H is pending for a
3971 * schedule disable or context deregistration as the GuC will fail a
3972 * schedule enable or context registration if either G2H is pending
3973 * respectfully. Once a G2H returns, the fence is released that is
3974 * blocking these requests (see guc_signal_context_fence).
3975 */
3976 spin_lock_irqsave(&ce->guc_state.lock, flags);
3977 if (context_wait_for_deregister_to_register(ce) ||
3978 context_pending_disable(ce)) {
3979 init_irq_work(work: &rq->submit_work, func: submit_work_cb);
3980 i915_sw_fence_await(fence: &rq->submit);
3981
3982 list_add_tail(new: &rq->guc_fence_link, head: &ce->guc_state.fences);
3983 }
3984 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
3985
3986 return 0;
3987}
3988
3989static int guc_virtual_context_pre_pin(struct intel_context *ce,
3990 struct i915_gem_ww_ctx *ww,
3991 void **vaddr)
3992{
3993 struct intel_engine_cs *engine = guc_virtual_get_sibling(ve: ce->engine, sibling: 0);
3994
3995 return __guc_context_pre_pin(ce, engine, ww, vaddr);
3996}
3997
3998static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr)
3999{
4000 struct intel_engine_cs *engine = guc_virtual_get_sibling(ve: ce->engine, sibling: 0);
4001 int ret = __guc_context_pin(ce, engine, vaddr);
4002 intel_engine_mask_t tmp, mask = ce->engine->mask;
4003
4004 if (likely(!ret))
4005 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
4006 intel_engine_pm_get(engine);
4007
4008 return ret;
4009}
4010
4011static void guc_virtual_context_unpin(struct intel_context *ce)
4012{
4013 intel_engine_mask_t tmp, mask = ce->engine->mask;
4014 struct intel_engine_cs *engine;
4015 struct intel_guc *guc = ce_to_guc(ce);
4016
4017 GEM_BUG_ON(context_enabled(ce));
4018 GEM_BUG_ON(intel_context_is_barrier(ce));
4019
4020 unpin_guc_id(guc, ce);
4021 lrc_unpin(ce);
4022
4023 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
4024 intel_engine_pm_put_async(engine);
4025}
4026
4027static void guc_virtual_context_enter(struct intel_context *ce)
4028{
4029 intel_engine_mask_t tmp, mask = ce->engine->mask;
4030 struct intel_engine_cs *engine;
4031
4032 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
4033 intel_engine_pm_get(engine);
4034
4035 intel_timeline_enter(tl: ce->timeline);
4036}
4037
4038static void guc_virtual_context_exit(struct intel_context *ce)
4039{
4040 intel_engine_mask_t tmp, mask = ce->engine->mask;
4041 struct intel_engine_cs *engine;
4042
4043 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
4044 intel_engine_pm_put(engine);
4045
4046 intel_timeline_exit(tl: ce->timeline);
4047}
4048
4049static int guc_virtual_context_alloc(struct intel_context *ce)
4050{
4051 struct intel_engine_cs *engine = guc_virtual_get_sibling(ve: ce->engine, sibling: 0);
4052
4053 return lrc_alloc(ce, engine);
4054}
4055
4056static const struct intel_context_ops virtual_guc_context_ops = {
4057 .flags = COPS_RUNTIME_CYCLES,
4058 .alloc = guc_virtual_context_alloc,
4059
4060 .close = guc_context_close,
4061
4062 .pre_pin = guc_virtual_context_pre_pin,
4063 .pin = guc_virtual_context_pin,
4064 .unpin = guc_virtual_context_unpin,
4065 .post_unpin = guc_context_post_unpin,
4066
4067 .revoke = guc_context_revoke,
4068
4069 .cancel_request = guc_context_cancel_request,
4070
4071 .enter = guc_virtual_context_enter,
4072 .exit = guc_virtual_context_exit,
4073
4074 .sched_disable = guc_context_sched_disable,
4075 .update_stats = guc_context_update_stats,
4076
4077 .destroy = guc_context_destroy,
4078
4079 .get_sibling = guc_virtual_get_sibling,
4080};
4081
4082static int guc_parent_context_pin(struct intel_context *ce, void *vaddr)
4083{
4084 struct intel_engine_cs *engine = guc_virtual_get_sibling(ve: ce->engine, sibling: 0);
4085 struct intel_guc *guc = ce_to_guc(ce);
4086 int ret;
4087
4088 GEM_BUG_ON(!intel_context_is_parent(ce));
4089 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
4090
4091 ret = pin_guc_id(guc, ce);
4092 if (unlikely(ret < 0))
4093 return ret;
4094
4095 return __guc_context_pin(ce, engine, vaddr);
4096}
4097
4098static int guc_child_context_pin(struct intel_context *ce, void *vaddr)
4099{
4100 struct intel_engine_cs *engine = guc_virtual_get_sibling(ve: ce->engine, sibling: 0);
4101
4102 GEM_BUG_ON(!intel_context_is_child(ce));
4103 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
4104
4105 __intel_context_pin(ce: ce->parallel.parent);
4106 return __guc_context_pin(ce, engine, vaddr);
4107}
4108
4109static void guc_parent_context_unpin(struct intel_context *ce)
4110{
4111 struct intel_guc *guc = ce_to_guc(ce);
4112
4113 GEM_BUG_ON(context_enabled(ce));
4114 GEM_BUG_ON(intel_context_is_barrier(ce));
4115 GEM_BUG_ON(!intel_context_is_parent(ce));
4116 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
4117
4118 unpin_guc_id(guc, ce);
4119 lrc_unpin(ce);
4120}
4121
4122static void guc_child_context_unpin(struct intel_context *ce)
4123{
4124 GEM_BUG_ON(context_enabled(ce));
4125 GEM_BUG_ON(intel_context_is_barrier(ce));
4126 GEM_BUG_ON(!intel_context_is_child(ce));
4127 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
4128
4129 lrc_unpin(ce);
4130}
4131
4132static void guc_child_context_post_unpin(struct intel_context *ce)
4133{
4134 GEM_BUG_ON(!intel_context_is_child(ce));
4135 GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent));
4136 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
4137
4138 lrc_post_unpin(ce);
4139 intel_context_unpin(ce: ce->parallel.parent);
4140}
4141
4142static void guc_child_context_destroy(struct kref *kref)
4143{
4144 struct intel_context *ce = container_of(kref, typeof(*ce), ref);
4145
4146 __guc_context_destroy(ce);
4147}
4148
4149static const struct intel_context_ops virtual_parent_context_ops = {
4150 .alloc = guc_virtual_context_alloc,
4151
4152 .close = guc_context_close,
4153
4154 .pre_pin = guc_context_pre_pin,
4155 .pin = guc_parent_context_pin,
4156 .unpin = guc_parent_context_unpin,
4157 .post_unpin = guc_context_post_unpin,
4158
4159 .revoke = guc_context_revoke,
4160
4161 .cancel_request = guc_context_cancel_request,
4162
4163 .enter = guc_virtual_context_enter,
4164 .exit = guc_virtual_context_exit,
4165
4166 .sched_disable = guc_context_sched_disable,
4167
4168 .destroy = guc_context_destroy,
4169
4170 .get_sibling = guc_virtual_get_sibling,
4171};
4172
4173static const struct intel_context_ops virtual_child_context_ops = {
4174 .alloc = guc_virtual_context_alloc,
4175
4176 .pre_pin = guc_context_pre_pin,
4177 .pin = guc_child_context_pin,
4178 .unpin = guc_child_context_unpin,
4179 .post_unpin = guc_child_context_post_unpin,
4180
4181 .cancel_request = guc_context_cancel_request,
4182
4183 .enter = guc_virtual_context_enter,
4184 .exit = guc_virtual_context_exit,
4185
4186 .destroy = guc_child_context_destroy,
4187
4188 .get_sibling = guc_virtual_get_sibling,
4189};
4190
4191/*
4192 * The below override of the breadcrumbs is enabled when the user configures a
4193 * context for parallel submission (multi-lrc, parent-child).
4194 *
4195 * The overridden breadcrumbs implements an algorithm which allows the GuC to
4196 * safely preempt all the hw contexts configured for parallel submission
4197 * between each BB. The contract between the i915 and GuC is if the parent
4198 * context can be preempted, all the children can be preempted, and the GuC will
4199 * always try to preempt the parent before the children. A handshake between the
4200 * parent / children breadcrumbs ensures the i915 holds up its end of the deal
4201 * creating a window to preempt between each set of BBs.
4202 */
4203static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
4204 u64 offset, u32 len,
4205 const unsigned int flags);
4206static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
4207 u64 offset, u32 len,
4208 const unsigned int flags);
4209static u32 *
4210emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
4211 u32 *cs);
4212static u32 *
4213emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
4214 u32 *cs);
4215
4216static struct intel_context *
4217guc_create_parallel(struct intel_engine_cs **engines,
4218 unsigned int num_siblings,
4219 unsigned int width)
4220{
4221 struct intel_engine_cs **siblings = NULL;
4222 struct intel_context *parent = NULL, *ce, *err;
4223 int i, j;
4224
4225 siblings = kmalloc_array(num_siblings,
4226 sizeof(*siblings),
4227 GFP_KERNEL);
4228 if (!siblings)
4229 return ERR_PTR(error: -ENOMEM);
4230
4231 for (i = 0; i < width; ++i) {
4232 for (j = 0; j < num_siblings; ++j)
4233 siblings[j] = engines[i * num_siblings + j];
4234
4235 ce = intel_engine_create_virtual(siblings, count: num_siblings,
4236 FORCE_VIRTUAL);
4237 if (IS_ERR(ptr: ce)) {
4238 err = ERR_CAST(ptr: ce);
4239 goto unwind;
4240 }
4241
4242 if (i == 0) {
4243 parent = ce;
4244 parent->ops = &virtual_parent_context_ops;
4245 } else {
4246 ce->ops = &virtual_child_context_ops;
4247 intel_context_bind_parent_child(parent, child: ce);
4248 }
4249 }
4250
4251 parent->parallel.fence_context = dma_fence_context_alloc(num: 1);
4252
4253 parent->engine->emit_bb_start =
4254 emit_bb_start_parent_no_preempt_mid_batch;
4255 parent->engine->emit_fini_breadcrumb =
4256 emit_fini_breadcrumb_parent_no_preempt_mid_batch;
4257 parent->engine->emit_fini_breadcrumb_dw =
4258 12 + 4 * parent->parallel.number_children;
4259 for_each_child(parent, ce) {
4260 ce->engine->emit_bb_start =
4261 emit_bb_start_child_no_preempt_mid_batch;
4262 ce->engine->emit_fini_breadcrumb =
4263 emit_fini_breadcrumb_child_no_preempt_mid_batch;
4264 ce->engine->emit_fini_breadcrumb_dw = 16;
4265 }
4266
4267 kfree(objp: siblings);
4268 return parent;
4269
4270unwind:
4271 if (parent)
4272 intel_context_put(ce: parent);
4273 kfree(objp: siblings);
4274 return err;
4275}
4276
4277static bool
4278guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b)
4279{
4280 struct intel_engine_cs *sibling;
4281 intel_engine_mask_t tmp, mask = b->engine_mask;
4282 bool result = false;
4283
4284 for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)
4285 result |= intel_engine_irq_enable(engine: sibling);
4286
4287 return result;
4288}
4289
4290static void
4291guc_irq_disable_breadcrumbs(struct intel_breadcrumbs *b)
4292{
4293 struct intel_engine_cs *sibling;
4294 intel_engine_mask_t tmp, mask = b->engine_mask;
4295
4296 for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)
4297 intel_engine_irq_disable(engine: sibling);
4298}
4299
4300static void guc_init_breadcrumbs(struct intel_engine_cs *engine)
4301{
4302 int i;
4303
4304 /*
4305 * In GuC submission mode we do not know which physical engine a request
4306 * will be scheduled on, this creates a problem because the breadcrumb
4307 * interrupt is per physical engine. To work around this we attach
4308 * requests and direct all breadcrumb interrupts to the first instance
4309 * of an engine per class. In addition all breadcrumb interrupts are
4310 * enabled / disabled across an engine class in unison.
4311 */
4312 for (i = 0; i < MAX_ENGINE_INSTANCE; ++i) {
4313 struct intel_engine_cs *sibling =
4314 engine->gt->engine_class[engine->class][i];
4315
4316 if (sibling) {
4317 if (engine->breadcrumbs != sibling->breadcrumbs) {
4318 intel_breadcrumbs_put(b: engine->breadcrumbs);
4319 engine->breadcrumbs =
4320 intel_breadcrumbs_get(b: sibling->breadcrumbs);
4321 }
4322 break;
4323 }
4324 }
4325
4326 if (engine->breadcrumbs) {
4327 engine->breadcrumbs->engine_mask |= engine->mask;
4328 engine->breadcrumbs->irq_enable = guc_irq_enable_breadcrumbs;
4329 engine->breadcrumbs->irq_disable = guc_irq_disable_breadcrumbs;
4330 }
4331}
4332
4333static void guc_bump_inflight_request_prio(struct i915_request *rq,
4334 int prio)
4335{
4336 struct intel_context *ce = request_to_scheduling_context(rq);
4337 u8 new_guc_prio = map_i915_prio_to_guc_prio(prio);
4338
4339 /* Short circuit function */
4340 if (prio < I915_PRIORITY_NORMAL)
4341 return;
4342
4343 spin_lock(lock: &ce->guc_state.lock);
4344
4345 if (rq->guc_prio == GUC_PRIO_FINI)
4346 goto exit;
4347
4348 if (!new_guc_prio_higher(old_guc_prio: rq->guc_prio, new_guc_prio))
4349 goto exit;
4350
4351 if (rq->guc_prio != GUC_PRIO_INIT)
4352 sub_context_inflight_prio(ce, guc_prio: rq->guc_prio);
4353
4354 rq->guc_prio = new_guc_prio;
4355 add_context_inflight_prio(ce, guc_prio: rq->guc_prio);
4356 update_context_prio(ce);
4357
4358exit:
4359 spin_unlock(lock: &ce->guc_state.lock);
4360}
4361
4362static void guc_retire_inflight_request_prio(struct i915_request *rq)
4363{
4364 struct intel_context *ce = request_to_scheduling_context(rq);
4365
4366 spin_lock(lock: &ce->guc_state.lock);
4367 guc_prio_fini(rq, ce);
4368 spin_unlock(lock: &ce->guc_state.lock);
4369}
4370
4371static void sanitize_hwsp(struct intel_engine_cs *engine)
4372{
4373 struct intel_timeline *tl;
4374
4375 list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
4376 intel_timeline_reset_seqno(tl);
4377}
4378
4379static void guc_sanitize(struct intel_engine_cs *engine)
4380{
4381 /*
4382 * Poison residual state on resume, in case the suspend didn't!
4383 *
4384 * We have to assume that across suspend/resume (or other loss
4385 * of control) that the contents of our pinned buffers has been
4386 * lost, replaced by garbage. Since this doesn't always happen,
4387 * let's poison such state so that we more quickly spot when
4388 * we falsely assume it has been preserved.
4389 */
4390 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
4391 memset(s: engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
4392
4393 /*
4394 * The kernel_context HWSP is stored in the status_page. As above,
4395 * that may be lost on resume/initialisation, and so we need to
4396 * reset the value in the HWSP.
4397 */
4398 sanitize_hwsp(engine);
4399
4400 /* And scrub the dirty cachelines for the HWSP */
4401 drm_clflush_virt_range(addr: engine->status_page.addr, PAGE_SIZE);
4402
4403 intel_engine_reset_pinned_contexts(engine);
4404}
4405
4406static void setup_hwsp(struct intel_engine_cs *engine)
4407{
4408 intel_engine_set_hwsp_writemask(engine, mask: ~0u); /* HWSTAM */
4409
4410 ENGINE_WRITE_FW(engine,
4411 RING_HWS_PGA,
4412 i915_ggtt_offset(engine->status_page.vma));
4413}
4414
4415static void start_engine(struct intel_engine_cs *engine)
4416{
4417 ENGINE_WRITE_FW(engine,
4418 RING_MODE_GEN7,
4419 _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
4420
4421 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
4422 ENGINE_POSTING_READ(engine, RING_MI_MODE);
4423}
4424
4425static int guc_resume(struct intel_engine_cs *engine)
4426{
4427 assert_forcewakes_active(uncore: engine->uncore, fw_domains: FORCEWAKE_ALL);
4428
4429 intel_mocs_init_engine(engine);
4430
4431 intel_breadcrumbs_reset(b: engine->breadcrumbs);
4432
4433 setup_hwsp(engine);
4434 start_engine(engine);
4435
4436 if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
4437 xehp_enable_ccs_engines(engine);
4438
4439 return 0;
4440}
4441
4442static bool guc_sched_engine_disabled(struct i915_sched_engine *sched_engine)
4443{
4444 return !sched_engine->tasklet.callback;
4445}
4446
4447static void guc_set_default_submission(struct intel_engine_cs *engine)
4448{
4449 engine->submit_request = guc_submit_request;
4450}
4451
4452static inline int guc_kernel_context_pin(struct intel_guc *guc,
4453 struct intel_context *ce)
4454{
4455 int ret;
4456
4457 /*
4458 * Note: we purposefully do not check the returns below because
4459 * the registration can only fail if a reset is just starting.
4460 * This is called at the end of reset so presumably another reset
4461 * isn't happening and even it did this code would be run again.
4462 */
4463
4464 if (context_guc_id_invalid(ce)) {
4465 ret = pin_guc_id(guc, ce);
4466
4467 if (ret < 0)
4468 return ret;
4469 }
4470
4471 if (!test_bit(CONTEXT_GUC_INIT, &ce->flags))
4472 guc_context_init(ce);
4473
4474 ret = try_context_registration(ce, loop: true);
4475 if (ret)
4476 unpin_guc_id(guc, ce);
4477
4478 return ret;
4479}
4480
4481static inline int guc_init_submission(struct intel_guc *guc)
4482{
4483 struct intel_gt *gt = guc_to_gt(guc);
4484 struct intel_engine_cs *engine;
4485 enum intel_engine_id id;
4486
4487 /* make sure all descriptors are clean... */
4488 xa_destroy(&guc->context_lookup);
4489
4490 /*
4491 * A reset might have occurred while we had a pending stalled request,
4492 * so make sure we clean that up.
4493 */
4494 guc->stalled_request = NULL;
4495 guc->submission_stall_reason = STALL_NONE;
4496
4497 /*
4498 * Some contexts might have been pinned before we enabled GuC
4499 * submission, so we need to add them to the GuC bookeeping.
4500 * Also, after a reset the of the GuC we want to make sure that the
4501 * information shared with GuC is properly reset. The kernel LRCs are
4502 * not attached to the gem_context, so they need to be added separately.
4503 */
4504 for_each_engine(engine, gt, id) {
4505 struct intel_context *ce;
4506
4507 list_for_each_entry(ce, &engine->pinned_contexts_list,
4508 pinned_contexts_link) {
4509 int ret = guc_kernel_context_pin(guc, ce);
4510
4511 if (ret) {
4512 /* No point in trying to clean up as i915 will wedge on failure */
4513 return ret;
4514 }
4515 }
4516 }
4517
4518 return 0;
4519}
4520
4521static void guc_release(struct intel_engine_cs *engine)
4522{
4523 engine->sanitize = NULL; /* no longer in control, nothing to sanitize */
4524
4525 intel_engine_cleanup_common(engine);
4526 lrc_fini_wa_ctx(engine);
4527}
4528
4529static void virtual_guc_bump_serial(struct intel_engine_cs *engine)
4530{
4531 struct intel_engine_cs *e;
4532 intel_engine_mask_t tmp, mask = engine->mask;
4533
4534 for_each_engine_masked(e, engine->gt, mask, tmp)
4535 e->serial++;
4536}
4537
4538static void guc_default_vfuncs(struct intel_engine_cs *engine)
4539{
4540 /* Default vfuncs which can be overridden by each engine. */
4541
4542 engine->resume = guc_resume;
4543
4544 engine->cops = &guc_context_ops;
4545 engine->request_alloc = guc_request_alloc;
4546 engine->add_active_request = add_to_context;
4547 engine->remove_active_request = remove_from_context;
4548
4549 engine->sched_engine->schedule = i915_schedule;
4550
4551 engine->reset.prepare = guc_engine_reset_prepare;
4552 engine->reset.rewind = guc_rewind_nop;
4553 engine->reset.cancel = guc_reset_nop;
4554 engine->reset.finish = guc_reset_nop;
4555
4556 engine->emit_flush = gen8_emit_flush_xcs;
4557 engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
4558 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
4559 if (GRAPHICS_VER(engine->i915) >= 12) {
4560 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
4561 engine->emit_flush = gen12_emit_flush_xcs;
4562 }
4563 engine->set_default_submission = guc_set_default_submission;
4564 engine->busyness = guc_engine_busyness;
4565
4566 engine->flags |= I915_ENGINE_SUPPORTS_STATS;
4567 engine->flags |= I915_ENGINE_HAS_PREEMPTION;
4568 engine->flags |= I915_ENGINE_HAS_TIMESLICES;
4569
4570 /* Wa_14014475959:dg2 */
4571 if (engine->class == COMPUTE_CLASS)
4572 if (IS_GFX_GT_IP_STEP(engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
4573 IS_DG2(engine->i915))
4574 engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
4575
4576 /* Wa_16019325821 */
4577 /* Wa_14019159160 */
4578 if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) &&
4579 IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))
4580 engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
4581
4582 /*
4583 * TODO: GuC supports timeslicing and semaphores as well, but they're
4584 * handled by the firmware so some minor tweaks are required before
4585 * enabling.
4586 *
4587 * engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
4588 */
4589
4590 engine->emit_bb_start = gen8_emit_bb_start;
4591 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
4592 engine->emit_bb_start = xehp_emit_bb_start;
4593}
4594
4595static void rcs_submission_override(struct intel_engine_cs *engine)
4596{
4597 switch (GRAPHICS_VER(engine->i915)) {
4598 case 12:
4599 engine->emit_flush = gen12_emit_flush_rcs;
4600 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
4601 break;
4602 case 11:
4603 engine->emit_flush = gen11_emit_flush_rcs;
4604 engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
4605 break;
4606 default:
4607 engine->emit_flush = gen8_emit_flush_rcs;
4608 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
4609 break;
4610 }
4611}
4612
4613static inline void guc_default_irqs(struct intel_engine_cs *engine)
4614{
4615 engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT;
4616 intel_engine_set_irq_handler(engine, fn: cs_irq_handler);
4617}
4618
4619static void guc_sched_engine_destroy(struct kref *kref)
4620{
4621 struct i915_sched_engine *sched_engine =
4622 container_of(kref, typeof(*sched_engine), ref);
4623 struct intel_guc *guc = sched_engine->private_data;
4624
4625 guc->sched_engine = NULL;
4626 tasklet_kill(t: &sched_engine->tasklet); /* flush the callback */
4627 kfree(objp: sched_engine);
4628}
4629
4630int intel_guc_submission_setup(struct intel_engine_cs *engine)
4631{
4632 struct drm_i915_private *i915 = engine->i915;
4633 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
4634
4635 /*
4636 * The setup relies on several assumptions (e.g. irqs always enabled)
4637 * that are only valid on gen11+
4638 */
4639 GEM_BUG_ON(GRAPHICS_VER(i915) < 11);
4640
4641 if (!guc->sched_engine) {
4642 guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
4643 if (!guc->sched_engine)
4644 return -ENOMEM;
4645
4646 guc->sched_engine->schedule = i915_schedule;
4647 guc->sched_engine->disabled = guc_sched_engine_disabled;
4648 guc->sched_engine->private_data = guc;
4649 guc->sched_engine->destroy = guc_sched_engine_destroy;
4650 guc->sched_engine->bump_inflight_request_prio =
4651 guc_bump_inflight_request_prio;
4652 guc->sched_engine->retire_inflight_request_prio =
4653 guc_retire_inflight_request_prio;
4654 tasklet_setup(t: &guc->sched_engine->tasklet,
4655 callback: guc_submission_tasklet);
4656 }
4657 i915_sched_engine_put(sched_engine: engine->sched_engine);
4658 engine->sched_engine = i915_sched_engine_get(sched_engine: guc->sched_engine);
4659
4660 guc_default_vfuncs(engine);
4661 guc_default_irqs(engine);
4662 guc_init_breadcrumbs(engine);
4663
4664 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)
4665 rcs_submission_override(engine);
4666
4667 lrc_init_wa_ctx(engine);
4668
4669 /* Finally, take ownership and responsibility for cleanup! */
4670 engine->sanitize = guc_sanitize;
4671 engine->release = guc_release;
4672
4673 return 0;
4674}
4675
4676struct scheduling_policy {
4677 /* internal data */
4678 u32 max_words, num_words;
4679 u32 count;
4680 /* API data */
4681 struct guc_update_scheduling_policy h2g;
4682};
4683
4684static u32 __guc_scheduling_policy_action_size(struct scheduling_policy *policy)
4685{
4686 u32 *start = (void *)&policy->h2g;
4687 u32 *end = policy->h2g.data + policy->num_words;
4688 size_t delta = end - start;
4689
4690 return delta;
4691}
4692
4693static struct scheduling_policy *__guc_scheduling_policy_start_klv(struct scheduling_policy *policy)
4694{
4695 policy->h2g.header.action = INTEL_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV;
4696 policy->max_words = ARRAY_SIZE(policy->h2g.data);
4697 policy->num_words = 0;
4698 policy->count = 0;
4699
4700 return policy;
4701}
4702
4703static void __guc_scheduling_policy_add_klv(struct scheduling_policy *policy,
4704 u32 action, u32 *data, u32 len)
4705{
4706 u32 *klv_ptr = policy->h2g.data + policy->num_words;
4707
4708 GEM_BUG_ON((policy->num_words + 1 + len) > policy->max_words);
4709 *(klv_ptr++) = FIELD_PREP(GUC_KLV_0_KEY, action) |
4710 FIELD_PREP(GUC_KLV_0_LEN, len);
4711 memcpy(to: klv_ptr, from: data, len: sizeof(u32) * len);
4712 policy->num_words += 1 + len;
4713 policy->count++;
4714}
4715
4716static int __guc_action_set_scheduling_policies(struct intel_guc *guc,
4717 struct scheduling_policy *policy)
4718{
4719 int ret;
4720
4721 ret = intel_guc_send(guc, action: (u32 *)&policy->h2g,
4722 len: __guc_scheduling_policy_action_size(policy));
4723 if (ret < 0) {
4724 guc_probe_error(guc, "Failed to configure global scheduling policies: %pe!\n",
4725 ERR_PTR(ret));
4726 return ret;
4727 }
4728
4729 if (ret != policy->count) {
4730 guc_warn(guc, "global scheduler policy processed %d of %d KLVs!",
4731 ret, policy->count);
4732 if (ret > policy->count)
4733 return -EPROTO;
4734 }
4735
4736 return 0;
4737}
4738
4739static int guc_init_global_schedule_policy(struct intel_guc *guc)
4740{
4741 struct scheduling_policy policy;
4742 struct intel_gt *gt = guc_to_gt(guc);
4743 intel_wakeref_t wakeref;
4744 int ret;
4745
4746 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))
4747 return 0;
4748
4749 __guc_scheduling_policy_start_klv(policy: &policy);
4750
4751 with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref) {
4752 u32 yield[] = {
4753 GLOBAL_SCHEDULE_POLICY_RC_YIELD_DURATION,
4754 GLOBAL_SCHEDULE_POLICY_RC_YIELD_RATIO,
4755 };
4756
4757 __guc_scheduling_policy_add_klv(policy: &policy,
4758 action: GUC_SCHEDULING_POLICIES_KLV_ID_RENDER_COMPUTE_YIELD,
4759 data: yield, ARRAY_SIZE(yield));
4760
4761 ret = __guc_action_set_scheduling_policies(guc, policy: &policy);
4762 }
4763
4764 return ret;
4765}
4766
4767static void guc_route_semaphores(struct intel_guc *guc, bool to_guc)
4768{
4769 struct intel_gt *gt = guc_to_gt(guc);
4770 u32 val;
4771
4772 if (GRAPHICS_VER(gt->i915) < 12)
4773 return;
4774
4775 if (to_guc)
4776 val = GUC_SEM_INTR_ROUTE_TO_GUC | GUC_SEM_INTR_ENABLE_ALL;
4777 else
4778 val = 0;
4779
4780 intel_uncore_write(uncore: gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, val);
4781}
4782
4783int intel_guc_submission_enable(struct intel_guc *guc)
4784{
4785 int ret;
4786
4787 /* Semaphore interrupt enable and route to GuC */
4788 guc_route_semaphores(guc, to_guc: true);
4789
4790 ret = guc_init_submission(guc);
4791 if (ret)
4792 goto fail_sem;
4793
4794 ret = guc_init_engine_stats(guc);
4795 if (ret)
4796 goto fail_sem;
4797
4798 ret = guc_init_global_schedule_policy(guc);
4799 if (ret)
4800 goto fail_stats;
4801
4802 return 0;
4803
4804fail_stats:
4805 guc_fini_engine_stats(guc);
4806fail_sem:
4807 guc_route_semaphores(guc, to_guc: false);
4808 return ret;
4809}
4810
4811/* Note: By the time we're here, GuC may have already been reset */
4812void intel_guc_submission_disable(struct intel_guc *guc)
4813{
4814 guc_cancel_busyness_worker(guc);
4815
4816 /* Semaphore interrupt disable and route to host */
4817 guc_route_semaphores(guc, to_guc: false);
4818}
4819
4820static bool __guc_submission_supported(struct intel_guc *guc)
4821{
4822 /* GuC submission is unavailable for pre-Gen11 */
4823 return intel_guc_is_supported(guc) &&
4824 GRAPHICS_VER(guc_to_i915(guc)) >= 11;
4825}
4826
4827static bool __guc_submission_selected(struct intel_guc *guc)
4828{
4829 struct drm_i915_private *i915 = guc_to_i915(guc);
4830
4831 if (!intel_guc_submission_is_supported(guc))
4832 return false;
4833
4834 return i915->params.enable_guc & ENABLE_GUC_SUBMISSION;
4835}
4836
4837int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc)
4838{
4839 return guc->submission_state.num_guc_ids - NUMBER_MULTI_LRC_GUC_ID(guc);
4840}
4841
4842/*
4843 * This default value of 33 milisecs (+1 milisec round up) ensures 30fps or higher
4844 * workloads are able to enjoy the latency reduction when delaying the schedule-disable
4845 * operation. This matches the 30fps game-render + encode (real world) workload this
4846 * knob was tested against.
4847 */
4848#define SCHED_DISABLE_DELAY_MS 34
4849
4850/*
4851 * A threshold of 75% is a reasonable starting point considering that real world apps
4852 * generally don't get anywhere near this.
4853 */
4854#define NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(__guc) \
4855 (((intel_guc_sched_disable_gucid_threshold_max(guc)) * 3) / 4)
4856
4857void intel_guc_submission_init_early(struct intel_guc *guc)
4858{
4859 xa_init_flags(xa: &guc->context_lookup, XA_FLAGS_LOCK_IRQ);
4860
4861 spin_lock_init(&guc->submission_state.lock);
4862 INIT_LIST_HEAD(list: &guc->submission_state.guc_id_list);
4863 ida_init(ida: &guc->submission_state.guc_ids);
4864 INIT_LIST_HEAD(list: &guc->submission_state.destroyed_contexts);
4865 INIT_WORK(&guc->submission_state.destroyed_worker,
4866 destroyed_worker_func);
4867 INIT_WORK(&guc->submission_state.reset_fail_worker,
4868 reset_fail_worker_func);
4869
4870 spin_lock_init(&guc->timestamp.lock);
4871 INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping);
4872
4873 guc->submission_state.sched_disable_delay_ms = SCHED_DISABLE_DELAY_MS;
4874 guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID;
4875 guc->submission_state.sched_disable_gucid_threshold =
4876 NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(guc);
4877 guc->submission_supported = __guc_submission_supported(guc);
4878 guc->submission_selected = __guc_submission_selected(guc);
4879}
4880
4881static inline struct intel_context *
4882g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)
4883{
4884 struct intel_context *ce;
4885
4886 if (unlikely(ctx_id >= GUC_MAX_CONTEXT_ID)) {
4887 guc_err(guc, "Invalid ctx_id %u\n", ctx_id);
4888 return NULL;
4889 }
4890
4891 ce = __get_context(guc, id: ctx_id);
4892 if (unlikely(!ce)) {
4893 guc_err(guc, "Context is NULL, ctx_id %u\n", ctx_id);
4894 return NULL;
4895 }
4896
4897 if (unlikely(intel_context_is_child(ce))) {
4898 guc_err(guc, "Context is child, ctx_id %u\n", ctx_id);
4899 return NULL;
4900 }
4901
4902 return ce;
4903}
4904
4905static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno)
4906{
4907 struct intel_guc_tlb_wait *wait;
4908 unsigned long flags;
4909
4910 xa_lock_irqsave(&guc->tlb_lookup, flags);
4911 wait = xa_load(&guc->tlb_lookup, index: seqno);
4912
4913 if (wait)
4914 wake_up(&wait->wq);
4915 else
4916 guc_dbg(guc,
4917 "Stale TLB invalidation response with seqno %d\n", seqno);
4918
4919 xa_unlock_irqrestore(&guc->tlb_lookup, flags);
4920}
4921
4922int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
4923 const u32 *payload, u32 len)
4924{
4925 if (len < 1)
4926 return -EPROTO;
4927
4928 wait_wake_outstanding_tlb_g2h(guc, seqno: payload[0]);
4929 return 0;
4930}
4931
4932static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout)
4933{
4934 /*
4935 * This is equivalent to wait_woken() with the exception that
4936 * we do not wake up early if the kthread task has been completed.
4937 * As we are called from page reclaim in any task context,
4938 * we may be invoked from stopped kthreads, but we *must*
4939 * complete the wait from the HW.
4940 */
4941 do {
4942 set_current_state(TASK_UNINTERRUPTIBLE);
4943 if (wq_entry->flags & WQ_FLAG_WOKEN)
4944 break;
4945
4946 timeout = schedule_timeout(timeout);
4947 } while (timeout);
4948
4949 /* See wait_woken() and woken_wake_function() */
4950 __set_current_state(TASK_RUNNING);
4951 smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN);
4952
4953 return timeout;
4954}
4955
4956static bool intel_gt_is_enabled(const struct intel_gt *gt)
4957{
4958 /* Check if GT is wedged or suspended */
4959 if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(dev_priv: gt->i915))
4960 return false;
4961 return true;
4962}
4963
4964static int guc_send_invalidate_tlb(struct intel_guc *guc,
4965 enum intel_guc_tlb_invalidation_type type)
4966{
4967 struct intel_guc_tlb_wait _wq, *wq = &_wq;
4968 struct intel_gt *gt = guc_to_gt(guc);
4969 DEFINE_WAIT_FUNC(wait, woken_wake_function);
4970 int err;
4971 u32 seqno;
4972 u32 action[] = {
4973 INTEL_GUC_ACTION_TLB_INVALIDATION,
4974 0,
4975 REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_TYPE_MASK, type) |
4976 REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_MODE_MASK,
4977 INTEL_GUC_TLB_INVAL_MODE_HEAVY) |
4978 INTEL_GUC_TLB_INVAL_FLUSH_CACHE,
4979 };
4980 u32 size = ARRAY_SIZE(action);
4981
4982 /*
4983 * Early guard against GT enablement. TLB invalidation should not be
4984 * attempted if the GT is disabled due to suspend/wedge.
4985 */
4986 if (!intel_gt_is_enabled(gt))
4987 return -EINVAL;
4988
4989 init_waitqueue_head(&_wq.wq);
4990
4991 if (xa_alloc_cyclic_irq(xa: &guc->tlb_lookup, id: &seqno, entry: wq,
4992 xa_limit_32b, next: &guc->next_seqno,
4993 GFP_ATOMIC | __GFP_NOWARN) < 0) {
4994 /* Under severe memory pressure? Serialise TLB allocations */
4995 xa_lock_irq(&guc->tlb_lookup);
4996 wq = xa_load(&guc->tlb_lookup, index: guc->serial_slot);
4997 wait_event_lock_irq(wq->wq,
4998 !READ_ONCE(wq->busy),
4999 guc->tlb_lookup.xa_lock);
5000 /*
5001 * Update wq->busy under lock to ensure only one waiter can
5002 * issue the TLB invalidation command using the serial slot at a
5003 * time. The condition is set to true before releasing the lock
5004 * so that other caller continue to wait until woken up again.
5005 */
5006 wq->busy = true;
5007 xa_unlock_irq(&guc->tlb_lookup);
5008
5009 seqno = guc->serial_slot;
5010 }
5011
5012 action[1] = seqno;
5013
5014 add_wait_queue(wq_head: &wq->wq, wq_entry: &wait);
5015
5016 /* This is a critical reclaim path and thus we must loop here. */
5017 err = intel_guc_send_busy_loop(guc, action, len: size, G2H_LEN_DW_INVALIDATE_TLB, loop: true);
5018 if (err)
5019 goto out;
5020
5021 /*
5022 * Late guard against GT enablement. It is not an error for the TLB
5023 * invalidation to time out if the GT is disabled during the process
5024 * due to suspend/wedge. In fact, the TLB invalidation is cancelled
5025 * in this case.
5026 */
5027 if (!must_wait_woken(wq_entry: &wait, timeout: intel_guc_ct_max_queue_time_jiffies()) &&
5028 intel_gt_is_enabled(gt)) {
5029 guc_err(guc,
5030 "TLB invalidation response timed out for seqno %u\n", seqno);
5031 err = -ETIME;
5032 }
5033out:
5034 remove_wait_queue(wq_head: &wq->wq, wq_entry: &wait);
5035 if (seqno != guc->serial_slot)
5036 xa_erase_irq(xa: &guc->tlb_lookup, index: seqno);
5037
5038 return err;
5039}
5040
5041/* Send a H2G command to invalidate the TLBs at engine level and beyond. */
5042int intel_guc_invalidate_tlb_engines(struct intel_guc *guc)
5043{
5044 return guc_send_invalidate_tlb(guc, type: INTEL_GUC_TLB_INVAL_ENGINES);
5045}
5046
5047/* Send a H2G command to invalidate the GuC's internal TLB. */
5048int intel_guc_invalidate_tlb_guc(struct intel_guc *guc)
5049{
5050 return guc_send_invalidate_tlb(guc, type: INTEL_GUC_TLB_INVAL_GUC);
5051}
5052
5053int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
5054 const u32 *msg,
5055 u32 len)
5056{
5057 struct intel_context *ce;
5058 u32 ctx_id;
5059
5060 if (unlikely(len < 1)) {
5061 guc_err(guc, "Invalid length %u\n", len);
5062 return -EPROTO;
5063 }
5064 ctx_id = msg[0];
5065
5066 ce = g2h_context_lookup(guc, ctx_id);
5067 if (unlikely(!ce))
5068 return -EPROTO;
5069
5070 trace_intel_context_deregister_done(ce);
5071
5072#ifdef CONFIG_DRM_I915_SELFTEST
5073 if (unlikely(ce->drop_deregister)) {
5074 ce->drop_deregister = false;
5075 return 0;
5076 }
5077#endif
5078
5079 if (context_wait_for_deregister_to_register(ce)) {
5080 struct intel_runtime_pm *runtime_pm =
5081 &ce->engine->gt->i915->runtime_pm;
5082 intel_wakeref_t wakeref;
5083
5084 /*
5085 * Previous owner of this guc_id has been deregistered, now safe
5086 * register this context.
5087 */
5088 with_intel_runtime_pm(runtime_pm, wakeref)
5089 register_context(ce, loop: true);
5090 guc_signal_context_fence(ce);
5091 intel_context_put(ce);
5092 } else if (context_destroyed(ce)) {
5093 /* Context has been destroyed */
5094 intel_gt_pm_put_async_untracked(gt: guc_to_gt(guc));
5095 release_guc_id(guc, ce);
5096 __guc_context_destroy(ce);
5097 }
5098
5099 decr_outstanding_submission_g2h(guc);
5100
5101 return 0;
5102}
5103
5104int intel_guc_sched_done_process_msg(struct intel_guc *guc,
5105 const u32 *msg,
5106 u32 len)
5107{
5108 struct intel_context *ce;
5109 unsigned long flags;
5110 u32 ctx_id;
5111
5112 if (unlikely(len < 2)) {
5113 guc_err(guc, "Invalid length %u\n", len);
5114 return -EPROTO;
5115 }
5116 ctx_id = msg[0];
5117
5118 ce = g2h_context_lookup(guc, ctx_id);
5119 if (unlikely(!ce))
5120 return -EPROTO;
5121
5122 if (unlikely(context_destroyed(ce) ||
5123 (!context_pending_enable(ce) &&
5124 !context_pending_disable(ce)))) {
5125 guc_err(guc, "Bad context sched_state 0x%x, ctx_id %u\n",
5126 ce->guc_state.sched_state, ctx_id);
5127 return -EPROTO;
5128 }
5129
5130 trace_intel_context_sched_done(ce);
5131
5132 if (context_pending_enable(ce)) {
5133#ifdef CONFIG_DRM_I915_SELFTEST
5134 if (unlikely(ce->drop_schedule_enable)) {
5135 ce->drop_schedule_enable = false;
5136 return 0;
5137 }
5138#endif
5139
5140 spin_lock_irqsave(&ce->guc_state.lock, flags);
5141 clr_context_pending_enable(ce);
5142 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
5143 } else if (context_pending_disable(ce)) {
5144 bool banned;
5145
5146#ifdef CONFIG_DRM_I915_SELFTEST
5147 if (unlikely(ce->drop_schedule_disable)) {
5148 ce->drop_schedule_disable = false;
5149 return 0;
5150 }
5151#endif
5152
5153 /*
5154 * Unpin must be done before __guc_signal_context_fence,
5155 * otherwise a race exists between the requests getting
5156 * submitted + retired before this unpin completes resulting in
5157 * the pin_count going to zero and the context still being
5158 * enabled.
5159 */
5160 intel_context_sched_disable_unpin(ce);
5161
5162 spin_lock_irqsave(&ce->guc_state.lock, flags);
5163 banned = context_banned(ce);
5164 clr_context_banned(ce);
5165 clr_context_pending_disable(ce);
5166 __guc_signal_context_fence(ce);
5167 guc_blocked_fence_complete(ce);
5168 spin_unlock_irqrestore(lock: &ce->guc_state.lock, flags);
5169
5170 if (banned) {
5171 guc_cancel_context_requests(ce);
5172 intel_engine_signal_breadcrumbs(engine: ce->engine);
5173 }
5174 }
5175
5176 decr_outstanding_submission_g2h(guc);
5177 intel_context_put(ce);
5178
5179 return 0;
5180}
5181
5182static void capture_error_state(struct intel_guc *guc,
5183 struct intel_context *ce)
5184{
5185 struct intel_gt *gt = guc_to_gt(guc);
5186 struct drm_i915_private *i915 = gt->i915;
5187 intel_wakeref_t wakeref;
5188 intel_engine_mask_t engine_mask;
5189
5190 if (intel_engine_is_virtual(engine: ce->engine)) {
5191 struct intel_engine_cs *e;
5192 intel_engine_mask_t tmp, virtual_mask = ce->engine->mask;
5193
5194 engine_mask = 0;
5195 for_each_engine_masked(e, ce->engine->gt, virtual_mask, tmp) {
5196 bool match = intel_guc_capture_is_matching_engine(gt, ce, engine: e);
5197
5198 if (match) {
5199 intel_engine_set_hung_context(engine: e, ce);
5200 engine_mask |= e->mask;
5201 i915_increase_reset_engine_count(error: &i915->gpu_error,
5202 engine: e);
5203 }
5204 }
5205
5206 if (!engine_mask) {
5207 guc_warn(guc, "No matching physical engine capture for virtual engine context 0x%04X / %s",
5208 ce->guc_id.id, ce->engine->name);
5209 engine_mask = ~0U;
5210 }
5211 } else {
5212 intel_engine_set_hung_context(engine: ce->engine, ce);
5213 engine_mask = ce->engine->mask;
5214 i915_increase_reset_engine_count(error: &i915->gpu_error, engine: ce->engine);
5215 }
5216
5217 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
5218 i915_capture_error_state(gt, engine_mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE);
5219}
5220
5221static void guc_context_replay(struct intel_context *ce)
5222{
5223 struct i915_sched_engine *sched_engine = ce->engine->sched_engine;
5224
5225 __guc_reset_context(ce, stalled: ce->engine->mask);
5226 tasklet_hi_schedule(t: &sched_engine->tasklet);
5227}
5228
5229static void guc_handle_context_reset(struct intel_guc *guc,
5230 struct intel_context *ce)
5231{
5232 bool capture = intel_context_is_schedulable(ce);
5233
5234 trace_intel_context_reset(ce);
5235
5236 guc_dbg(guc, "%s context reset notification: 0x%04X on %s, exiting = %s, banned = %s\n",
5237 capture ? "Got" : "Ignoring",
5238 ce->guc_id.id, ce->engine->name,
5239 str_yes_no(intel_context_is_exiting(ce)),
5240 str_yes_no(intel_context_is_banned(ce)));
5241
5242 if (capture) {
5243 capture_error_state(guc, ce);
5244 guc_context_replay(ce);
5245 }
5246}
5247
5248int intel_guc_context_reset_process_msg(struct intel_guc *guc,
5249 const u32 *msg, u32 len)
5250{
5251 struct intel_context *ce;
5252 unsigned long flags;
5253 int ctx_id;
5254
5255 if (unlikely(len != 1)) {
5256 guc_err(guc, "Invalid length %u", len);
5257 return -EPROTO;
5258 }
5259
5260 ctx_id = msg[0];
5261
5262 /*
5263 * The context lookup uses the xarray but lookups only require an RCU lock
5264 * not the full spinlock. So take the lock explicitly and keep it until the
5265 * context has been reference count locked to ensure it can't be destroyed
5266 * asynchronously until the reset is done.
5267 */
5268 xa_lock_irqsave(&guc->context_lookup, flags);
5269 ce = g2h_context_lookup(guc, ctx_id);
5270 if (ce)
5271 intel_context_get(ce);
5272 xa_unlock_irqrestore(&guc->context_lookup, flags);
5273
5274 if (unlikely(!ce))
5275 return -EPROTO;
5276
5277 guc_handle_context_reset(guc, ce);
5278 intel_context_put(ce);
5279
5280 return 0;
5281}
5282
5283int intel_guc_error_capture_process_msg(struct intel_guc *guc,
5284 const u32 *msg, u32 len)
5285{
5286 u32 status;
5287
5288 if (unlikely(len != 1)) {
5289 guc_dbg(guc, "Invalid length %u", len);
5290 return -EPROTO;
5291 }
5292
5293 status = msg[0] & INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK;
5294 if (status == INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)
5295 guc_warn(guc, "No space for error capture");
5296
5297 intel_guc_capture_process(guc);
5298
5299 return 0;
5300}
5301
5302struct intel_engine_cs *
5303intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance)
5304{
5305 struct intel_gt *gt = guc_to_gt(guc);
5306 u8 engine_class = guc_class_to_engine_class(guc_class);
5307
5308 /* Class index is checked in class converter */
5309 GEM_BUG_ON(instance > MAX_ENGINE_INSTANCE);
5310
5311 return gt->engine_class[engine_class][instance];
5312}
5313
5314static void reset_fail_worker_func(struct work_struct *w)
5315{
5316 struct intel_guc *guc = container_of(w, struct intel_guc,
5317 submission_state.reset_fail_worker);
5318 struct intel_gt *gt = guc_to_gt(guc);
5319 intel_engine_mask_t reset_fail_mask;
5320 unsigned long flags;
5321
5322 spin_lock_irqsave(&guc->submission_state.lock, flags);
5323 reset_fail_mask = guc->submission_state.reset_fail_mask;
5324 guc->submission_state.reset_fail_mask = 0;
5325 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
5326
5327 if (likely(reset_fail_mask)) {
5328 struct intel_engine_cs *engine;
5329 enum intel_engine_id id;
5330
5331 /*
5332 * GuC is toast at this point - it dead loops after sending the failed
5333 * reset notification. So need to manually determine the guilty context.
5334 * Note that it should be reliable to do this here because the GuC is
5335 * toast and will not be scheduling behind the KMD's back.
5336 */
5337 for_each_engine_masked(engine, gt, reset_fail_mask, id)
5338 intel_guc_find_hung_context(engine);
5339
5340 intel_gt_handle_error(gt, engine_mask: reset_fail_mask,
5341 I915_ERROR_CAPTURE,
5342 fmt: "GuC failed to reset engine mask=0x%x",
5343 reset_fail_mask);
5344 }
5345}
5346
5347int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
5348 const u32 *msg, u32 len)
5349{
5350 struct intel_engine_cs *engine;
5351 u8 guc_class, instance;
5352 u32 reason;
5353 unsigned long flags;
5354
5355 if (unlikely(len != 3)) {
5356 guc_err(guc, "Invalid length %u", len);
5357 return -EPROTO;
5358 }
5359
5360 guc_class = msg[0];
5361 instance = msg[1];
5362 reason = msg[2];
5363
5364 engine = intel_guc_lookup_engine(guc, guc_class, instance);
5365 if (unlikely(!engine)) {
5366 guc_err(guc, "Invalid engine %d:%d", guc_class, instance);
5367 return -EPROTO;
5368 }
5369
5370 /*
5371 * This is an unexpected failure of a hardware feature. So, log a real
5372 * error message not just the informational that comes with the reset.
5373 */
5374 guc_err(guc, "Engine reset failed on %d:%d (%s) because 0x%08X",
5375 guc_class, instance, engine->name, reason);
5376
5377 spin_lock_irqsave(&guc->submission_state.lock, flags);
5378 guc->submission_state.reset_fail_mask |= engine->mask;
5379 spin_unlock_irqrestore(lock: &guc->submission_state.lock, flags);
5380
5381 /*
5382 * A GT reset flushes this worker queue (G2H handler) so we must use
5383 * another worker to trigger a GT reset.
5384 */
5385 queue_work(wq: system_unbound_wq, work: &guc->submission_state.reset_fail_worker);
5386
5387 return 0;
5388}
5389
5390void intel_guc_find_hung_context(struct intel_engine_cs *engine)
5391{
5392 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
5393 struct intel_context *ce;
5394 struct i915_request *rq;
5395 unsigned long index;
5396 unsigned long flags;
5397
5398 /* Reset called during driver load? GuC not yet initialised! */
5399 if (unlikely(!guc_submission_initialized(guc)))
5400 return;
5401
5402 xa_lock_irqsave(&guc->context_lookup, flags);
5403 xa_for_each(&guc->context_lookup, index, ce) {
5404 bool found;
5405
5406 if (!kref_get_unless_zero(kref: &ce->ref))
5407 continue;
5408
5409 xa_unlock(&guc->context_lookup);
5410
5411 if (!intel_context_is_pinned(ce))
5412 goto next;
5413
5414 if (intel_engine_is_virtual(engine: ce->engine)) {
5415 if (!(ce->engine->mask & engine->mask))
5416 goto next;
5417 } else {
5418 if (ce->engine != engine)
5419 goto next;
5420 }
5421
5422 found = false;
5423 spin_lock(lock: &ce->guc_state.lock);
5424 list_for_each_entry(rq, &ce->guc_state.requests, sched.link) {
5425 if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE)
5426 continue;
5427
5428 found = true;
5429 break;
5430 }
5431 spin_unlock(lock: &ce->guc_state.lock);
5432
5433 if (found) {
5434 intel_engine_set_hung_context(engine, ce);
5435
5436 /* Can only cope with one hang at a time... */
5437 intel_context_put(ce);
5438 xa_lock(&guc->context_lookup);
5439 goto done;
5440 }
5441
5442next:
5443 intel_context_put(ce);
5444 xa_lock(&guc->context_lookup);
5445 }
5446done:
5447 xa_unlock_irqrestore(&guc->context_lookup, flags);
5448}
5449
5450void intel_guc_dump_active_requests(struct intel_engine_cs *engine,
5451 struct i915_request *hung_rq,
5452 struct drm_printer *m)
5453{
5454 struct intel_guc *guc = gt_to_guc(gt: engine->gt);
5455 struct intel_context *ce;
5456 unsigned long index;
5457 unsigned long flags;
5458
5459 /* Reset called during driver load? GuC not yet initialised! */
5460 if (unlikely(!guc_submission_initialized(guc)))
5461 return;
5462
5463 xa_lock_irqsave(&guc->context_lookup, flags);
5464 xa_for_each(&guc->context_lookup, index, ce) {
5465 if (!kref_get_unless_zero(kref: &ce->ref))
5466 continue;
5467
5468 xa_unlock(&guc->context_lookup);
5469
5470 if (!intel_context_is_pinned(ce))
5471 goto next;
5472
5473 if (intel_engine_is_virtual(engine: ce->engine)) {
5474 if (!(ce->engine->mask & engine->mask))
5475 goto next;
5476 } else {
5477 if (ce->engine != engine)
5478 goto next;
5479 }
5480
5481 spin_lock(lock: &ce->guc_state.lock);
5482 intel_engine_dump_active_requests(requests: &ce->guc_state.requests,
5483 hung_rq, m);
5484 spin_unlock(lock: &ce->guc_state.lock);
5485
5486next:
5487 intel_context_put(ce);
5488 xa_lock(&guc->context_lookup);
5489 }
5490 xa_unlock_irqrestore(&guc->context_lookup, flags);
5491}
5492
5493void intel_guc_submission_print_info(struct intel_guc *guc,
5494 struct drm_printer *p)
5495{
5496 struct i915_sched_engine *sched_engine = guc->sched_engine;
5497 struct rb_node *rb;
5498 unsigned long flags;
5499
5500 if (!sched_engine)
5501 return;
5502
5503 drm_printf(p, f: "GuC Submission API Version: %d.%d.%d\n",
5504 guc->submission_version.major, guc->submission_version.minor,
5505 guc->submission_version.patch);
5506 drm_printf(p, f: "GuC Number Outstanding Submission G2H: %u\n",
5507 atomic_read(v: &guc->outstanding_submission_g2h));
5508 drm_printf(p, f: "GuC tasklet count: %u\n",
5509 atomic_read(v: &sched_engine->tasklet.count));
5510
5511 spin_lock_irqsave(&sched_engine->lock, flags);
5512 drm_printf(p, f: "Requests in GuC submit tasklet:\n");
5513 for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {
5514 struct i915_priolist *pl = to_priolist(rb);
5515 struct i915_request *rq;
5516
5517 priolist_for_each_request(rq, pl)
5518 drm_printf(p, f: "guc_id=%u, seqno=%llu\n",
5519 rq->context->guc_id.id,
5520 rq->fence.seqno);
5521 }
5522 spin_unlock_irqrestore(lock: &sched_engine->lock, flags);
5523 drm_printf(p, f: "\n");
5524}
5525
5526static inline void guc_log_context_priority(struct drm_printer *p,
5527 struct intel_context *ce)
5528{
5529 int i;
5530
5531 drm_printf(p, f: "\t\tPriority: %d\n", ce->guc_state.prio);
5532 drm_printf(p, f: "\t\tNumber Requests (lower index == higher priority)\n");
5533 for (i = GUC_CLIENT_PRIORITY_KMD_HIGH;
5534 i < GUC_CLIENT_PRIORITY_NUM; ++i) {
5535 drm_printf(p, f: "\t\tNumber requests in priority band[%d]: %d\n",
5536 i, ce->guc_state.prio_count[i]);
5537 }
5538 drm_printf(p, f: "\n");
5539}
5540
5541static inline void guc_log_context(struct drm_printer *p,
5542 struct intel_context *ce)
5543{
5544 drm_printf(p, f: "GuC lrc descriptor %u:\n", ce->guc_id.id);
5545 drm_printf(p, f: "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca);
5546 if (intel_context_pin_if_active(ce)) {
5547 drm_printf(p, f: "\t\tLRC Head: Internal %u, Memory %u\n",
5548 ce->ring->head,
5549 ce->lrc_reg_state[CTX_RING_HEAD]);
5550 drm_printf(p, f: "\t\tLRC Tail: Internal %u, Memory %u\n",
5551 ce->ring->tail,
5552 ce->lrc_reg_state[CTX_RING_TAIL]);
5553 intel_context_unpin(ce);
5554 } else {
5555 drm_printf(p, f: "\t\tLRC Head: Internal %u, Memory not pinned\n",
5556 ce->ring->head);
5557 drm_printf(p, f: "\t\tLRC Tail: Internal %u, Memory not pinned\n",
5558 ce->ring->tail);
5559 }
5560 drm_printf(p, f: "\t\tContext Pin Count: %u\n",
5561 atomic_read(v: &ce->pin_count));
5562 drm_printf(p, f: "\t\tGuC ID Ref Count: %u\n",
5563 atomic_read(v: &ce->guc_id.ref));
5564 drm_printf(p, f: "\t\tSchedule State: 0x%x\n",
5565 ce->guc_state.sched_state);
5566}
5567
5568void intel_guc_submission_print_context_info(struct intel_guc *guc,
5569 struct drm_printer *p)
5570{
5571 struct intel_context *ce;
5572 unsigned long index;
5573 unsigned long flags;
5574
5575 xa_lock_irqsave(&guc->context_lookup, flags);
5576 xa_for_each(&guc->context_lookup, index, ce) {
5577 GEM_BUG_ON(intel_context_is_child(ce));
5578
5579 guc_log_context(p, ce);
5580 guc_log_context_priority(p, ce);
5581
5582 if (intel_context_is_parent(ce)) {
5583 struct intel_context *child;
5584
5585 drm_printf(p, f: "\t\tNumber children: %u\n",
5586 ce->parallel.number_children);
5587
5588 if (ce->parallel.guc.wq_status) {
5589 drm_printf(p, f: "\t\tWQI Head: %u\n",
5590 READ_ONCE(*ce->parallel.guc.wq_head));
5591 drm_printf(p, f: "\t\tWQI Tail: %u\n",
5592 READ_ONCE(*ce->parallel.guc.wq_tail));
5593 drm_printf(p, f: "\t\tWQI Status: %u\n",
5594 READ_ONCE(*ce->parallel.guc.wq_status));
5595 }
5596
5597 if (ce->engine->emit_bb_start ==
5598 emit_bb_start_parent_no_preempt_mid_batch) {
5599 u8 i;
5600
5601 drm_printf(p, f: "\t\tChildren Go: %u\n",
5602 get_children_go_value(ce));
5603 for (i = 0; i < ce->parallel.number_children; ++i)
5604 drm_printf(p, f: "\t\tChildren Join: %u\n",
5605 get_children_join_value(ce, child_index: i));
5606 }
5607
5608 for_each_child(ce, child)
5609 guc_log_context(p, ce: child);
5610 }
5611 }
5612 xa_unlock_irqrestore(&guc->context_lookup, flags);
5613}
5614
5615static inline u32 get_children_go_addr(struct intel_context *ce)
5616{
5617 GEM_BUG_ON(!intel_context_is_parent(ce));
5618
5619 return i915_ggtt_offset(vma: ce->state) +
5620 __get_parent_scratch_offset(ce) +
5621 offsetof(struct parent_scratch, go.semaphore);
5622}
5623
5624static inline u32 get_children_join_addr(struct intel_context *ce,
5625 u8 child_index)
5626{
5627 GEM_BUG_ON(!intel_context_is_parent(ce));
5628
5629 return i915_ggtt_offset(vma: ce->state) +
5630 __get_parent_scratch_offset(ce) +
5631 offsetof(struct parent_scratch, join[child_index].semaphore);
5632}
5633
5634#define PARENT_GO_BB 1
5635#define PARENT_GO_FINI_BREADCRUMB 0
5636#define CHILD_GO_BB 1
5637#define CHILD_GO_FINI_BREADCRUMB 0
5638static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
5639 u64 offset, u32 len,
5640 const unsigned int flags)
5641{
5642 struct intel_context *ce = rq->context;
5643 u32 *cs;
5644 u8 i;
5645
5646 GEM_BUG_ON(!intel_context_is_parent(ce));
5647
5648 cs = intel_ring_begin(rq, num_dwords: 10 + 4 * ce->parallel.number_children);
5649 if (IS_ERR(ptr: cs))
5650 return PTR_ERR(ptr: cs);
5651
5652 /* Wait on children */
5653 for (i = 0; i < ce->parallel.number_children; ++i) {
5654 *cs++ = (MI_SEMAPHORE_WAIT |
5655 MI_SEMAPHORE_GLOBAL_GTT |
5656 MI_SEMAPHORE_POLL |
5657 MI_SEMAPHORE_SAD_EQ_SDD);
5658 *cs++ = PARENT_GO_BB;
5659 *cs++ = get_children_join_addr(ce, child_index: i);
5660 *cs++ = 0;
5661 }
5662
5663 /* Turn off preemption */
5664 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5665 *cs++ = MI_NOOP;
5666
5667 /* Tell children go */
5668 cs = gen8_emit_ggtt_write(cs,
5669 CHILD_GO_BB,
5670 gtt_offset: get_children_go_addr(ce),
5671 flags: 0);
5672
5673 /* Jump to batch */
5674 *cs++ = MI_BATCH_BUFFER_START_GEN8 |
5675 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
5676 *cs++ = lower_32_bits(offset);
5677 *cs++ = upper_32_bits(offset);
5678 *cs++ = MI_NOOP;
5679
5680 intel_ring_advance(rq, cs);
5681
5682 return 0;
5683}
5684
5685static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
5686 u64 offset, u32 len,
5687 const unsigned int flags)
5688{
5689 struct intel_context *ce = rq->context;
5690 struct intel_context *parent = intel_context_to_parent(ce);
5691 u32 *cs;
5692
5693 GEM_BUG_ON(!intel_context_is_child(ce));
5694
5695 cs = intel_ring_begin(rq, num_dwords: 12);
5696 if (IS_ERR(ptr: cs))
5697 return PTR_ERR(ptr: cs);
5698
5699 /* Signal parent */
5700 cs = gen8_emit_ggtt_write(cs,
5701 PARENT_GO_BB,
5702 gtt_offset: get_children_join_addr(ce: parent,
5703 child_index: ce->parallel.child_index),
5704 flags: 0);
5705
5706 /* Wait on parent for go */
5707 *cs++ = (MI_SEMAPHORE_WAIT |
5708 MI_SEMAPHORE_GLOBAL_GTT |
5709 MI_SEMAPHORE_POLL |
5710 MI_SEMAPHORE_SAD_EQ_SDD);
5711 *cs++ = CHILD_GO_BB;
5712 *cs++ = get_children_go_addr(ce: parent);
5713 *cs++ = 0;
5714
5715 /* Turn off preemption */
5716 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5717
5718 /* Jump to batch */
5719 *cs++ = MI_BATCH_BUFFER_START_GEN8 |
5720 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
5721 *cs++ = lower_32_bits(offset);
5722 *cs++ = upper_32_bits(offset);
5723
5724 intel_ring_advance(rq, cs);
5725
5726 return 0;
5727}
5728
5729static u32 *
5730__emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
5731 u32 *cs)
5732{
5733 struct intel_context *ce = rq->context;
5734 u8 i;
5735
5736 GEM_BUG_ON(!intel_context_is_parent(ce));
5737
5738 /* Wait on children */
5739 for (i = 0; i < ce->parallel.number_children; ++i) {
5740 *cs++ = (MI_SEMAPHORE_WAIT |
5741 MI_SEMAPHORE_GLOBAL_GTT |
5742 MI_SEMAPHORE_POLL |
5743 MI_SEMAPHORE_SAD_EQ_SDD);
5744 *cs++ = PARENT_GO_FINI_BREADCRUMB;
5745 *cs++ = get_children_join_addr(ce, child_index: i);
5746 *cs++ = 0;
5747 }
5748
5749 /* Turn on preemption */
5750 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5751 *cs++ = MI_NOOP;
5752
5753 /* Tell children go */
5754 cs = gen8_emit_ggtt_write(cs,
5755 CHILD_GO_FINI_BREADCRUMB,
5756 gtt_offset: get_children_go_addr(ce),
5757 flags: 0);
5758
5759 return cs;
5760}
5761
5762/*
5763 * If this true, a submission of multi-lrc requests had an error and the
5764 * requests need to be skipped. The front end (execuf IOCTL) should've called
5765 * i915_request_skip which squashes the BB but we still need to emit the fini
5766 * breadrcrumbs seqno write. At this point we don't know how many of the
5767 * requests in the multi-lrc submission were generated so we can't do the
5768 * handshake between the parent and children (e.g. if 4 requests should be
5769 * generated but 2nd hit an error only 1 would be seen by the GuC backend).
5770 * Simply skip the handshake, but still emit the breadcrumbd seqno, if an error
5771 * has occurred on any of the requests in submission / relationship.
5772 */
5773static inline bool skip_handshake(struct i915_request *rq)
5774{
5775 return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags);
5776}
5777
5778#define NON_SKIP_LEN 6
5779static u32 *
5780emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
5781 u32 *cs)
5782{
5783 struct intel_context *ce = rq->context;
5784 __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;
5785 __maybe_unused u32 *start_fini_breadcrumb_cs = cs;
5786
5787 GEM_BUG_ON(!intel_context_is_parent(ce));
5788
5789 if (unlikely(skip_handshake(rq))) {
5790 /*
5791 * NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch,
5792 * the NON_SKIP_LEN comes from the length of the emits below.
5793 */
5794 memset(s: cs, c: 0, n: sizeof(u32) *
5795 (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));
5796 cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
5797 } else {
5798 cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs);
5799 }
5800
5801 /* Emit fini breadcrumb */
5802 before_fini_breadcrumb_user_interrupt_cs = cs;
5803 cs = gen8_emit_ggtt_write(cs,
5804 value: rq->fence.seqno,
5805 gtt_offset: i915_request_active_timeline(rq)->hwsp_offset,
5806 flags: 0);
5807
5808 /* User interrupt */
5809 *cs++ = MI_USER_INTERRUPT;
5810 *cs++ = MI_NOOP;
5811
5812 /* Ensure our math for skip + emit is correct */
5813 GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=
5814 cs);
5815 GEM_BUG_ON(start_fini_breadcrumb_cs +
5816 ce->engine->emit_fini_breadcrumb_dw != cs);
5817
5818 rq->tail = intel_ring_offset(rq, addr: cs);
5819
5820 return cs;
5821}
5822
5823static u32 *
5824__emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
5825 u32 *cs)
5826{
5827 struct intel_context *ce = rq->context;
5828 struct intel_context *parent = intel_context_to_parent(ce);
5829
5830 GEM_BUG_ON(!intel_context_is_child(ce));
5831
5832 /* Turn on preemption */
5833 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5834 *cs++ = MI_NOOP;
5835
5836 /* Signal parent */
5837 cs = gen8_emit_ggtt_write(cs,
5838 PARENT_GO_FINI_BREADCRUMB,
5839 gtt_offset: get_children_join_addr(ce: parent,
5840 child_index: ce->parallel.child_index),
5841 flags: 0);
5842
5843 /* Wait parent on for go */
5844 *cs++ = (MI_SEMAPHORE_WAIT |
5845 MI_SEMAPHORE_GLOBAL_GTT |
5846 MI_SEMAPHORE_POLL |
5847 MI_SEMAPHORE_SAD_EQ_SDD);
5848 *cs++ = CHILD_GO_FINI_BREADCRUMB;
5849 *cs++ = get_children_go_addr(ce: parent);
5850 *cs++ = 0;
5851
5852 return cs;
5853}
5854
5855static u32 *
5856emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
5857 u32 *cs)
5858{
5859 struct intel_context *ce = rq->context;
5860 __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;
5861 __maybe_unused u32 *start_fini_breadcrumb_cs = cs;
5862
5863 GEM_BUG_ON(!intel_context_is_child(ce));
5864
5865 if (unlikely(skip_handshake(rq))) {
5866 /*
5867 * NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch,
5868 * the NON_SKIP_LEN comes from the length of the emits below.
5869 */
5870 memset(s: cs, c: 0, n: sizeof(u32) *
5871 (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));
5872 cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
5873 } else {
5874 cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs);
5875 }
5876
5877 /* Emit fini breadcrumb */
5878 before_fini_breadcrumb_user_interrupt_cs = cs;
5879 cs = gen8_emit_ggtt_write(cs,
5880 value: rq->fence.seqno,
5881 gtt_offset: i915_request_active_timeline(rq)->hwsp_offset,
5882 flags: 0);
5883
5884 /* User interrupt */
5885 *cs++ = MI_USER_INTERRUPT;
5886 *cs++ = MI_NOOP;
5887
5888 /* Ensure our math for skip + emit is correct */
5889 GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=
5890 cs);
5891 GEM_BUG_ON(start_fini_breadcrumb_cs +
5892 ce->engine->emit_fini_breadcrumb_dw != cs);
5893
5894 rq->tail = intel_ring_offset(rq, addr: cs);
5895
5896 return cs;
5897}
5898
5899#undef NON_SKIP_LEN
5900
5901static struct intel_context *
5902guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
5903 unsigned long flags)
5904{
5905 struct guc_virtual_engine *ve;
5906 struct intel_guc *guc;
5907 unsigned int n;
5908 int err;
5909
5910 ve = kzalloc(sizeof(*ve), GFP_KERNEL);
5911 if (!ve)
5912 return ERR_PTR(error: -ENOMEM);
5913
5914 guc = gt_to_guc(gt: siblings[0]->gt);
5915
5916 ve->base.i915 = siblings[0]->i915;
5917 ve->base.gt = siblings[0]->gt;
5918 ve->base.uncore = siblings[0]->uncore;
5919 ve->base.id = -1;
5920
5921 ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
5922 ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
5923 ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
5924 ve->base.saturated = ALL_ENGINES;
5925
5926 snprintf(buf: ve->base.name, size: sizeof(ve->base.name), fmt: "virtual");
5927
5928 ve->base.sched_engine = i915_sched_engine_get(sched_engine: guc->sched_engine);
5929
5930 ve->base.cops = &virtual_guc_context_ops;
5931 ve->base.request_alloc = guc_request_alloc;
5932 ve->base.bump_serial = virtual_guc_bump_serial;
5933
5934 ve->base.submit_request = guc_submit_request;
5935
5936 ve->base.flags = I915_ENGINE_IS_VIRTUAL;
5937
5938 BUILD_BUG_ON(ilog2(VIRTUAL_ENGINES) < I915_NUM_ENGINES);
5939 ve->base.mask = VIRTUAL_ENGINES;
5940
5941 intel_context_init(ce: &ve->context, engine: &ve->base);
5942
5943 for (n = 0; n < count; n++) {
5944 struct intel_engine_cs *sibling = siblings[n];
5945
5946 GEM_BUG_ON(!is_power_of_2(sibling->mask));
5947 if (sibling->mask & ve->base.mask) {
5948 guc_dbg(guc, "duplicate %s entry in load balancer\n",
5949 sibling->name);
5950 err = -EINVAL;
5951 goto err_put;
5952 }
5953
5954 ve->base.mask |= sibling->mask;
5955 ve->base.logical_mask |= sibling->logical_mask;
5956
5957 if (n != 0 && ve->base.class != sibling->class) {
5958 guc_dbg(guc, "invalid mixing of engine class, sibling %d, already %d\n",
5959 sibling->class, ve->base.class);
5960 err = -EINVAL;
5961 goto err_put;
5962 } else if (n == 0) {
5963 ve->base.class = sibling->class;
5964 ve->base.uabi_class = sibling->uabi_class;
5965 snprintf(buf: ve->base.name, size: sizeof(ve->base.name),
5966 fmt: "v%dx%d", ve->base.class, count);
5967 ve->base.context_size = sibling->context_size;
5968
5969 ve->base.add_active_request =
5970 sibling->add_active_request;
5971 ve->base.remove_active_request =
5972 sibling->remove_active_request;
5973 ve->base.emit_bb_start = sibling->emit_bb_start;
5974 ve->base.emit_flush = sibling->emit_flush;
5975 ve->base.emit_init_breadcrumb =
5976 sibling->emit_init_breadcrumb;
5977 ve->base.emit_fini_breadcrumb =
5978 sibling->emit_fini_breadcrumb;
5979 ve->base.emit_fini_breadcrumb_dw =
5980 sibling->emit_fini_breadcrumb_dw;
5981 ve->base.breadcrumbs =
5982 intel_breadcrumbs_get(b: sibling->breadcrumbs);
5983
5984 ve->base.flags |= sibling->flags;
5985
5986 ve->base.props.timeslice_duration_ms =
5987 sibling->props.timeslice_duration_ms;
5988 ve->base.props.preempt_timeout_ms =
5989 sibling->props.preempt_timeout_ms;
5990 }
5991 }
5992
5993 return &ve->context;
5994
5995err_put:
5996 intel_context_put(ce: &ve->context);
5997 return ERR_PTR(error: err);
5998}
5999
6000bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve)
6001{
6002 struct intel_engine_cs *engine;
6003 intel_engine_mask_t tmp, mask = ve->mask;
6004
6005 for_each_engine_masked(engine, ve->gt, mask, tmp)
6006 if (READ_ONCE(engine->props.heartbeat_interval_ms))
6007 return true;
6008
6009 return false;
6010}
6011
6012#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
6013#include "selftest_guc.c"
6014#include "selftest_guc_multi_lrc.c"
6015#include "selftest_guc_hangcheck.c"
6016#endif
6017