1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2015-2021 Intel Corporation
4 */
5
6#include <linux/kthread.h>
7#include <linux/string_helpers.h>
8#include <trace/events/dma_fence.h>
9#include <uapi/linux/sched/types.h>
10
11#include "i915_drv.h"
12#include "i915_trace.h"
13#include "intel_breadcrumbs.h"
14#include "intel_context.h"
15#include "intel_engine_pm.h"
16#include "intel_gt_pm.h"
17#include "intel_gt_requests.h"
18
19static bool irq_enable(struct intel_breadcrumbs *b)
20{
21 return intel_engine_irq_enable(engine: b->irq_engine);
22}
23
24static void irq_disable(struct intel_breadcrumbs *b)
25{
26 intel_engine_irq_disable(engine: b->irq_engine);
27}
28
29static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
30{
31 intel_wakeref_t wakeref;
32
33 /*
34 * Since we are waiting on a request, the GPU should be busy
35 * and should have its own rpm reference.
36 */
37 wakeref = intel_gt_pm_get_if_awake(gt: b->irq_engine->gt);
38 if (GEM_WARN_ON(!wakeref))
39 return;
40
41 /*
42 * The breadcrumb irq will be disarmed on the interrupt after the
43 * waiters are signaled. This gives us a single interrupt window in
44 * which we can add a new waiter and avoid the cost of re-enabling
45 * the irq.
46 */
47 WRITE_ONCE(b->irq_armed, wakeref);
48
49 /* Requests may have completed before we could enable the interrupt. */
50 if (!b->irq_enabled++ && b->irq_enable(b))
51 irq_work_queue(work: &b->irq_work);
52}
53
54static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
55{
56 if (!b->irq_engine)
57 return;
58
59 spin_lock(lock: &b->irq_lock);
60 if (!b->irq_armed)
61 __intel_breadcrumbs_arm_irq(b);
62 spin_unlock(lock: &b->irq_lock);
63}
64
65static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
66{
67 intel_wakeref_t wakeref = b->irq_armed;
68
69 GEM_BUG_ON(!b->irq_enabled);
70 if (!--b->irq_enabled)
71 b->irq_disable(b);
72
73 WRITE_ONCE(b->irq_armed, NULL);
74 intel_gt_pm_put_async(gt: b->irq_engine->gt, handle: wakeref);
75}
76
77static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
78{
79 spin_lock(lock: &b->irq_lock);
80 if (b->irq_armed)
81 __intel_breadcrumbs_disarm_irq(b);
82 spin_unlock(lock: &b->irq_lock);
83}
84
85static void add_signaling_context(struct intel_breadcrumbs *b,
86 struct intel_context *ce)
87{
88 lockdep_assert_held(&ce->signal_lock);
89
90 spin_lock(lock: &b->signalers_lock);
91 list_add_rcu(new: &ce->signal_link, head: &b->signalers);
92 spin_unlock(lock: &b->signalers_lock);
93}
94
95static bool remove_signaling_context(struct intel_breadcrumbs *b,
96 struct intel_context *ce)
97{
98 lockdep_assert_held(&ce->signal_lock);
99
100 if (!list_empty(head: &ce->signals))
101 return false;
102
103 spin_lock(lock: &b->signalers_lock);
104 list_del_rcu(entry: &ce->signal_link);
105 spin_unlock(lock: &b->signalers_lock);
106
107 return true;
108}
109
110__maybe_unused static bool
111check_signal_order(struct intel_context *ce, struct i915_request *rq)
112{
113 if (rq->context != ce)
114 return false;
115
116 if (!list_is_last(list: &rq->signal_link, head: &ce->signals) &&
117 i915_seqno_passed(seq1: rq->fence.seqno,
118 list_next_entry(rq, signal_link)->fence.seqno))
119 return false;
120
121 if (!list_is_first(list: &rq->signal_link, head: &ce->signals) &&
122 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
123 seq2: rq->fence.seqno))
124 return false;
125
126 return true;
127}
128
129static bool
130__dma_fence_signal(struct dma_fence *fence)
131{
132 return !test_and_set_bit(nr: DMA_FENCE_FLAG_SIGNALED_BIT, addr: &fence->flags);
133}
134
135static void
136__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
137{
138 fence->timestamp = timestamp;
139 set_bit(nr: DMA_FENCE_FLAG_TIMESTAMP_BIT, addr: &fence->flags);
140 trace_dma_fence_signaled(fence);
141}
142
143static void
144__dma_fence_signal__notify(struct dma_fence *fence,
145 const struct list_head *list)
146{
147 struct dma_fence_cb *cur, *tmp;
148
149 lockdep_assert_held(fence->lock);
150
151 list_for_each_entry_safe(cur, tmp, list, node) {
152 INIT_LIST_HEAD(list: &cur->node);
153 cur->func(fence, cur);
154 }
155}
156
157static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
158{
159 if (b->irq_engine)
160 intel_engine_add_retire(engine: b->irq_engine, tl);
161}
162
163static struct llist_node *
164slist_add(struct llist_node *node, struct llist_node *head)
165{
166 node->next = head;
167 return node;
168}
169
170static void signal_irq_work(struct irq_work *work)
171{
172 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
173 const ktime_t timestamp = ktime_get();
174 struct llist_node *signal, *sn;
175 struct intel_context *ce;
176
177 signal = NULL;
178 if (unlikely(!llist_empty(&b->signaled_requests)))
179 signal = llist_del_all(head: &b->signaled_requests);
180
181 /*
182 * Keep the irq armed until the interrupt after all listeners are gone.
183 *
184 * Enabling/disabling the interrupt is rather costly, roughly a couple
185 * of hundred microseconds. If we are proactive and enable/disable
186 * the interrupt around every request that wants a breadcrumb, we
187 * quickly drown in the extra orders of magnitude of latency imposed
188 * on request submission.
189 *
190 * So we try to be lazy, and keep the interrupts enabled until no
191 * more listeners appear within a breadcrumb interrupt interval (that
192 * is until a request completes that no one cares about). The
193 * observation is that listeners come in batches, and will often
194 * listen to a bunch of requests in succession. Though note on icl+,
195 * interrupts are always enabled due to concerns with rc6 being
196 * dysfunctional with per-engine interrupt masking.
197 *
198 * We also try to avoid raising too many interrupts, as they may
199 * be generated by userspace batches and it is unfortunately rather
200 * too easy to drown the CPU under a flood of GPU interrupts. Thus
201 * whenever no one appears to be listening, we turn off the interrupts.
202 * Fewer interrupts should conserve power -- at the very least, fewer
203 * interrupt draw less ire from other users of the system and tools
204 * like powertop.
205 */
206 if (!signal && READ_ONCE(b->irq_armed) && list_empty(head: &b->signalers))
207 intel_breadcrumbs_disarm_irq(b);
208
209 rcu_read_lock();
210 atomic_inc(v: &b->signaler_active);
211 list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
212 struct i915_request *rq;
213
214 list_for_each_entry_rcu(rq, &ce->signals, signal_link) {
215 bool release;
216
217 if (!__i915_request_is_complete(rq))
218 break;
219
220 if (!test_and_clear_bit(nr: I915_FENCE_FLAG_SIGNAL,
221 addr: &rq->fence.flags))
222 break;
223
224 /*
225 * Queue for execution after dropping the signaling
226 * spinlock as the callback chain may end up adding
227 * more signalers to the same context or engine.
228 */
229 spin_lock(lock: &ce->signal_lock);
230 list_del_rcu(entry: &rq->signal_link);
231 release = remove_signaling_context(b, ce);
232 spin_unlock(lock: &ce->signal_lock);
233 if (release) {
234 if (intel_timeline_is_last(tl: ce->timeline, rq))
235 add_retire(b, tl: ce->timeline);
236 intel_context_put(ce);
237 }
238
239 if (__dma_fence_signal(fence: &rq->fence))
240 /* We own signal_node now, xfer to local list */
241 signal = slist_add(node: &rq->signal_node, head: signal);
242 else
243 i915_request_put(rq);
244 }
245 }
246 atomic_dec(v: &b->signaler_active);
247 rcu_read_unlock();
248
249 llist_for_each_safe(signal, sn, signal) {
250 struct i915_request *rq =
251 llist_entry(signal, typeof(*rq), signal_node);
252 struct list_head cb_list;
253
254 if (rq->engine->sched_engine->retire_inflight_request_prio)
255 rq->engine->sched_engine->retire_inflight_request_prio(rq);
256
257 spin_lock(lock: &rq->lock);
258 list_replace(old: &rq->fence.cb_list, new: &cb_list);
259 __dma_fence_signal__timestamp(fence: &rq->fence, timestamp);
260 __dma_fence_signal__notify(fence: &rq->fence, list: &cb_list);
261 spin_unlock(lock: &rq->lock);
262
263 i915_request_put(rq);
264 }
265
266 /* Lazy irq enabling after HW submission */
267 if (!READ_ONCE(b->irq_armed) && !list_empty(head: &b->signalers))
268 intel_breadcrumbs_arm_irq(b);
269
270 /* And confirm that we still want irqs enabled before we yield */
271 if (READ_ONCE(b->irq_armed) && !atomic_read(v: &b->active))
272 intel_breadcrumbs_disarm_irq(b);
273}
274
275struct intel_breadcrumbs *
276intel_breadcrumbs_create(struct intel_engine_cs *irq_engine)
277{
278 struct intel_breadcrumbs *b;
279
280 b = kzalloc(sizeof(*b), GFP_KERNEL);
281 if (!b)
282 return NULL;
283
284 kref_init(kref: &b->ref);
285
286 spin_lock_init(&b->signalers_lock);
287 INIT_LIST_HEAD(list: &b->signalers);
288 init_llist_head(list: &b->signaled_requests);
289
290 spin_lock_init(&b->irq_lock);
291 init_irq_work(work: &b->irq_work, func: signal_irq_work);
292
293 b->irq_engine = irq_engine;
294 b->irq_enable = irq_enable;
295 b->irq_disable = irq_disable;
296
297 return b;
298}
299
300void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)
301{
302 unsigned long flags;
303
304 if (!b->irq_engine)
305 return;
306
307 spin_lock_irqsave(&b->irq_lock, flags);
308
309 if (b->irq_enabled)
310 b->irq_enable(b);
311 else
312 b->irq_disable(b);
313
314 spin_unlock_irqrestore(lock: &b->irq_lock, flags);
315}
316
317void __intel_breadcrumbs_park(struct intel_breadcrumbs *b)
318{
319 if (!READ_ONCE(b->irq_armed))
320 return;
321
322 /* Kick the work once more to drain the signalers, and disarm the irq */
323 irq_work_queue(work: &b->irq_work);
324}
325
326void intel_breadcrumbs_free(struct kref *kref)
327{
328 struct intel_breadcrumbs *b = container_of(kref, typeof(*b), ref);
329
330 irq_work_sync(work: &b->irq_work);
331 GEM_BUG_ON(!list_empty(&b->signalers));
332 GEM_BUG_ON(b->irq_armed);
333
334 kfree(objp: b);
335}
336
337static void irq_signal_request(struct i915_request *rq,
338 struct intel_breadcrumbs *b)
339{
340 if (!__dma_fence_signal(fence: &rq->fence))
341 return;
342
343 i915_request_get(rq);
344 if (llist_add(new: &rq->signal_node, head: &b->signaled_requests))
345 irq_work_queue(work: &b->irq_work);
346}
347
348static void insert_breadcrumb(struct i915_request *rq)
349{
350 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
351 struct intel_context *ce = rq->context;
352 struct list_head *pos;
353
354 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
355 return;
356
357 /*
358 * If the request is already completed, we can transfer it
359 * straight onto a signaled list, and queue the irq worker for
360 * its signal completion.
361 */
362 if (__i915_request_is_complete(rq)) {
363 irq_signal_request(rq, b);
364 return;
365 }
366
367 if (list_empty(head: &ce->signals)) {
368 intel_context_get(ce);
369 add_signaling_context(b, ce);
370 pos = &ce->signals;
371 } else {
372 /*
373 * We keep the seqno in retirement order, so we can break
374 * inside intel_engine_signal_breadcrumbs as soon as we've
375 * passed the last completed request (or seen a request that
376 * hasn't event started). We could walk the timeline->requests,
377 * but keeping a separate signalers_list has the advantage of
378 * hopefully being much smaller than the full list and so
379 * provides faster iteration and detection when there are no
380 * more interrupts required for this context.
381 *
382 * We typically expect to add new signalers in order, so we
383 * start looking for our insertion point from the tail of
384 * the list.
385 */
386 list_for_each_prev(pos, &ce->signals) {
387 struct i915_request *it =
388 list_entry(pos, typeof(*it), signal_link);
389
390 if (i915_seqno_passed(seq1: rq->fence.seqno, seq2: it->fence.seqno))
391 break;
392 }
393 }
394
395 i915_request_get(rq);
396 list_add_rcu(new: &rq->signal_link, head: pos);
397 GEM_BUG_ON(!check_signal_order(ce, rq));
398 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));
399 set_bit(nr: I915_FENCE_FLAG_SIGNAL, addr: &rq->fence.flags);
400
401 /*
402 * Defer enabling the interrupt to after HW submission and recheck
403 * the request as it may have completed and raised the interrupt as
404 * we were attaching it into the lists.
405 */
406 if (!READ_ONCE(b->irq_armed) || __i915_request_is_complete(rq))
407 irq_work_queue(work: &b->irq_work);
408}
409
410bool i915_request_enable_breadcrumb(struct i915_request *rq)
411{
412 struct intel_context *ce = rq->context;
413
414 /* Serialises with i915_request_retire() using rq->lock */
415 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
416 return true;
417
418 /*
419 * Peek at i915_request_submit()/i915_request_unsubmit() status.
420 *
421 * If the request is not yet active (and not signaled), we will
422 * attach the breadcrumb later.
423 */
424 if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
425 return true;
426
427 spin_lock(lock: &ce->signal_lock);
428 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
429 insert_breadcrumb(rq);
430 spin_unlock(lock: &ce->signal_lock);
431
432 return true;
433}
434
435void i915_request_cancel_breadcrumb(struct i915_request *rq)
436{
437 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
438 struct intel_context *ce = rq->context;
439 bool release;
440
441 spin_lock(lock: &ce->signal_lock);
442 if (!test_and_clear_bit(nr: I915_FENCE_FLAG_SIGNAL, addr: &rq->fence.flags)) {
443 spin_unlock(lock: &ce->signal_lock);
444 return;
445 }
446
447 list_del_rcu(entry: &rq->signal_link);
448 release = remove_signaling_context(b, ce);
449 spin_unlock(lock: &ce->signal_lock);
450 if (release)
451 intel_context_put(ce);
452
453 if (__i915_request_is_complete(rq))
454 irq_signal_request(rq, b);
455
456 i915_request_put(rq);
457}
458
459void intel_context_remove_breadcrumbs(struct intel_context *ce,
460 struct intel_breadcrumbs *b)
461{
462 struct i915_request *rq, *rn;
463 bool release = false;
464 unsigned long flags;
465
466 spin_lock_irqsave(&ce->signal_lock, flags);
467
468 if (list_empty(head: &ce->signals))
469 goto unlock;
470
471 list_for_each_entry_safe(rq, rn, &ce->signals, signal_link) {
472 GEM_BUG_ON(!__i915_request_is_complete(rq));
473 if (!test_and_clear_bit(nr: I915_FENCE_FLAG_SIGNAL,
474 addr: &rq->fence.flags))
475 continue;
476
477 list_del_rcu(entry: &rq->signal_link);
478 irq_signal_request(rq, b);
479 i915_request_put(rq);
480 }
481 release = remove_signaling_context(b, ce);
482
483unlock:
484 spin_unlock_irqrestore(lock: &ce->signal_lock, flags);
485 if (release)
486 intel_context_put(ce);
487
488 while (atomic_read(v: &b->signaler_active))
489 cpu_relax();
490}
491
492static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)
493{
494 struct intel_context *ce;
495 struct i915_request *rq;
496
497 drm_printf(p, f: "Signals:\n");
498
499 rcu_read_lock();
500 list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
501 list_for_each_entry_rcu(rq, &ce->signals, signal_link)
502 drm_printf(p, f: "\t[%llx:%llx%s] @ %dms\n",
503 rq->fence.context, rq->fence.seqno,
504 __i915_request_is_complete(rq) ? "!" :
505 __i915_request_has_started(rq) ? "*" :
506 "",
507 jiffies_to_msecs(j: jiffies - rq->emitted_jiffies));
508 }
509 rcu_read_unlock();
510}
511
512void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
513 struct drm_printer *p)
514{
515 struct intel_breadcrumbs *b;
516
517 b = engine->breadcrumbs;
518 if (!b)
519 return;
520
521 drm_printf(p, f: "IRQ: %s\n", str_enabled_disabled(v: b->irq_armed));
522 if (!list_empty(head: &b->signalers))
523 print_signals(b, p);
524}
525