1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <linux/workqueue.h>
7
8#include "i915_drv.h" /* for_each_engine() */
9#include "i915_request.h"
10#include "intel_engine_heartbeat.h"
11#include "intel_execlists_submission.h"
12#include "intel_gt.h"
13#include "intel_gt_pm.h"
14#include "intel_gt_requests.h"
15#include "intel_timeline.h"
16
17static bool retire_requests(struct intel_timeline *tl)
18{
19 struct i915_request *rq, *rn;
20
21 list_for_each_entry_safe(rq, rn, &tl->requests, link)
22 if (!i915_request_retire(rq))
23 return false;
24
25 /* And check nothing new was submitted */
26 return !i915_active_fence_isset(active: &tl->last_request);
27}
28
29static bool engine_active(const struct intel_engine_cs *engine)
30{
31 return !list_empty(head: &engine->kernel_context->timeline->requests);
32}
33
34static bool flush_submission(struct intel_gt *gt, long timeout)
35{
36 struct intel_engine_cs *engine;
37 enum intel_engine_id id;
38 bool active = false;
39
40 if (!timeout)
41 return false;
42
43 if (!intel_gt_pm_is_awake(gt))
44 return false;
45
46 for_each_engine(engine, gt, id) {
47 intel_engine_flush_submission(engine);
48
49 /* Flush the background retirement and idle barriers */
50 flush_work(work: &engine->retire_work);
51 flush_delayed_work(dwork: &engine->wakeref.work);
52
53 /* Is the idle barrier still outstanding? */
54 active |= engine_active(engine);
55 }
56
57 return active;
58}
59
60static void engine_retire(struct work_struct *work)
61{
62 struct intel_engine_cs *engine =
63 container_of(work, typeof(*engine), retire_work);
64 struct intel_timeline *tl = xchg(&engine->retire, NULL);
65
66 do {
67 struct intel_timeline *next = xchg(&tl->retire, NULL);
68
69 /*
70 * Our goal here is to retire _idle_ timelines as soon as
71 * possible (as they are idle, we do not expect userspace
72 * to be cleaning up anytime soon).
73 *
74 * If the timeline is currently locked, either it is being
75 * retired elsewhere or about to be!
76 */
77 if (mutex_trylock(lock: &tl->mutex)) {
78 retire_requests(tl);
79 mutex_unlock(lock: &tl->mutex);
80 }
81 intel_timeline_put(timeline: tl);
82
83 GEM_BUG_ON(!next);
84 tl = ptr_mask_bits(next, 1);
85 } while (tl);
86}
87
88static bool add_retire(struct intel_engine_cs *engine,
89 struct intel_timeline *tl)
90{
91#define STUB ((struct intel_timeline *)1)
92 struct intel_timeline *first;
93
94 /*
95 * We open-code a llist here to include the additional tag [BIT(0)]
96 * so that we know when the timeline is already on a
97 * retirement queue: either this engine or another.
98 */
99
100 if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
101 return false;
102
103 intel_timeline_get(timeline: tl);
104 first = READ_ONCE(engine->retire);
105 do
106 tl->retire = ptr_pack_bits(first, 1, 1);
107 while (!try_cmpxchg(&engine->retire, &first, tl));
108
109 return !first;
110}
111
112void intel_engine_add_retire(struct intel_engine_cs *engine,
113 struct intel_timeline *tl)
114{
115 /* We don't deal well with the engine disappearing beneath us */
116 GEM_BUG_ON(intel_engine_is_virtual(engine));
117
118 if (add_retire(engine, tl))
119 queue_work(wq: engine->i915->unordered_wq, work: &engine->retire_work);
120}
121
122void intel_engine_init_retire(struct intel_engine_cs *engine)
123{
124 INIT_WORK(&engine->retire_work, engine_retire);
125}
126
127void intel_engine_fini_retire(struct intel_engine_cs *engine)
128{
129 flush_work(work: &engine->retire_work);
130 GEM_BUG_ON(engine->retire);
131}
132
133long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout,
134 long *remaining_timeout)
135{
136 struct intel_gt_timelines *timelines = &gt->timelines;
137 struct intel_timeline *tl, *tn;
138 unsigned long active_count = 0;
139 LIST_HEAD(free);
140
141 flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */
142 spin_lock(lock: &timelines->lock);
143 list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
144 if (!mutex_trylock(lock: &tl->mutex)) {
145 active_count++; /* report busy to caller, try again? */
146 continue;
147 }
148
149 intel_timeline_get(timeline: tl);
150 GEM_BUG_ON(!atomic_read(&tl->active_count));
151 atomic_inc(v: &tl->active_count); /* pin the list element */
152 spin_unlock(lock: &timelines->lock);
153
154 if (timeout > 0) {
155 struct dma_fence *fence;
156
157 fence = i915_active_fence_get(active: &tl->last_request);
158 if (fence) {
159 mutex_unlock(lock: &tl->mutex);
160
161 timeout = dma_fence_wait_timeout(fence,
162 intr: true,
163 timeout);
164 dma_fence_put(fence);
165
166 /* Retirement is best effort */
167 if (!mutex_trylock(lock: &tl->mutex)) {
168 active_count++;
169 goto out_active;
170 }
171 }
172 }
173
174 if (!retire_requests(tl))
175 active_count++;
176 mutex_unlock(lock: &tl->mutex);
177
178out_active: spin_lock(lock: &timelines->lock);
179
180 /* Resume list iteration after reacquiring spinlock */
181 list_safe_reset_next(tl, tn, link);
182 if (atomic_dec_and_test(v: &tl->active_count))
183 list_del(entry: &tl->link);
184
185 /* Defer the final release to after the spinlock */
186 if (refcount_dec_and_test(r: &tl->kref.refcount)) {
187 GEM_BUG_ON(atomic_read(&tl->active_count));
188 list_add(new: &tl->link, head: &free);
189 }
190 }
191 spin_unlock(lock: &timelines->lock);
192
193 list_for_each_entry_safe(tl, tn, &free, link)
194 __intel_timeline_free(kref: &tl->kref);
195
196 if (flush_submission(gt, timeout)) /* Wait, there's more! */
197 active_count++;
198
199 if (remaining_timeout)
200 *remaining_timeout = timeout;
201
202 return active_count ? timeout ?: -ETIME : 0;
203}
204
205static void retire_work_handler(struct work_struct *work)
206{
207 struct intel_gt *gt =
208 container_of(work, typeof(*gt), requests.retire_work.work);
209
210 queue_delayed_work(wq: gt->i915->unordered_wq, dwork: &gt->requests.retire_work,
211 delay: round_jiffies_up_relative(HZ));
212 intel_gt_retire_requests(gt);
213}
214
215void intel_gt_init_requests(struct intel_gt *gt)
216{
217 INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
218}
219
220void intel_gt_park_requests(struct intel_gt *gt)
221{
222 cancel_delayed_work(dwork: &gt->requests.retire_work);
223}
224
225void intel_gt_unpark_requests(struct intel_gt *gt)
226{
227 queue_delayed_work(wq: gt->i915->unordered_wq, dwork: &gt->requests.retire_work,
228 delay: round_jiffies_up_relative(HZ));
229}
230
231void intel_gt_fini_requests(struct intel_gt *gt)
232{
233 /* Wait until the work is marked as finished before unloading! */
234 cancel_delayed_work_sync(dwork: &gt->requests.retire_work);
235
236 flush_work(work: &gt->watchdog.work);
237}
238
239void intel_gt_watchdog_work(struct work_struct *work)
240{
241 struct intel_gt *gt =
242 container_of(work, typeof(*gt), watchdog.work);
243 struct i915_request *rq, *rn;
244 struct llist_node *first;
245
246 first = llist_del_all(head: &gt->watchdog.list);
247 if (!first)
248 return;
249
250 llist_for_each_entry_safe(rq, rn, first, watchdog.link) {
251 if (!i915_request_completed(rq)) {
252 struct dma_fence *f = &rq->fence;
253 const char __rcu *timeline;
254 const char __rcu *driver;
255
256 rcu_read_lock();
257 driver = dma_fence_driver_name(fence: f);
258 timeline = dma_fence_timeline_name(fence: f);
259 pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",
260 rcu_dereference(driver),
261 rcu_dereference(timeline),
262 f->seqno);
263 rcu_read_unlock();
264 i915_request_cancel(rq, error: -EINTR);
265 }
266 i915_request_put(rq);
267 }
268}
269