1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2016 Intel Corporation
4 */
5
6#include <linux/dma-fence-array.h>
7#include <linux/dma-fence-chain.h>
8#include <linux/jiffies.h>
9
10#include "gt/intel_engine.h"
11#include "gt/intel_rps.h"
12
13#include "i915_gem_ioctls.h"
14#include "i915_gem_object.h"
15
16static long
17i915_gem_object_wait_fence(struct dma_fence *fence,
18 unsigned int flags,
19 long timeout)
20{
21 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
22
23 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
24 return timeout;
25
26 if (dma_fence_is_i915(fence))
27 return i915_request_wait_timeout(rq: to_request(fence), flags, timeout);
28
29 return dma_fence_wait_timeout(fence,
30 intr: flags & I915_WAIT_INTERRUPTIBLE,
31 timeout);
32}
33
34static void
35i915_gem_object_boost(struct dma_resv *resv, unsigned int flags)
36{
37 struct dma_resv_iter cursor;
38 struct dma_fence *fence;
39
40 /*
41 * Prescan all fences for potential boosting before we begin waiting.
42 *
43 * When we wait, we wait on outstanding fences serially. If the
44 * dma-resv contains a sequence such as 1:1, 1:2 instead of a reduced
45 * form 1:2, then as we look at each wait in turn we see that each
46 * request is currently executing and not worthy of boosting. But if
47 * we only happen to look at the final fence in the sequence (because
48 * of request coalescing or splitting between read/write arrays by
49 * the iterator), then we would boost. As such our decision to boost
50 * or not is delicately balanced on the order we wait on fences.
51 *
52 * So instead of looking for boosts sequentially, look for all boosts
53 * upfront and then wait on the outstanding fences.
54 */
55
56 dma_resv_iter_begin(cursor: &cursor, obj: resv,
57 usage: dma_resv_usage_rw(write: flags & I915_WAIT_ALL));
58 dma_resv_for_each_fence_unlocked(&cursor, fence)
59 if (dma_fence_is_i915(fence) &&
60 !i915_request_started(rq: to_request(fence)))
61 intel_rps_boost(rq: to_request(fence));
62 dma_resv_iter_end(cursor: &cursor);
63}
64
65static long
66i915_gem_object_wait_reservation(struct dma_resv *resv,
67 unsigned int flags,
68 long timeout)
69{
70 struct dma_resv_iter cursor;
71 struct dma_fence *fence;
72 long ret = timeout ?: 1;
73
74 i915_gem_object_boost(resv, flags);
75
76 dma_resv_iter_begin(cursor: &cursor, obj: resv,
77 usage: dma_resv_usage_rw(write: flags & I915_WAIT_ALL));
78 dma_resv_for_each_fence_unlocked(&cursor, fence) {
79 ret = i915_gem_object_wait_fence(fence, flags, timeout);
80 if (ret <= 0)
81 break;
82
83 if (timeout)
84 timeout = ret;
85 }
86 dma_resv_iter_end(cursor: &cursor);
87
88 return ret;
89}
90
91static void fence_set_priority(struct dma_fence *fence,
92 const struct i915_sched_attr *attr)
93{
94 struct i915_request *rq;
95 struct intel_engine_cs *engine;
96
97 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
98 return;
99
100 rq = to_request(fence);
101 engine = rq->engine;
102
103 rcu_read_lock(); /* RCU serialisation for set-wedged protection */
104 if (engine->sched_engine->schedule)
105 engine->sched_engine->schedule(rq, attr);
106 rcu_read_unlock();
107}
108
109void i915_gem_fence_wait_priority(struct dma_fence *fence,
110 const struct i915_sched_attr *attr)
111{
112 if (dma_fence_is_signaled(fence))
113 return;
114
115 local_bh_disable();
116
117 /* Recurse once into a fence-array */
118 if (dma_fence_is_array(fence)) {
119 struct dma_fence_array *array = to_dma_fence_array(fence);
120 int i;
121
122 for (i = 0; i < array->num_fences; i++)
123 fence_set_priority(fence: array->fences[i], attr);
124 } else if (dma_fence_is_chain(fence)) {
125 struct dma_fence *iter;
126
127 /* The chain is ordered; if we boost the last, we boost all */
128 dma_fence_chain_for_each(iter, fence) {
129 fence_set_priority(fence: to_dma_fence_chain(fence: iter)->fence,
130 attr);
131 break;
132 }
133 dma_fence_put(fence: iter);
134 } else {
135 fence_set_priority(fence, attr);
136 }
137
138 local_bh_enable(); /* kick the tasklets if queues were reprioritised */
139}
140
141int
142i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
143 unsigned int flags,
144 const struct i915_sched_attr *attr)
145{
146 struct dma_resv_iter cursor;
147 struct dma_fence *fence;
148
149 dma_resv_iter_begin(cursor: &cursor, obj: obj->base.resv,
150 usage: dma_resv_usage_rw(write: flags & I915_WAIT_ALL));
151 dma_resv_for_each_fence_unlocked(&cursor, fence)
152 i915_gem_fence_wait_priority(fence, attr);
153 dma_resv_iter_end(cursor: &cursor);
154 return 0;
155}
156
157/**
158 * i915_gem_object_wait - Waits for rendering to the object to be completed
159 * @obj: i915 gem object
160 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
161 * @timeout: how long to wait
162 */
163int
164i915_gem_object_wait(struct drm_i915_gem_object *obj,
165 unsigned int flags,
166 long timeout)
167{
168 might_sleep();
169 GEM_BUG_ON(timeout < 0);
170
171 timeout = i915_gem_object_wait_reservation(resv: obj->base.resv,
172 flags, timeout);
173
174 if (timeout < 0)
175 return timeout;
176
177 return !timeout ? -ETIME : 0;
178}
179
180static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
181{
182 /* nsecs_to_jiffies64() does not guard against overflow */
183 if ((NSEC_PER_SEC % HZ) != 0 &&
184 div_u64(dividend: n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
185 return MAX_JIFFY_OFFSET;
186
187 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
188}
189
190static unsigned long to_wait_timeout(s64 timeout_ns)
191{
192 if (timeout_ns < 0)
193 return MAX_SCHEDULE_TIMEOUT;
194
195 if (timeout_ns == 0)
196 return 0;
197
198 return nsecs_to_jiffies_timeout(n: timeout_ns);
199}
200
201/**
202 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
203 * @dev: drm device pointer
204 * @data: ioctl data blob
205 * @file: drm file pointer
206 *
207 * Returns 0 if successful, else an error is returned with the remaining time in
208 * the timeout parameter.
209 * -ETIME: object is still busy after timeout
210 * -ERESTARTSYS: signal interrupted the wait
211 * -ENONENT: object doesn't exist
212 * Also possible, but rare:
213 * -EAGAIN: incomplete, restart syscall
214 * -ENOMEM: damn
215 * -ENODEV: Internal IRQ fail
216 * -E?: The add request failed
217 *
218 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
219 * non-zero timeout parameter the wait ioctl will wait for the given number of
220 * nanoseconds on an object becoming unbusy. Since the wait occurs without
221 * holding a global or exclusive lock the object may become re-busied before
222 * this function completes. A similar but shorter * race condition exists
223 * in the busy ioctl
224 */
225int
226i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
227{
228 struct drm_i915_gem_wait *args = data;
229 struct drm_i915_gem_object *obj;
230 ktime_t start;
231 long ret;
232
233 if (args->flags != 0)
234 return -EINVAL;
235
236 obj = i915_gem_object_lookup(file, handle: args->bo_handle);
237 if (!obj)
238 return -ENOENT;
239
240 start = ktime_get();
241
242 ret = i915_gem_object_wait(obj,
243 I915_WAIT_INTERRUPTIBLE |
244 I915_WAIT_PRIORITY |
245 I915_WAIT_ALL,
246 timeout: to_wait_timeout(timeout_ns: args->timeout_ns));
247
248 if (args->timeout_ns > 0) {
249 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
250 if (args->timeout_ns < 0)
251 args->timeout_ns = 0;
252
253 /*
254 * Apparently ktime isn't accurate enough and occasionally has a
255 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
256 * things up to make the test happy. We allow up to 1 jiffy.
257 *
258 * This is a regression from the timespec->ktime conversion.
259 */
260 if (ret == -ETIME && !nsecs_to_jiffies(n: args->timeout_ns))
261 args->timeout_ns = 0;
262
263 /* Asked to wait beyond the jiffy/scheduler precision? */
264 if (ret == -ETIME && args->timeout_ns)
265 ret = -EAGAIN;
266 }
267
268 i915_gem_object_put(obj);
269 return ret;
270}
271
272/**
273 * i915_gem_object_wait_migration - Sync an accelerated migration operation
274 * @obj: The migrating object.
275 * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE.
276 *
277 * Wait for any pending async migration operation on the object,
278 * whether it's explicitly (i915_gem_object_migrate()) or implicitly
279 * (swapin, initial clearing) initiated.
280 *
281 * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting.
282 */
283int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj,
284 unsigned int flags)
285{
286 might_sleep();
287
288 return i915_gem_object_wait_moving_fence(obj, intr: !!(flags & I915_WAIT_INTERRUPTIBLE));
289}
290