1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6/**
7 * DOC: display pinning helpers
8 */
9
10#include "gem/i915_gem_domain.h"
11#include "gem/i915_gem_object.h"
12
13#include "i915_drv.h"
14#include "i915_vma.h"
15#include "intel_display_core.h"
16#include "intel_display_rpm.h"
17#include "intel_display_types.h"
18#include "intel_dpt.h"
19#include "intel_fb.h"
20#include "intel_fb_pin.h"
21#include "intel_plane.h"
22
23static struct i915_vma *
24intel_fb_pin_to_dpt(const struct drm_framebuffer *fb,
25 const struct i915_gtt_view *view,
26 unsigned int alignment,
27 unsigned long *out_flags,
28 struct i915_address_space *vm)
29{
30 struct drm_device *dev = fb->dev;
31 struct intel_display *display = to_intel_display(dev);
32 struct drm_i915_private *dev_priv = to_i915(dev);
33 struct drm_gem_object *_obj = intel_fb_bo(fb);
34 struct drm_i915_gem_object *obj = to_intel_bo(gem: _obj);
35 struct i915_gem_ww_ctx ww;
36 struct i915_vma *vma;
37 int ret;
38
39 /*
40 * We are not syncing against the binding (and potential migrations)
41 * below, so this vm must never be async.
42 */
43 if (drm_WARN_ON(&dev_priv->drm, vm->bind_async_flags))
44 return ERR_PTR(error: -EINVAL);
45
46 if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
47 return ERR_PTR(error: -EINVAL);
48
49 atomic_inc(v: &display->restore.pending_fb_pin);
50
51 for_i915_gem_ww(&ww, ret, true) {
52 ret = i915_gem_object_lock(obj, ww: &ww);
53 if (ret)
54 continue;
55
56 if (HAS_LMEM(dev_priv)) {
57 unsigned int flags = obj->flags;
58
59 /*
60 * For this type of buffer we need to able to read from the CPU
61 * the clear color value found in the buffer, hence we need to
62 * ensure it is always in the mappable part of lmem, if this is
63 * a small-bar device.
64 */
65 if (intel_fb_rc_ccs_cc_plane(fb) >= 0)
66 flags &= ~I915_BO_ALLOC_GPU_ONLY;
67 ret = __i915_gem_object_migrate(obj, ww: &ww, id: INTEL_REGION_LMEM_0,
68 flags);
69 if (ret)
70 continue;
71 }
72
73 ret = i915_gem_object_set_cache_level(obj, cache_level: I915_CACHE_NONE);
74 if (ret)
75 continue;
76
77 vma = i915_vma_instance(obj, vm, view);
78 if (IS_ERR(ptr: vma)) {
79 ret = PTR_ERR(ptr: vma);
80 continue;
81 }
82
83 if (i915_vma_misplaced(vma, size: 0, alignment, flags: 0)) {
84 ret = i915_vma_unbind(vma);
85 if (ret)
86 continue;
87 }
88
89 ret = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment, PIN_GLOBAL);
90 if (ret)
91 continue;
92 }
93 if (ret) {
94 vma = ERR_PTR(error: ret);
95 goto err;
96 }
97
98 vma->display_alignment = max(vma->display_alignment, alignment);
99
100 i915_gem_object_flush_if_display(obj);
101
102 i915_vma_get(vma);
103err:
104 atomic_dec(v: &display->restore.pending_fb_pin);
105
106 return vma;
107}
108
109struct i915_vma *
110intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
111 const struct i915_gtt_view *view,
112 unsigned int alignment,
113 unsigned int phys_alignment,
114 unsigned int vtd_guard,
115 bool uses_fence,
116 unsigned long *out_flags)
117{
118 struct drm_device *dev = fb->dev;
119 struct intel_display *display = to_intel_display(dev);
120 struct drm_i915_private *dev_priv = to_i915(dev);
121 struct drm_gem_object *_obj = intel_fb_bo(fb);
122 struct drm_i915_gem_object *obj = to_intel_bo(gem: _obj);
123 struct ref_tracker *wakeref;
124 struct i915_gem_ww_ctx ww;
125 struct i915_vma *vma;
126 unsigned int pinctl;
127 int ret;
128
129 if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
130 return ERR_PTR(error: -EINVAL);
131
132 if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
133 return ERR_PTR(error: -EINVAL);
134
135 /*
136 * Global gtt pte registers are special registers which actually forward
137 * writes to a chunk of system memory. Which means that there is no risk
138 * that the register values disappear as soon as we call
139 * intel_runtime_pm_put(), so it is correct to wrap only the
140 * pin/unpin/fence and not more.
141 */
142 wakeref = intel_display_rpm_get(display);
143
144 atomic_inc(v: &display->restore.pending_fb_pin);
145
146 /*
147 * Valleyview is definitely limited to scanning out the first
148 * 512MiB. Lets presume this behaviour was inherited from the
149 * g4x display engine and that all earlier gen are similarly
150 * limited. Testing suggests that it is a little more
151 * complicated than this. For example, Cherryview appears quite
152 * happy to scanout from anywhere within its global aperture.
153 */
154 pinctl = 0;
155 if (HAS_GMCH(display))
156 pinctl |= PIN_MAPPABLE;
157
158 i915_gem_ww_ctx_init(ctx: &ww, intr: true);
159retry:
160 ret = i915_gem_object_lock(obj, ww: &ww);
161 if (!ret && phys_alignment)
162 ret = i915_gem_object_attach_phys(obj, align: phys_alignment);
163 else if (!ret && HAS_LMEM(dev_priv))
164 ret = i915_gem_object_migrate(obj, ww: &ww, id: INTEL_REGION_LMEM_0);
165 if (!ret)
166 ret = i915_gem_object_pin_pages(obj);
167 if (ret)
168 goto err;
169
170 vma = i915_gem_object_pin_to_display_plane(obj, ww: &ww, alignment,
171 guard: vtd_guard, view, flags: pinctl);
172 if (IS_ERR(ptr: vma)) {
173 ret = PTR_ERR(ptr: vma);
174 goto err_unpin;
175 }
176
177 if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
178 /*
179 * Install a fence for tiled scan-out. Pre-i965 always needs a
180 * fence, whereas 965+ only requires a fence if using
181 * framebuffer compression. For simplicity, we always, when
182 * possible, install a fence as the cost is not that onerous.
183 *
184 * If we fail to fence the tiled scanout, then either the
185 * modeset will reject the change (which is highly unlikely as
186 * the affected systems, all but one, do not have unmappable
187 * space) or we will not be able to enable full powersaving
188 * techniques (also likely not to apply due to various limits
189 * FBC and the like impose on the size of the buffer, which
190 * presumably we violated anyway with this unmappable buffer).
191 * Anyway, it is presumably better to stumble onwards with
192 * something and try to run the system in a "less than optimal"
193 * mode that matches the user configuration.
194 */
195 ret = i915_vma_pin_fence(vma);
196 if (ret != 0 && DISPLAY_VER(display) < 4) {
197 i915_vma_unpin(vma);
198 goto err_unpin;
199 }
200 ret = 0;
201
202 if (vma->fence)
203 *out_flags |= PLANE_HAS_FENCE;
204 }
205
206 i915_vma_get(vma);
207
208err_unpin:
209 i915_gem_object_unpin_pages(obj);
210err:
211 if (ret == -EDEADLK) {
212 ret = i915_gem_ww_ctx_backoff(ctx: &ww);
213 if (!ret)
214 goto retry;
215 }
216 i915_gem_ww_ctx_fini(ctx: &ww);
217 if (ret)
218 vma = ERR_PTR(error: ret);
219
220 atomic_dec(v: &display->restore.pending_fb_pin);
221 intel_display_rpm_put(display, wakeref);
222 return vma;
223}
224
225void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
226{
227 if (flags & PLANE_HAS_FENCE)
228 i915_vma_unpin_fence(vma);
229 i915_vma_unpin(vma);
230 i915_vma_put(vma);
231}
232
233static unsigned int
234intel_plane_fb_min_alignment(const struct intel_plane_state *plane_state)
235{
236 const struct intel_framebuffer *fb = to_intel_framebuffer(plane_state->hw.fb);
237
238 return fb->min_alignment;
239}
240
241static unsigned int
242intel_plane_fb_min_phys_alignment(const struct intel_plane_state *plane_state)
243{
244 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
245 const struct drm_framebuffer *fb = plane_state->hw.fb;
246
247 if (!intel_plane_needs_physical(plane))
248 return 0;
249
250 return plane->min_alignment(plane, fb, 0);
251}
252
253static unsigned int
254intel_plane_fb_vtd_guard(const struct intel_plane_state *plane_state)
255{
256 return intel_fb_view_vtd_guard(fb: plane_state->hw.fb,
257 view: &plane_state->view,
258 rotation: plane_state->hw.rotation);
259}
260
261int intel_plane_pin_fb(struct intel_plane_state *plane_state,
262 const struct intel_plane_state *old_plane_state)
263{
264 struct intel_display *display = to_intel_display(plane_state);
265 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
266 const struct intel_framebuffer *fb =
267 to_intel_framebuffer(plane_state->hw.fb);
268 struct i915_vma *vma;
269
270 if (!intel_fb_uses_dpt(fb: &fb->base)) {
271 vma = intel_fb_pin_to_ggtt(fb: &fb->base, view: &plane_state->view.gtt,
272 alignment: intel_plane_fb_min_alignment(plane_state),
273 phys_alignment: intel_plane_fb_min_phys_alignment(plane_state),
274 vtd_guard: intel_plane_fb_vtd_guard(plane_state),
275 uses_fence: intel_plane_uses_fence(plane_state),
276 out_flags: &plane_state->flags);
277 if (IS_ERR(ptr: vma))
278 return PTR_ERR(ptr: vma);
279
280 plane_state->ggtt_vma = vma;
281
282 } else {
283 unsigned int alignment = intel_plane_fb_min_alignment(plane_state);
284
285 vma = intel_dpt_pin_to_ggtt(vm: fb->dpt_vm, alignment: alignment / 512);
286 if (IS_ERR(ptr: vma))
287 return PTR_ERR(ptr: vma);
288
289 plane_state->ggtt_vma = vma;
290
291 vma = intel_fb_pin_to_dpt(fb: &fb->base, view: &plane_state->view.gtt,
292 alignment, out_flags: &plane_state->flags,
293 vm: fb->dpt_vm);
294 if (IS_ERR(ptr: vma)) {
295 intel_dpt_unpin_from_ggtt(vm: fb->dpt_vm);
296 plane_state->ggtt_vma = NULL;
297 return PTR_ERR(ptr: vma);
298 }
299
300 plane_state->dpt_vma = vma;
301
302 WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma);
303
304 /*
305 * The DPT object contains only one vma, and there is no VT-d
306 * guard, so the VMA's offset within the DPT is always 0.
307 */
308 drm_WARN_ON(display->drm, intel_dpt_offset(plane_state->dpt_vma));
309 }
310
311 /*
312 * Pre-populate the dma address before we enter the vblank
313 * evade critical section as i915_gem_object_get_dma_address()
314 * will trigger might_sleep() even if it won't actually sleep,
315 * which is the case when the fb has already been pinned.
316 */
317 if (intel_plane_needs_physical(plane)) {
318 struct drm_i915_gem_object *obj = to_intel_bo(gem: intel_fb_bo(fb: &fb->base));
319
320 plane_state->surf = i915_gem_object_get_dma_address(obj, 0) +
321 plane->surf_offset(plane_state);
322 } else {
323 plane_state->surf = i915_ggtt_offset(vma: plane_state->ggtt_vma) +
324 plane->surf_offset(plane_state);
325 }
326
327 return 0;
328}
329
330void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
331{
332 const struct intel_framebuffer *fb =
333 to_intel_framebuffer(old_plane_state->hw.fb);
334 struct i915_vma *vma;
335
336 if (!intel_fb_uses_dpt(fb: &fb->base)) {
337 vma = fetch_and_zero(&old_plane_state->ggtt_vma);
338 if (vma)
339 intel_fb_unpin_vma(vma, flags: old_plane_state->flags);
340 } else {
341 vma = fetch_and_zero(&old_plane_state->dpt_vma);
342 if (vma)
343 intel_fb_unpin_vma(vma, flags: old_plane_state->flags);
344
345 vma = fetch_and_zero(&old_plane_state->ggtt_vma);
346 if (vma)
347 intel_dpt_unpin_from_ggtt(vm: fb->dpt_vm);
348 }
349}
350
351void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map)
352{
353 iosys_map_set_vaddr_iomem(map, vaddr_iomem: i915_vma_get_iomap(vma));
354}
355