1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include <drm/ttm/ttm_tt.h>
7
8#include "i915_deps.h"
9#include "i915_drv.h"
10#include "intel_memory_region.h"
11#include "intel_region_ttm.h"
12
13#include "gem/i915_gem_object.h"
14#include "gem/i915_gem_region.h"
15#include "gem/i915_gem_ttm.h"
16#include "gem/i915_gem_ttm_move.h"
17
18#include "gt/intel_engine_pm.h"
19#include "gt/intel_gt.h"
20#include "gt/intel_migrate.h"
21
22/**
23 * DOC: Selftest failure modes for failsafe migration:
24 *
25 * For fail_gpu_migration, the gpu blit scheduled is always a clear blit
26 * rather than a copy blit, and then we force the failure paths as if
27 * the blit fence returned an error.
28 *
29 * For fail_work_allocation we fail the kmalloc of the async worker, we
30 * sync the gpu blit. If it then fails, or fail_gpu_migration is set to
31 * true, then a memcpy operation is performed sync.
32 */
33#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
34static bool fail_gpu_migration;
35static bool fail_work_allocation;
36static bool ban_memcpy;
37
38void i915_ttm_migrate_set_failure_modes(bool gpu_migration,
39 bool work_allocation)
40{
41 fail_gpu_migration = gpu_migration;
42 fail_work_allocation = work_allocation;
43}
44
45void i915_ttm_migrate_set_ban_memcpy(bool ban)
46{
47 ban_memcpy = ban;
48}
49#endif
50
51static enum i915_cache_level
52i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res,
53 struct ttm_tt *ttm)
54{
55 return ((HAS_LLC(i915) || HAS_SNOOP(i915)) &&
56 !i915_ttm_gtt_binds_lmem(mem: res) &&
57 ttm->caching == ttm_cached) ? I915_CACHE_LLC :
58 I915_CACHE_NONE;
59}
60
61static struct intel_memory_region *
62i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type)
63{
64 struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev);
65
66 /* There's some room for optimization here... */
67 GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM &&
68 ttm_mem_type < I915_PL_LMEM0);
69 if (ttm_mem_type == I915_PL_SYSTEM)
70 return intel_memory_region_lookup(i915, class: INTEL_MEMORY_SYSTEM,
71 instance: 0);
72
73 return intel_memory_region_lookup(i915, class: INTEL_MEMORY_LOCAL,
74 instance: ttm_mem_type - I915_PL_LMEM0);
75}
76
77/**
78 * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a
79 * TTM move
80 * @obj: The gem object
81 */
82void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj)
83{
84 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
85
86 if (i915_ttm_cpu_maps_iomem(mem: bo->resource) || bo->ttm->caching != ttm_cached) {
87 obj->write_domain = I915_GEM_DOMAIN_WC;
88 obj->read_domains = I915_GEM_DOMAIN_WC;
89 } else {
90 obj->write_domain = I915_GEM_DOMAIN_CPU;
91 obj->read_domains = I915_GEM_DOMAIN_CPU;
92 }
93}
94
95/**
96 * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move
97 * @obj: The gem object
98 *
99 * Adjusts the GEM object's region, mem_flags and cache coherency after a
100 * TTM move.
101 */
102void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj)
103{
104 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
105 unsigned int cache_level;
106 unsigned int mem_flags;
107 unsigned int i;
108 int mem_type;
109
110 /*
111 * We might have been purged (or swapped out) if the resource is NULL,
112 * in which case the SYSTEM placement is the closest match to describe
113 * the current domain. If the object is ever used in this state then we
114 * will require moving it again.
115 */
116 if (!bo->resource) {
117 mem_flags = I915_BO_FLAG_STRUCT_PAGE;
118 mem_type = I915_PL_SYSTEM;
119 cache_level = I915_CACHE_NONE;
120 } else {
121 mem_flags = i915_ttm_cpu_maps_iomem(mem: bo->resource) ? I915_BO_FLAG_IOMEM :
122 I915_BO_FLAG_STRUCT_PAGE;
123 mem_type = bo->resource->mem_type;
124 cache_level = i915_ttm_cache_level(i915: to_i915(dev: bo->base.dev), res: bo->resource,
125 ttm: bo->ttm);
126 }
127
128 /*
129 * If object was moved to an allowable region, update the object
130 * region to consider it migrated. Note that if it's currently not
131 * in an allowable region, it's evicted and we don't update the
132 * object region.
133 */
134 if (intel_region_to_ttm_type(mem: obj->mm.region) != mem_type) {
135 for (i = 0; i < obj->mm.n_placements; ++i) {
136 struct intel_memory_region *mr = obj->mm.placements[i];
137
138 if (intel_region_to_ttm_type(mem: mr) == mem_type &&
139 mr != obj->mm.region) {
140 i915_gem_object_release_memory_region(obj);
141 i915_gem_object_init_memory_region(obj, mem: mr);
142 break;
143 }
144 }
145 }
146
147 obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM);
148 obj->mem_flags |= mem_flags;
149
150 i915_gem_object_set_cache_coherency(obj, cache_level);
151}
152
153/**
154 * i915_ttm_move_notify - Prepare an object for move
155 * @bo: The ttm buffer object.
156 *
157 * This function prepares an object for move by removing all GPU bindings,
158 * removing all CPU mappings and finally releasing the pages sg-table.
159 *
160 * Return: 0 if successful, negative error code on error.
161 */
162int i915_ttm_move_notify(struct ttm_buffer_object *bo)
163{
164 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
165 int ret;
166
167 /*
168 * Note: The async unbinding here will actually transform the
169 * blocking wait for unbind into a wait before finally submitting
170 * evict / migration blit and thus stall the migration timeline
171 * which may not be good for overall throughput. We should make
172 * sure we await the unbind fences *after* the migration blit
173 * instead of *before* as we currently do.
174 */
175 ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE |
176 I915_GEM_OBJECT_UNBIND_ASYNC);
177 if (ret)
178 return ret;
179
180 ret = __i915_gem_object_put_pages(obj);
181 if (ret)
182 return ret;
183
184 return 0;
185}
186
187static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
188 bool clear,
189 struct ttm_resource *dst_mem,
190 struct ttm_tt *dst_ttm,
191 struct sg_table *dst_st,
192 const struct i915_deps *deps)
193{
194 struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915),
195 bdev);
196 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
197 struct i915_request *rq;
198 struct ttm_tt *src_ttm = bo->ttm;
199 enum i915_cache_level src_level, dst_level;
200 int ret;
201
202 if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(gt: to_gt(i915)))
203 return ERR_PTR(error: -EINVAL);
204
205 /* With fail_gpu_migration, we always perform a GPU clear. */
206 if (I915_SELFTEST_ONLY(fail_gpu_migration))
207 clear = true;
208
209 dst_level = i915_ttm_cache_level(i915, res: dst_mem, ttm: dst_ttm);
210 if (clear) {
211 if (bo->type == ttm_bo_type_kernel &&
212 !I915_SELFTEST_ONLY(fail_gpu_migration))
213 return ERR_PTR(error: -EINVAL);
214
215 intel_engine_pm_get(engine: to_gt(i915)->migrate.context->engine);
216 ret = intel_context_migrate_clear(ce: to_gt(i915)->migrate.context, deps,
217 sg: dst_st->sgl,
218 pat_index: i915_gem_get_pat_index(i915, level: dst_level),
219 is_lmem: i915_ttm_gtt_binds_lmem(mem: dst_mem),
220 value: 0, out: &rq);
221 } else {
222 struct i915_refct_sgt *src_rsgt =
223 i915_ttm_resource_get_st(obj, res: bo->resource);
224
225 if (IS_ERR(ptr: src_rsgt))
226 return ERR_CAST(ptr: src_rsgt);
227
228 src_level = i915_ttm_cache_level(i915, res: bo->resource, ttm: src_ttm);
229 intel_engine_pm_get(engine: to_gt(i915)->migrate.context->engine);
230 ret = intel_context_migrate_copy(ce: to_gt(i915)->migrate.context,
231 deps, src: src_rsgt->table.sgl,
232 src_pat_index: i915_gem_get_pat_index(i915, level: src_level),
233 src_is_lmem: i915_ttm_gtt_binds_lmem(mem: bo->resource),
234 dst: dst_st->sgl,
235 dst_pat_index: i915_gem_get_pat_index(i915, level: dst_level),
236 dst_is_lmem: i915_ttm_gtt_binds_lmem(mem: dst_mem),
237 out: &rq);
238
239 i915_refct_sgt_put(rsgt: src_rsgt);
240 }
241
242 intel_engine_pm_put(engine: to_gt(i915)->migrate.context->engine);
243
244 if (ret && rq) {
245 i915_request_wait(rq, flags: 0, MAX_SCHEDULE_TIMEOUT);
246 i915_request_put(rq);
247 }
248
249 return ret ? ERR_PTR(error: ret) : &rq->fence;
250}
251
252/**
253 * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality.
254 * @_dst_iter: Storage space for the destination kmap iterator.
255 * @_src_iter: Storage space for the source kmap iterator.
256 * @dst_iter: Pointer to the destination kmap iterator.
257 * @src_iter: Pointer to the source kmap iterator.
258 * @num_pages: Number of pages
259 * @clear: Whether to clear instead of copy.
260 * @src_rsgt: Refcounted scatter-gather list of source memory.
261 * @dst_rsgt: Refcounted scatter-gather list of destination memory.
262 */
263struct i915_ttm_memcpy_arg {
264 union {
265 struct ttm_kmap_iter_tt tt;
266 struct ttm_kmap_iter_iomap io;
267 } _dst_iter,
268 _src_iter;
269 struct ttm_kmap_iter *dst_iter;
270 struct ttm_kmap_iter *src_iter;
271 unsigned long num_pages;
272 bool clear;
273 struct i915_refct_sgt *src_rsgt;
274 struct i915_refct_sgt *dst_rsgt;
275};
276
277/**
278 * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence.
279 * @fence: The dma-fence.
280 * @work: The work struct use for the memcpy work.
281 * @lock: The fence lock. Not used to protect anything else ATM.
282 * @irq_work: Low latency worker to signal the fence since it can't be done
283 * from the callback for lockdep reasons.
284 * @cb: Callback for the accelerated migration fence.
285 * @arg: The argument for the memcpy functionality.
286 * @i915: The i915 pointer.
287 * @obj: The GEM object.
288 * @memcpy_allowed: Instead of processing the @arg, and falling back to memcpy
289 * or memset, we wedge the device and set the @obj unknown_state, to prevent
290 * further access to the object with the CPU or GPU. On some devices we might
291 * only be permitted to use the blitter engine for such operations.
292 */
293struct i915_ttm_memcpy_work {
294 struct dma_fence fence;
295 struct work_struct work;
296 spinlock_t lock;
297 struct irq_work irq_work;
298 struct dma_fence_cb cb;
299 struct i915_ttm_memcpy_arg arg;
300 struct drm_i915_private *i915;
301 struct drm_i915_gem_object *obj;
302 bool memcpy_allowed;
303};
304
305static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg)
306{
307 ttm_move_memcpy(clear: arg->clear, num_pages: arg->num_pages,
308 dst_iter: arg->dst_iter, src_iter: arg->src_iter);
309}
310
311static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg,
312 struct ttm_buffer_object *bo, bool clear,
313 struct ttm_resource *dst_mem,
314 struct ttm_tt *dst_ttm,
315 struct i915_refct_sgt *dst_rsgt)
316{
317 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
318 struct intel_memory_region *dst_reg, *src_reg;
319
320 dst_reg = i915_ttm_region(bdev: bo->bdev, ttm_mem_type: dst_mem->mem_type);
321 src_reg = i915_ttm_region(bdev: bo->bdev, ttm_mem_type: bo->resource->mem_type);
322 GEM_BUG_ON(!dst_reg || !src_reg);
323
324 arg->dst_iter = !i915_ttm_cpu_maps_iomem(mem: dst_mem) ?
325 ttm_kmap_iter_tt_init(iter_tt: &arg->_dst_iter.tt, tt: dst_ttm) :
326 ttm_kmap_iter_iomap_init(iter_io: &arg->_dst_iter.io, iomap: &dst_reg->iomap,
327 st: &dst_rsgt->table, start: dst_reg->region.start);
328
329 arg->src_iter = !i915_ttm_cpu_maps_iomem(mem: bo->resource) ?
330 ttm_kmap_iter_tt_init(iter_tt: &arg->_src_iter.tt, tt: bo->ttm) :
331 ttm_kmap_iter_iomap_init(iter_io: &arg->_src_iter.io, iomap: &src_reg->iomap,
332 st: &obj->ttm.cached_io_rsgt->table,
333 start: src_reg->region.start);
334 arg->clear = clear;
335 arg->num_pages = bo->base.size >> PAGE_SHIFT;
336
337 arg->dst_rsgt = i915_refct_sgt_get(rsgt: dst_rsgt);
338 arg->src_rsgt = clear ? NULL :
339 i915_ttm_resource_get_st(obj, res: bo->resource);
340}
341
342static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg)
343{
344 i915_refct_sgt_put(rsgt: arg->src_rsgt);
345 i915_refct_sgt_put(rsgt: arg->dst_rsgt);
346}
347
348static void __memcpy_work(struct work_struct *work)
349{
350 struct i915_ttm_memcpy_work *copy_work =
351 container_of(work, typeof(*copy_work), work);
352 struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
353 bool cookie;
354
355 /*
356 * FIXME: We need to take a closer look here. We should be able to plonk
357 * this into the fence critical section.
358 */
359 if (!copy_work->memcpy_allowed) {
360 struct intel_gt *gt;
361 unsigned int id;
362
363 for_each_gt(gt, copy_work->i915, id)
364 intel_gt_set_wedged(gt);
365 }
366
367 cookie = dma_fence_begin_signalling();
368
369 if (copy_work->memcpy_allowed) {
370 i915_ttm_move_memcpy(arg);
371 } else {
372 /*
373 * Prevent further use of the object. Any future GTT binding or
374 * CPU access is not allowed once we signal the fence. Outside
375 * of the fence critical section, we then also then wedge the gpu
376 * to indicate the device is not functional.
377 *
378 * The below dma_fence_signal() is our write-memory-barrier.
379 */
380 copy_work->obj->mm.unknown_state = true;
381 }
382
383 dma_fence_end_signalling(cookie);
384
385 dma_fence_signal(fence: &copy_work->fence);
386
387 i915_ttm_memcpy_release(arg);
388 i915_gem_object_put(obj: copy_work->obj);
389 dma_fence_put(fence: &copy_work->fence);
390}
391
392static void __memcpy_irq_work(struct irq_work *irq_work)
393{
394 struct i915_ttm_memcpy_work *copy_work =
395 container_of(irq_work, typeof(*copy_work), irq_work);
396 struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
397
398 dma_fence_signal(fence: &copy_work->fence);
399 i915_ttm_memcpy_release(arg);
400 i915_gem_object_put(obj: copy_work->obj);
401 dma_fence_put(fence: &copy_work->fence);
402}
403
404static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
405{
406 struct i915_ttm_memcpy_work *copy_work =
407 container_of(cb, typeof(*copy_work), cb);
408
409 if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) {
410 INIT_WORK(&copy_work->work, __memcpy_work);
411 queue_work(wq: system_unbound_wq, work: &copy_work->work);
412 } else {
413 init_irq_work(work: &copy_work->irq_work, func: __memcpy_irq_work);
414 irq_work_queue(work: &copy_work->irq_work);
415 }
416}
417
418static const char *get_driver_name(struct dma_fence *fence)
419{
420 return "i915_ttm_memcpy_work";
421}
422
423static const char *get_timeline_name(struct dma_fence *fence)
424{
425 return "unbound";
426}
427
428static const struct dma_fence_ops dma_fence_memcpy_ops = {
429 .get_driver_name = get_driver_name,
430 .get_timeline_name = get_timeline_name,
431};
432
433static struct dma_fence *
434i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work,
435 struct dma_fence *dep)
436{
437 int ret;
438
439 spin_lock_init(&work->lock);
440 dma_fence_init(fence: &work->fence, ops: &dma_fence_memcpy_ops, lock: &work->lock, context: 0, seqno: 0);
441 dma_fence_get(fence: &work->fence);
442 ret = dma_fence_add_callback(fence: dep, cb: &work->cb, func: __memcpy_cb);
443 if (ret) {
444 if (ret != -ENOENT)
445 dma_fence_wait(fence: dep, intr: false);
446
447 return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL :
448 dep->error);
449 }
450
451 return &work->fence;
452}
453
454static bool i915_ttm_memcpy_allowed(struct ttm_buffer_object *bo,
455 struct ttm_resource *dst_mem)
456{
457 if (i915_gem_object_needs_ccs_pages(obj: i915_ttm_to_gem(bo)))
458 return false;
459
460 if (!(i915_ttm_resource_mappable(res: bo->resource) &&
461 i915_ttm_resource_mappable(res: dst_mem)))
462 return false;
463
464 return I915_SELFTEST_ONLY(ban_memcpy) ? false : true;
465}
466
467static struct dma_fence *
468__i915_ttm_move(struct ttm_buffer_object *bo,
469 const struct ttm_operation_ctx *ctx, bool clear,
470 struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm,
471 struct i915_refct_sgt *dst_rsgt, bool allow_accel,
472 const struct i915_deps *move_deps)
473{
474 const bool memcpy_allowed = i915_ttm_memcpy_allowed(bo, dst_mem);
475 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
476 struct drm_i915_private *i915 = to_i915(dev: bo->base.dev);
477 struct i915_ttm_memcpy_work *copy_work = NULL;
478 struct i915_ttm_memcpy_arg _arg, *arg = &_arg;
479 struct dma_fence *fence = ERR_PTR(error: -EINVAL);
480
481 if (allow_accel) {
482 fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm,
483 dst_st: &dst_rsgt->table, deps: move_deps);
484
485 /*
486 * We only need to intercept the error when moving to lmem.
487 * When moving to system, TTM or shmem will provide us with
488 * cleared pages.
489 */
490 if (!IS_ERR(ptr: fence) && !i915_ttm_gtt_binds_lmem(mem: dst_mem) &&
491 !I915_SELFTEST_ONLY(fail_gpu_migration ||
492 fail_work_allocation))
493 goto out;
494 }
495
496 /* If we've scheduled gpu migration. Try to arm error intercept. */
497 if (!IS_ERR(ptr: fence)) {
498 struct dma_fence *dep = fence;
499
500 if (!I915_SELFTEST_ONLY(fail_work_allocation))
501 copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL);
502
503 if (copy_work) {
504 copy_work->i915 = i915;
505 copy_work->memcpy_allowed = memcpy_allowed;
506 copy_work->obj = i915_gem_object_get(obj);
507 arg = &copy_work->arg;
508 if (memcpy_allowed)
509 i915_ttm_memcpy_init(arg, bo, clear, dst_mem,
510 dst_ttm, dst_rsgt);
511
512 fence = i915_ttm_memcpy_work_arm(work: copy_work, dep);
513 } else {
514 dma_fence_wait(fence: dep, intr: false);
515 fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ?
516 -EINVAL : fence->error);
517 }
518 dma_fence_put(fence: dep);
519
520 if (!IS_ERR(ptr: fence))
521 goto out;
522 } else {
523 int err = PTR_ERR(ptr: fence);
524
525 if (err == -EINTR || err == -ERESTARTSYS || err == -EAGAIN)
526 return fence;
527
528 if (move_deps) {
529 err = i915_deps_sync(deps: move_deps, ctx);
530 if (err)
531 return ERR_PTR(error: err);
532 }
533 }
534
535 /* Error intercept failed or no accelerated migration to start with */
536
537 if (memcpy_allowed) {
538 if (!copy_work)
539 i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm,
540 dst_rsgt);
541 i915_ttm_move_memcpy(arg);
542 i915_ttm_memcpy_release(arg);
543 }
544 if (copy_work)
545 i915_gem_object_put(obj: copy_work->obj);
546 kfree(objp: copy_work);
547
548 return memcpy_allowed ? NULL : ERR_PTR(error: -EIO);
549out:
550 if (!fence && copy_work) {
551 i915_ttm_memcpy_release(arg);
552 i915_gem_object_put(obj: copy_work->obj);
553 kfree(objp: copy_work);
554 }
555
556 return fence;
557}
558
559/**
560 * i915_ttm_move - The TTM move callback used by i915.
561 * @bo: The buffer object.
562 * @evict: Whether this is an eviction.
563 * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits should be
564 * performed if waiting
565 * @dst_mem: The destination ttm resource.
566 * @hop: If we need multihop, what temporary memory type to move to.
567 *
568 * Return: 0 if successful, negative error code otherwise.
569 */
570int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,
571 struct ttm_operation_ctx *ctx,
572 struct ttm_resource *dst_mem,
573 struct ttm_place *hop)
574{
575 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
576 struct ttm_resource_manager *dst_man =
577 ttm_manager_type(bdev: bo->bdev, mem_type: dst_mem->mem_type);
578 struct dma_fence *migration_fence = NULL;
579 struct ttm_tt *ttm = bo->ttm;
580 struct i915_refct_sgt *dst_rsgt;
581 bool clear, prealloc_bo;
582 int ret;
583
584 if (GEM_WARN_ON(i915_ttm_is_ghost_object(bo))) {
585 ttm_bo_move_null(bo, new_mem: dst_mem);
586 return 0;
587 }
588
589 if (!bo->resource) {
590 if (dst_mem->mem_type != TTM_PL_SYSTEM) {
591 hop->mem_type = TTM_PL_SYSTEM;
592 hop->flags = TTM_PL_FLAG_TEMPORARY;
593 return -EMULTIHOP;
594 }
595
596 /*
597 * This is only reached when first creating the object, or if
598 * the object was purged or swapped out (pipeline-gutting). For
599 * the former we can safely skip all of the below since we are
600 * only using a dummy SYSTEM placement here. And with the latter
601 * we will always re-enter here with bo->resource set correctly
602 * (as per the above), since this is part of a multi-hop
603 * sequence, where at the end we can do the move for real.
604 *
605 * The special case here is when the dst_mem is TTM_PL_SYSTEM,
606 * which doesn't require any kind of move, so it should be safe
607 * to skip all the below and call ttm_bo_move_null() here, where
608 * the caller in __i915_ttm_get_pages() will take care of the
609 * rest, since we should have a valid ttm_tt.
610 */
611 ttm_bo_move_null(bo, new_mem: dst_mem);
612 return 0;
613 }
614
615 ret = i915_ttm_move_notify(bo);
616 if (ret)
617 return ret;
618
619 if (obj->mm.madv != I915_MADV_WILLNEED) {
620 i915_ttm_purge(obj);
621 ttm_resource_free(bo, res: &dst_mem);
622 return 0;
623 }
624
625 /* Populate ttm with pages if needed. Typically system memory. */
626 if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) {
627 ret = ttm_bo_populate(bo, ctx);
628 if (ret)
629 return ret;
630 }
631
632 dst_rsgt = i915_ttm_resource_get_st(obj, res: dst_mem);
633 if (IS_ERR(ptr: dst_rsgt))
634 return PTR_ERR(ptr: dst_rsgt);
635
636 clear = !i915_ttm_cpu_maps_iomem(mem: bo->resource) && (!ttm || !ttm_tt_is_populated(tt: ttm));
637 prealloc_bo = obj->flags & I915_BO_PREALLOC;
638 if (!(clear && ttm && !((ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) && !prealloc_bo))) {
639 struct i915_deps deps;
640
641 i915_deps_init(deps: &deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
642 ret = i915_deps_add_resv(deps: &deps, resv: bo->base.resv, ctx);
643 if (ret) {
644 i915_refct_sgt_put(rsgt: dst_rsgt);
645 return ret;
646 }
647
648 migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, dst_ttm: ttm,
649 dst_rsgt, allow_accel: true, move_deps: &deps);
650 i915_deps_fini(deps: &deps);
651 }
652
653 /* We can possibly get an -ERESTARTSYS here */
654 if (IS_ERR(ptr: migration_fence)) {
655 i915_refct_sgt_put(rsgt: dst_rsgt);
656 return PTR_ERR(ptr: migration_fence);
657 }
658
659 if (migration_fence) {
660 if (I915_SELFTEST_ONLY(evict && fail_gpu_migration))
661 ret = -EIO; /* never feed non-migrate fences into ttm */
662 else
663 ret = ttm_bo_move_accel_cleanup(bo, fence: migration_fence, evict,
664 pipeline: true, new_mem: dst_mem);
665 if (ret) {
666 dma_fence_wait(fence: migration_fence, intr: false);
667 ttm_bo_move_sync_cleanup(bo, new_mem: dst_mem);
668 }
669 dma_fence_put(fence: migration_fence);
670 } else {
671 ttm_bo_move_sync_cleanup(bo, new_mem: dst_mem);
672 }
673
674 i915_ttm_adjust_domains_after_move(obj);
675 i915_ttm_free_cached_io_rsgt(obj);
676
677 if (i915_ttm_gtt_binds_lmem(mem: dst_mem) || i915_ttm_cpu_maps_iomem(mem: dst_mem)) {
678 obj->ttm.cached_io_rsgt = dst_rsgt;
679 obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl;
680 obj->ttm.get_io_page.sg_idx = 0;
681 } else {
682 i915_refct_sgt_put(rsgt: dst_rsgt);
683 }
684
685 i915_ttm_adjust_lru(obj);
686 i915_ttm_adjust_gem_after_move(obj);
687 return 0;
688}
689
690/**
691 * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to
692 * another
693 * @dst: The destination object
694 * @src: The source object
695 * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used.
696 * @intr: Whether to perform waits interruptible:
697 *
698 * Note: The caller is responsible for assuring that the underlying
699 * TTM objects are populated if needed and locked.
700 *
701 * Return: Zero on success. Negative error code on error. If @intr == true,
702 * then it may return -ERESTARTSYS or -EINTR.
703 */
704int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
705 struct drm_i915_gem_object *src,
706 bool allow_accel, bool intr)
707{
708 struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(obj: dst);
709 struct ttm_buffer_object *src_bo = i915_gem_to_ttm(obj: src);
710 struct ttm_operation_ctx ctx = {
711 .interruptible = intr,
712 };
713 struct i915_refct_sgt *dst_rsgt;
714 struct dma_fence *copy_fence;
715 struct i915_deps deps;
716 int ret;
717
718 assert_object_held(dst);
719 assert_object_held(src);
720
721 if (GEM_WARN_ON(!src_bo->resource || !dst_bo->resource))
722 return -EINVAL;
723
724 i915_deps_init(deps: &deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
725
726 ret = dma_resv_reserve_fences(obj: src_bo->base.resv, num_fences: 1);
727 if (ret)
728 return ret;
729
730 ret = dma_resv_reserve_fences(obj: dst_bo->base.resv, num_fences: 1);
731 if (ret)
732 return ret;
733
734 ret = i915_deps_add_resv(deps: &deps, resv: dst_bo->base.resv, ctx: &ctx);
735 if (ret)
736 return ret;
737
738 ret = i915_deps_add_resv(deps: &deps, resv: src_bo->base.resv, ctx: &ctx);
739 if (ret)
740 return ret;
741
742 dst_rsgt = i915_ttm_resource_get_st(obj: dst, res: dst_bo->resource);
743 copy_fence = __i915_ttm_move(bo: src_bo, ctx: &ctx, clear: false, dst_mem: dst_bo->resource,
744 dst_ttm: dst_bo->ttm, dst_rsgt, allow_accel,
745 move_deps: &deps);
746
747 i915_deps_fini(deps: &deps);
748 i915_refct_sgt_put(rsgt: dst_rsgt);
749 if (IS_ERR_OR_NULL(ptr: copy_fence))
750 return PTR_ERR_OR_ZERO(ptr: copy_fence);
751
752 dma_resv_add_fence(obj: dst_bo->base.resv, fence: copy_fence, usage: DMA_RESV_USAGE_WRITE);
753 dma_resv_add_fence(obj: src_bo->base.resv, fence: copy_fence, usage: DMA_RESV_USAGE_READ);
754 dma_fence_put(fence: copy_fence);
755
756 return 0;
757}
758