1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/**************************************************************************
3 *
4 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28/*
29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30 */
31
32#include <linux/export.h>
33#include <linux/swap.h>
34#include <linux/vmalloc.h>
35
36#include <drm/ttm/ttm_bo.h>
37#include <drm/ttm/ttm_placement.h>
38#include <drm/ttm/ttm_tt.h>
39
40#include <drm/drm_cache.h>
41
42#include "ttm_bo_internal.h"
43
44struct ttm_transfer_obj {
45 struct ttm_buffer_object base;
46 struct ttm_buffer_object *bo;
47};
48
49int ttm_mem_io_reserve(struct ttm_device *bdev,
50 struct ttm_resource *mem)
51{
52 if (mem->bus.offset || mem->bus.addr)
53 return 0;
54
55 mem->bus.is_iomem = false;
56 if (!bdev->funcs->io_mem_reserve)
57 return 0;
58
59 return bdev->funcs->io_mem_reserve(bdev, mem);
60}
61
62void ttm_mem_io_free(struct ttm_device *bdev,
63 struct ttm_resource *mem)
64{
65 if (!mem)
66 return;
67
68 if (!mem->bus.offset && !mem->bus.addr)
69 return;
70
71 if (bdev->funcs->io_mem_free)
72 bdev->funcs->io_mem_free(bdev, mem);
73
74 mem->bus.offset = 0;
75 mem->bus.addr = NULL;
76}
77
78/**
79 * ttm_move_memcpy - Helper to perform a memcpy ttm move operation.
80 * @clear: Whether to clear rather than copy.
81 * @num_pages: Number of pages of the operation.
82 * @dst_iter: A struct ttm_kmap_iter representing the destination resource.
83 * @src_iter: A struct ttm_kmap_iter representing the source resource.
84 *
85 * This function is intended to be able to move out async under a
86 * dma-fence if desired.
87 */
88void ttm_move_memcpy(bool clear,
89 u32 num_pages,
90 struct ttm_kmap_iter *dst_iter,
91 struct ttm_kmap_iter *src_iter)
92{
93 const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops;
94 const struct ttm_kmap_iter_ops *src_ops = src_iter->ops;
95 struct iosys_map src_map, dst_map;
96 pgoff_t i;
97
98 /* Single TTM move. NOP */
99 if (dst_ops->maps_tt && src_ops->maps_tt)
100 return;
101
102 /* Don't move nonexistent data. Clear destination instead. */
103 if (clear) {
104 for (i = 0; i < num_pages; ++i) {
105 dst_ops->map_local(dst_iter, &dst_map, i);
106 if (dst_map.is_iomem)
107 memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE);
108 else
109 memset(s: dst_map.vaddr, c: 0, PAGE_SIZE);
110 if (dst_ops->unmap_local)
111 dst_ops->unmap_local(dst_iter, &dst_map);
112 }
113 return;
114 }
115
116 for (i = 0; i < num_pages; ++i) {
117 dst_ops->map_local(dst_iter, &dst_map, i);
118 src_ops->map_local(src_iter, &src_map, i);
119
120 drm_memcpy_from_wc(dst: &dst_map, src: &src_map, PAGE_SIZE);
121
122 if (src_ops->unmap_local)
123 src_ops->unmap_local(src_iter, &src_map);
124 if (dst_ops->unmap_local)
125 dst_ops->unmap_local(dst_iter, &dst_map);
126 }
127}
128EXPORT_SYMBOL(ttm_move_memcpy);
129
130/**
131 * ttm_bo_move_memcpy
132 *
133 * @bo: A pointer to a struct ttm_buffer_object.
134 * @ctx: operation context
135 * @dst_mem: struct ttm_resource indicating where to move.
136 *
137 * Fallback move function for a mappable buffer object in mappable memory.
138 * The function will, if successful,
139 * free any old aperture space, and set (@new_mem)->mm_node to NULL,
140 * and update the (@bo)->mem placement flags. If unsuccessful, the old
141 * data remains untouched, and it's up to the caller to free the
142 * memory space indicated by @new_mem.
143 * Returns:
144 * !0: Failure.
145 */
146int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
147 struct ttm_operation_ctx *ctx,
148 struct ttm_resource *dst_mem)
149{
150 struct ttm_device *bdev = bo->bdev;
151 struct ttm_resource_manager *dst_man =
152 ttm_manager_type(bdev: bo->bdev, mem_type: dst_mem->mem_type);
153 struct ttm_tt *ttm = bo->ttm;
154 struct ttm_resource *src_mem = bo->resource;
155 struct ttm_resource_manager *src_man;
156 union {
157 struct ttm_kmap_iter_tt tt;
158 struct ttm_kmap_iter_linear_io io;
159 } _dst_iter, _src_iter;
160 struct ttm_kmap_iter *dst_iter, *src_iter;
161 bool clear;
162 int ret = 0;
163
164 if (WARN_ON(!src_mem))
165 return -EINVAL;
166
167 src_man = ttm_manager_type(bdev, mem_type: src_mem->mem_type);
168 if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) ||
169 dst_man->use_tt)) {
170 ret = ttm_bo_populate(bo, ctx);
171 if (ret)
172 return ret;
173 }
174
175 dst_iter = ttm_kmap_iter_linear_io_init(iter_io: &_dst_iter.io, bdev, mem: dst_mem);
176 if (PTR_ERR(ptr: dst_iter) == -EINVAL && dst_man->use_tt)
177 dst_iter = ttm_kmap_iter_tt_init(iter_tt: &_dst_iter.tt, tt: bo->ttm);
178 if (IS_ERR(ptr: dst_iter))
179 return PTR_ERR(ptr: dst_iter);
180
181 src_iter = ttm_kmap_iter_linear_io_init(iter_io: &_src_iter.io, bdev, mem: src_mem);
182 if (PTR_ERR(ptr: src_iter) == -EINVAL && src_man->use_tt)
183 src_iter = ttm_kmap_iter_tt_init(iter_tt: &_src_iter.tt, tt: bo->ttm);
184 if (IS_ERR(ptr: src_iter)) {
185 ret = PTR_ERR(ptr: src_iter);
186 goto out_src_iter;
187 }
188
189 clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(tt: ttm));
190 if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC)))
191 ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter);
192
193 if (!src_iter->ops->maps_tt)
194 ttm_kmap_iter_linear_io_fini(iter_io: &_src_iter.io, bdev, mem: src_mem);
195 ttm_bo_move_sync_cleanup(bo, new_mem: dst_mem);
196
197out_src_iter:
198 if (!dst_iter->ops->maps_tt)
199 ttm_kmap_iter_linear_io_fini(iter_io: &_dst_iter.io, bdev, mem: dst_mem);
200
201 return ret;
202}
203EXPORT_SYMBOL(ttm_bo_move_memcpy);
204
205static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
206{
207 struct ttm_transfer_obj *fbo;
208
209 fbo = container_of(bo, struct ttm_transfer_obj, base);
210 dma_resv_fini(obj: &fbo->base.base._resv);
211 ttm_bo_put(bo: fbo->bo);
212 kfree(objp: fbo);
213}
214
215/**
216 * ttm_buffer_object_transfer
217 *
218 * @bo: A pointer to a struct ttm_buffer_object.
219 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
220 * holding the data of @bo with the old placement.
221 *
222 * This is a utility function that may be called after an accelerated move
223 * has been scheduled. A new buffer object is created as a placeholder for
224 * the old data while it's being copied. When that buffer object is idle,
225 * it can be destroyed, releasing the space of the old placement.
226 * Returns:
227 * !0: Failure.
228 */
229
230static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
231 struct ttm_buffer_object **new_obj)
232{
233 struct ttm_transfer_obj *fbo;
234 int ret;
235
236 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
237 if (!fbo)
238 return -ENOMEM;
239
240 fbo->base = *bo;
241
242 /**
243 * Fix up members that we shouldn't copy directly:
244 * TODO: Explicit member copy would probably be better here.
245 */
246
247 atomic_inc(v: &ttm_glob.bo_count);
248 drm_vma_node_reset(node: &fbo->base.base.vma_node);
249
250 kref_init(kref: &fbo->base.kref);
251 fbo->base.destroy = &ttm_transfered_destroy;
252 fbo->base.pin_count = 0;
253 if (bo->type != ttm_bo_type_sg)
254 fbo->base.base.resv = &fbo->base.base._resv;
255
256 dma_resv_init(obj: &fbo->base.base._resv);
257 fbo->base.base.dev = NULL;
258 ret = dma_resv_trylock(obj: &fbo->base.base._resv);
259 WARN_ON(!ret);
260
261 ret = dma_resv_reserve_fences(obj: &fbo->base.base._resv, num_fences: 1);
262 if (ret) {
263 dma_resv_unlock(obj: &fbo->base.base._resv);
264 kfree(objp: fbo);
265 return ret;
266 }
267
268 if (fbo->base.resource) {
269 ttm_resource_set_bo(res: fbo->base.resource, bo: &fbo->base);
270 bo->resource = NULL;
271 ttm_bo_set_bulk_move(bo: &fbo->base, NULL);
272 } else {
273 fbo->base.bulk_move = NULL;
274 }
275
276 ttm_bo_get(bo);
277 fbo->bo = bo;
278
279 ttm_bo_move_to_lru_tail_unlocked(bo: &fbo->base);
280
281 *new_obj = &fbo->base;
282 return 0;
283}
284
285/**
286 * ttm_io_prot
287 *
288 * @bo: ttm buffer object
289 * @res: ttm resource object
290 * @tmp: Page protection flag for a normal, cached mapping.
291 *
292 * Utility function that returns the pgprot_t that should be used for
293 * setting up a PTE with the caching model indicated by @c_state.
294 */
295pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res,
296 pgprot_t tmp)
297{
298 struct ttm_resource_manager *man;
299 enum ttm_caching caching;
300
301 man = ttm_manager_type(bdev: bo->bdev, mem_type: res->mem_type);
302 if (man->use_tt) {
303 caching = bo->ttm->caching;
304 if (bo->ttm->page_flags & TTM_TT_FLAG_DECRYPTED)
305 tmp = pgprot_decrypted(tmp);
306 } else {
307 caching = res->bus.caching;
308 }
309
310 return ttm_prot_from_caching(caching, tmp);
311}
312EXPORT_SYMBOL(ttm_io_prot);
313
314static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
315 unsigned long offset,
316 unsigned long size,
317 struct ttm_bo_kmap_obj *map)
318{
319 struct ttm_resource *mem = bo->resource;
320
321 if (bo->resource->bus.addr) {
322 map->bo_kmap_type = ttm_bo_map_premapped;
323 map->virtual = ((u8 *)bo->resource->bus.addr) + offset;
324 } else {
325 resource_size_t res = bo->resource->bus.offset + offset;
326
327 map->bo_kmap_type = ttm_bo_map_iomap;
328 if (mem->bus.caching == ttm_write_combined)
329 map->virtual = ioremap_wc(offset: res, size);
330#ifdef CONFIG_X86
331 else if (mem->bus.caching == ttm_cached)
332 map->virtual = ioremap_cache(offset: res, size);
333#endif
334 else
335 map->virtual = ioremap(offset: res, size);
336 }
337 return (!map->virtual) ? -ENOMEM : 0;
338}
339
340static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
341 unsigned long start_page,
342 unsigned long num_pages,
343 struct ttm_bo_kmap_obj *map)
344{
345 struct ttm_resource *mem = bo->resource;
346 struct ttm_operation_ctx ctx = {
347 .interruptible = false,
348 .no_wait_gpu = false
349 };
350 struct ttm_tt *ttm = bo->ttm;
351 struct ttm_resource_manager *man =
352 ttm_manager_type(bdev: bo->bdev, mem_type: bo->resource->mem_type);
353 pgprot_t prot;
354 int ret;
355
356 BUG_ON(!ttm);
357
358 ret = ttm_bo_populate(bo, ctx: &ctx);
359 if (ret)
360 return ret;
361
362 if (num_pages == 1 && ttm->caching == ttm_cached &&
363 !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) {
364 /*
365 * We're mapping a single page, and the desired
366 * page protection is consistent with the bo.
367 */
368
369 map->bo_kmap_type = ttm_bo_map_kmap;
370 map->page = ttm->pages[start_page];
371 map->virtual = kmap(page: map->page);
372 } else {
373 /*
374 * We need to use vmap to get the desired page protection
375 * or to make the buffer object look contiguous.
376 */
377 prot = ttm_io_prot(bo, mem, PAGE_KERNEL);
378 map->bo_kmap_type = ttm_bo_map_vmap;
379 map->virtual = vmap(pages: ttm->pages + start_page, count: num_pages,
380 flags: 0, prot);
381 }
382 return (!map->virtual) ? -ENOMEM : 0;
383}
384
385/**
386 * ttm_bo_kmap_try_from_panic
387 *
388 * @bo: The buffer object
389 * @page: The page to map
390 *
391 * Sets up a kernel virtual mapping using kmap_local_page_try_from_panic().
392 * This should only be called from the panic handler, if you make sure the bo
393 * is the one being displayed, so is properly allocated, and protected.
394 *
395 * Returns the vaddr, that you can use to write to the bo, and that you should
396 * pass to kunmap_local() when you're done with this page, or NULL if the bo
397 * is in iomem.
398 */
399void *ttm_bo_kmap_try_from_panic(struct ttm_buffer_object *bo, unsigned long page)
400{
401 if (page + 1 > PFN_UP(bo->resource->size))
402 return NULL;
403
404 if (!bo->resource->bus.is_iomem && bo->ttm->pages && bo->ttm->pages[page])
405 return kmap_local_page_try_from_panic(page: bo->ttm->pages[page]);
406
407 return NULL;
408}
409EXPORT_SYMBOL(ttm_bo_kmap_try_from_panic);
410
411/**
412 * ttm_bo_kmap
413 *
414 * @bo: The buffer object.
415 * @start_page: The first page to map.
416 * @num_pages: Number of pages to map.
417 * @map: pointer to a struct ttm_bo_kmap_obj representing the map.
418 *
419 * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the
420 * data in the buffer object. The ttm_kmap_obj_virtual function can then be
421 * used to obtain a virtual address to the data.
422 *
423 * Returns
424 * -ENOMEM: Out of memory.
425 * -EINVAL: Invalid range.
426 */
427int ttm_bo_kmap(struct ttm_buffer_object *bo,
428 unsigned long start_page, unsigned long num_pages,
429 struct ttm_bo_kmap_obj *map)
430{
431 unsigned long offset, size;
432 int ret;
433
434 map->virtual = NULL;
435 map->bo = bo;
436 if (num_pages > PFN_UP(bo->resource->size))
437 return -EINVAL;
438 if ((start_page + num_pages) > PFN_UP(bo->resource->size))
439 return -EINVAL;
440
441 ret = ttm_mem_io_reserve(bdev: bo->bdev, mem: bo->resource);
442 if (ret)
443 return ret;
444 if (!bo->resource->bus.is_iomem) {
445 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
446 } else {
447 offset = start_page << PAGE_SHIFT;
448 size = num_pages << PAGE_SHIFT;
449 return ttm_bo_ioremap(bo, offset, size, map);
450 }
451}
452EXPORT_SYMBOL(ttm_bo_kmap);
453
454/**
455 * ttm_bo_kunmap
456 *
457 * @map: Object describing the map to unmap.
458 *
459 * Unmaps a kernel map set up by ttm_bo_kmap.
460 */
461void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
462{
463 if (!map->virtual)
464 return;
465 switch (map->bo_kmap_type) {
466 case ttm_bo_map_iomap:
467 iounmap(addr: map->virtual);
468 break;
469 case ttm_bo_map_vmap:
470 vunmap(addr: map->virtual);
471 break;
472 case ttm_bo_map_kmap:
473 kunmap(page: map->page);
474 break;
475 case ttm_bo_map_premapped:
476 break;
477 default:
478 BUG();
479 }
480 ttm_mem_io_free(bdev: map->bo->bdev, mem: map->bo->resource);
481 map->virtual = NULL;
482 map->page = NULL;
483}
484EXPORT_SYMBOL(ttm_bo_kunmap);
485
486/**
487 * ttm_bo_vmap
488 *
489 * @bo: The buffer object.
490 * @map: pointer to a struct iosys_map representing the map.
491 *
492 * Sets up a kernel virtual mapping, using ioremap or vmap to the
493 * data in the buffer object. The parameter @map returns the virtual
494 * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap().
495 *
496 * Returns
497 * -ENOMEM: Out of memory.
498 * -EINVAL: Invalid range.
499 */
500int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map)
501{
502 struct ttm_resource *mem = bo->resource;
503 int ret;
504
505 dma_resv_assert_held(bo->base.resv);
506
507 ret = ttm_mem_io_reserve(bdev: bo->bdev, mem);
508 if (ret)
509 return ret;
510
511 if (mem->bus.is_iomem) {
512 void __iomem *vaddr_iomem;
513
514 if (mem->bus.addr)
515 vaddr_iomem = (void __iomem *)mem->bus.addr;
516 else if (mem->bus.caching == ttm_write_combined)
517 vaddr_iomem = ioremap_wc(offset: mem->bus.offset,
518 size: bo->base.size);
519#ifdef CONFIG_X86
520 else if (mem->bus.caching == ttm_cached)
521 vaddr_iomem = ioremap_cache(offset: mem->bus.offset,
522 size: bo->base.size);
523#endif
524 else
525 vaddr_iomem = ioremap(offset: mem->bus.offset, size: bo->base.size);
526
527 if (!vaddr_iomem)
528 return -ENOMEM;
529
530 iosys_map_set_vaddr_iomem(map, vaddr_iomem);
531
532 } else {
533 struct ttm_operation_ctx ctx = {
534 .interruptible = false,
535 .no_wait_gpu = false
536 };
537 struct ttm_tt *ttm = bo->ttm;
538 pgprot_t prot;
539 void *vaddr;
540
541 ret = ttm_bo_populate(bo, ctx: &ctx);
542 if (ret)
543 return ret;
544
545 /*
546 * We need to use vmap to get the desired page protection
547 * or to make the buffer object look contiguous.
548 */
549 prot = ttm_io_prot(bo, mem, PAGE_KERNEL);
550 vaddr = vmap(pages: ttm->pages, count: ttm->num_pages, flags: 0, prot);
551 if (!vaddr)
552 return -ENOMEM;
553
554 iosys_map_set_vaddr(map, vaddr);
555 }
556
557 return 0;
558}
559EXPORT_SYMBOL(ttm_bo_vmap);
560
561/**
562 * ttm_bo_vunmap
563 *
564 * @bo: The buffer object.
565 * @map: Object describing the map to unmap.
566 *
567 * Unmaps a kernel map set up by ttm_bo_vmap().
568 */
569void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map)
570{
571 struct ttm_resource *mem = bo->resource;
572
573 dma_resv_assert_held(bo->base.resv);
574
575 if (iosys_map_is_null(map))
576 return;
577
578 if (!map->is_iomem)
579 vunmap(addr: map->vaddr);
580 else if (!mem->bus.addr)
581 iounmap(addr: map->vaddr_iomem);
582 iosys_map_clear(map);
583
584 ttm_mem_io_free(bdev: bo->bdev, mem: bo->resource);
585}
586EXPORT_SYMBOL(ttm_bo_vunmap);
587
588static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo,
589 bool dst_use_tt)
590{
591 long ret;
592
593 ret = dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP,
594 intr: false, timeout: 15 * HZ);
595 if (ret == 0)
596 return -EBUSY;
597 if (ret < 0)
598 return ret;
599
600 if (!dst_use_tt)
601 ttm_bo_tt_destroy(bo);
602 ttm_resource_free(bo, res: &bo->resource);
603 return 0;
604}
605
606static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo,
607 struct dma_fence *fence,
608 bool dst_use_tt)
609{
610 struct ttm_buffer_object *ghost_obj;
611 int ret;
612
613 /**
614 * This should help pipeline ordinary buffer moves.
615 *
616 * Hang old buffer memory on a new buffer object,
617 * and leave it to be released when the GPU
618 * operation has completed.
619 */
620
621 ret = ttm_buffer_object_transfer(bo, new_obj: &ghost_obj);
622 if (ret)
623 return ret;
624
625 dma_resv_add_fence(obj: &ghost_obj->base._resv, fence,
626 usage: DMA_RESV_USAGE_KERNEL);
627
628 /**
629 * If we're not moving to fixed memory, the TTM object
630 * needs to stay alive. Otherwhise hang it on the ghost
631 * bo to be unbound and destroyed.
632 */
633
634 if (dst_use_tt)
635 ghost_obj->ttm = NULL;
636 else
637 bo->ttm = NULL;
638
639 dma_resv_unlock(obj: &ghost_obj->base._resv);
640 ttm_bo_put(bo: ghost_obj);
641 return 0;
642}
643
644static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo,
645 struct dma_fence *fence)
646{
647 struct ttm_device *bdev = bo->bdev;
648 struct ttm_resource_manager *from;
649
650 from = ttm_manager_type(bdev, mem_type: bo->resource->mem_type);
651
652 /**
653 * BO doesn't have a TTM we need to bind/unbind. Just remember
654 * this eviction and free up the allocation
655 */
656 spin_lock(lock: &from->move_lock);
657 if (!from->move || dma_fence_is_later(f1: fence, f2: from->move)) {
658 dma_fence_put(fence: from->move);
659 from->move = dma_fence_get(fence);
660 }
661 spin_unlock(lock: &from->move_lock);
662
663 ttm_resource_free(bo, res: &bo->resource);
664}
665
666/**
667 * ttm_bo_move_accel_cleanup - cleanup helper for hw copies
668 *
669 * @bo: A pointer to a struct ttm_buffer_object.
670 * @fence: A fence object that signals when moving is complete.
671 * @evict: This is an evict move. Don't return until the buffer is idle.
672 * @pipeline: evictions are to be pipelined.
673 * @new_mem: struct ttm_resource indicating where to move.
674 *
675 * Accelerated move function to be called when an accelerated move
676 * has been scheduled. The function will create a new temporary buffer object
677 * representing the old placement, and put the sync object on both buffer
678 * objects. After that the newly created buffer object is unref'd to be
679 * destroyed when the move is complete. This will help pipeline
680 * buffer moves.
681 */
682int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
683 struct dma_fence *fence,
684 bool evict,
685 bool pipeline,
686 struct ttm_resource *new_mem)
687{
688 struct ttm_device *bdev = bo->bdev;
689 struct ttm_resource_manager *from = ttm_manager_type(bdev, mem_type: bo->resource->mem_type);
690 struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type: new_mem->mem_type);
691 int ret = 0;
692
693 dma_resv_add_fence(obj: bo->base.resv, fence, usage: DMA_RESV_USAGE_KERNEL);
694 if (!evict)
695 ret = ttm_bo_move_to_ghost(bo, fence, dst_use_tt: man->use_tt);
696 else if (!from->use_tt && pipeline)
697 ttm_bo_move_pipeline_evict(bo, fence);
698 else
699 ret = ttm_bo_wait_free_node(bo, dst_use_tt: man->use_tt);
700
701 if (ret)
702 return ret;
703
704 ttm_bo_assign_mem(bo, new_mem);
705
706 return 0;
707}
708EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
709
710/**
711 * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish
712 *
713 * @bo: A pointer to a struct ttm_buffer_object.
714 * @new_mem: struct ttm_resource indicating where to move.
715 *
716 * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed
717 * by the caller to be idle. Typically used after memcpy buffer moves.
718 */
719void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo,
720 struct ttm_resource *new_mem)
721{
722 struct ttm_device *bdev = bo->bdev;
723 struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type: new_mem->mem_type);
724 int ret;
725
726 ret = ttm_bo_wait_free_node(bo, dst_use_tt: man->use_tt);
727 if (WARN_ON(ret))
728 return;
729
730 ttm_bo_assign_mem(bo, new_mem);
731}
732EXPORT_SYMBOL(ttm_bo_move_sync_cleanup);
733
734/**
735 * ttm_bo_pipeline_gutting - purge the contents of a bo
736 * @bo: The buffer object
737 *
738 * Purge the contents of a bo, async if the bo is not idle.
739 * After a successful call, the bo is left unpopulated in
740 * system placement. The function may wait uninterruptible
741 * for idle on OOM.
742 *
743 * Return: 0 if successful, negative error code on failure.
744 */
745int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
746{
747 struct ttm_buffer_object *ghost;
748 struct ttm_tt *ttm;
749 int ret;
750
751 /* If already idle, no need for ghost object dance. */
752 if (dma_resv_test_signaled(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP)) {
753 if (!bo->ttm) {
754 /* See comment below about clearing. */
755 ret = ttm_tt_create(bo, zero_alloc: true);
756 if (ret)
757 return ret;
758 } else {
759 ttm_tt_unpopulate(bdev: bo->bdev, ttm: bo->ttm);
760 if (bo->type == ttm_bo_type_device)
761 ttm_tt_mark_for_clear(ttm: bo->ttm);
762 }
763 ttm_resource_free(bo, res: &bo->resource);
764 return 0;
765 }
766
767 /*
768 * We need an unpopulated ttm_tt after giving our current one,
769 * if any, to the ghost object. And we can't afford to fail
770 * creating one *after* the operation. If the bo subsequently gets
771 * resurrected, make sure it's cleared (if ttm_bo_type_device)
772 * to avoid leaking sensitive information to user-space.
773 */
774
775 ttm = bo->ttm;
776 bo->ttm = NULL;
777 ret = ttm_tt_create(bo, zero_alloc: true);
778 swap(bo->ttm, ttm);
779 if (ret)
780 return ret;
781
782 ret = ttm_buffer_object_transfer(bo, new_obj: &ghost);
783 if (ret)
784 goto error_destroy_tt;
785
786 ret = dma_resv_copy_fences(dst: &ghost->base._resv, src: bo->base.resv);
787 /* Last resort, wait for the BO to be idle when we are OOM */
788 if (ret) {
789 dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP,
790 intr: false, MAX_SCHEDULE_TIMEOUT);
791 }
792
793 dma_resv_unlock(obj: &ghost->base._resv);
794 ttm_bo_put(bo: ghost);
795 bo->ttm = ttm;
796 return 0;
797
798error_destroy_tt:
799 ttm_tt_destroy(bdev: bo->bdev, ttm);
800 return ret;
801}
802
803static bool ttm_lru_walk_trylock(struct ttm_bo_lru_cursor *curs,
804 struct ttm_buffer_object *bo)
805{
806 struct ttm_operation_ctx *ctx = curs->arg->ctx;
807
808 curs->needs_unlock = false;
809
810 if (dma_resv_trylock(obj: bo->base.resv)) {
811 curs->needs_unlock = true;
812 return true;
813 }
814
815 if (bo->base.resv == ctx->resv && ctx->allow_res_evict) {
816 dma_resv_assert_held(bo->base.resv);
817 return true;
818 }
819
820 return false;
821}
822
823static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs,
824 struct ttm_buffer_object *bo)
825{
826 struct ttm_lru_walk_arg *arg = curs->arg;
827 struct dma_resv *resv = bo->base.resv;
828 int ret;
829
830 if (arg->ctx->interruptible)
831 ret = dma_resv_lock_interruptible(obj: resv, ctx: arg->ticket);
832 else
833 ret = dma_resv_lock(obj: resv, ctx: arg->ticket);
834
835 if (!ret) {
836 curs->needs_unlock = true;
837 /*
838 * Only a single ticketlock per loop. Ticketlocks are prone
839 * to return -EDEADLK causing the eviction to fail, so
840 * after waiting for the ticketlock, revert back to
841 * trylocking for this walk.
842 */
843 arg->ticket = NULL;
844 } else if (ret == -EDEADLK) {
845 /* Caller needs to exit the ww transaction. */
846 ret = -ENOSPC;
847 }
848
849 return ret;
850}
851
852/**
853 * ttm_lru_walk_for_evict() - Perform a LRU list walk, with actions taken on
854 * valid items.
855 * @walk: describe the walks and actions taken
856 * @bdev: The TTM device.
857 * @man: The struct ttm_resource manager whose LRU lists we're walking.
858 * @target: The end condition for the walk.
859 *
860 * The LRU lists of @man are walk, and for each struct ttm_resource encountered,
861 * the corresponding ttm_buffer_object is locked and taken a reference on, and
862 * the LRU lock is dropped. the LRU lock may be dropped before locking and, in
863 * that case, it's verified that the item actually remains on the LRU list after
864 * the lock, and that the buffer object didn't switch resource in between.
865 *
866 * With a locked object, the actions indicated by @walk->process_bo are
867 * performed, and after that, the bo is unlocked, the refcount dropped and the
868 * next struct ttm_resource is processed. Here, the walker relies on
869 * TTM's restartable LRU list implementation.
870 *
871 * Typically @walk->process_bo() would return the number of pages evicted,
872 * swapped or shrunken, so that when the total exceeds @target, or when the
873 * LRU list has been walked in full, iteration is terminated. It's also terminated
874 * on error. Note that the definition of @target is done by the caller, it
875 * could have a different meaning than the number of pages.
876 *
877 * Note that the way dma_resv individualization is done, locking needs to be done
878 * either with the LRU lock held (trylocking only) or with a reference on the
879 * object.
880 *
881 * Return: The progress made towards target or negative error code on error.
882 */
883s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev,
884 struct ttm_resource_manager *man, s64 target)
885{
886 struct ttm_bo_lru_cursor cursor;
887 struct ttm_buffer_object *bo;
888 s64 progress = 0;
889 s64 lret;
890
891 ttm_bo_lru_for_each_reserved_guarded(&cursor, man, &walk->arg, bo) {
892 lret = walk->ops->process_bo(walk, bo);
893 if (lret == -EBUSY || lret == -EALREADY)
894 lret = 0;
895 progress = (lret < 0) ? lret : progress + lret;
896 if (progress < 0 || progress >= target)
897 break;
898 }
899 if (IS_ERR(ptr: bo))
900 return PTR_ERR(ptr: bo);
901
902 return progress;
903}
904EXPORT_SYMBOL(ttm_lru_walk_for_evict);
905
906static void ttm_bo_lru_cursor_cleanup_bo(struct ttm_bo_lru_cursor *curs)
907{
908 struct ttm_buffer_object *bo = curs->bo;
909
910 if (bo) {
911 if (curs->needs_unlock)
912 dma_resv_unlock(obj: bo->base.resv);
913 ttm_bo_put(bo);
914 curs->bo = NULL;
915 }
916}
917
918/**
919 * ttm_bo_lru_cursor_fini() - Stop using a struct ttm_bo_lru_cursor
920 * and clean up any iteration it was used for.
921 * @curs: The cursor.
922 */
923void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs)
924{
925 spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock;
926
927 ttm_bo_lru_cursor_cleanup_bo(curs);
928 spin_lock(lock: lru_lock);
929 ttm_resource_cursor_fini(cursor: &curs->res_curs);
930 spin_unlock(lock: lru_lock);
931}
932EXPORT_SYMBOL(ttm_bo_lru_cursor_fini);
933
934/**
935 * ttm_bo_lru_cursor_init() - Initialize a struct ttm_bo_lru_cursor
936 * @curs: The ttm_bo_lru_cursor to initialize.
937 * @man: The ttm resource_manager whose LRU lists to iterate over.
938 * @arg: The ttm_lru_walk_arg to govern the walk.
939 *
940 * Initialize a struct ttm_bo_lru_cursor.
941 *
942 * Return: Pointer to @curs. The function does not fail.
943 */
944struct ttm_bo_lru_cursor *
945ttm_bo_lru_cursor_init(struct ttm_bo_lru_cursor *curs,
946 struct ttm_resource_manager *man,
947 struct ttm_lru_walk_arg *arg)
948{
949 memset(s: curs, c: 0, n: sizeof(*curs));
950 ttm_resource_cursor_init(cursor: &curs->res_curs, man);
951 curs->arg = arg;
952
953 return curs;
954}
955EXPORT_SYMBOL(ttm_bo_lru_cursor_init);
956
957static struct ttm_buffer_object *
958__ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs)
959{
960 spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock;
961 struct ttm_resource *res = NULL;
962 struct ttm_buffer_object *bo;
963 struct ttm_lru_walk_arg *arg = curs->arg;
964 bool first = !curs->bo;
965
966 ttm_bo_lru_cursor_cleanup_bo(curs);
967
968 spin_lock(lock: lru_lock);
969 for (;;) {
970 int mem_type, ret = 0;
971 bool bo_locked = false;
972
973 if (first) {
974 res = ttm_resource_manager_first(cursor: &curs->res_curs);
975 first = false;
976 } else {
977 res = ttm_resource_manager_next(cursor: &curs->res_curs);
978 }
979 if (!res)
980 break;
981
982 bo = res->bo;
983 if (ttm_lru_walk_trylock(curs, bo))
984 bo_locked = true;
985 else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only)
986 continue;
987
988 if (!ttm_bo_get_unless_zero(bo)) {
989 if (curs->needs_unlock)
990 dma_resv_unlock(obj: bo->base.resv);
991 continue;
992 }
993
994 mem_type = res->mem_type;
995 spin_unlock(lock: lru_lock);
996 if (!bo_locked)
997 ret = ttm_lru_walk_ticketlock(curs, bo);
998
999 /*
1000 * Note that in between the release of the lru lock and the
1001 * ticketlock, the bo may have switched resource,
1002 * and also memory type, since the resource may have been
1003 * freed and allocated again with a different memory type.
1004 * In that case, just skip it.
1005 */
1006 curs->bo = bo;
1007 if (!ret && bo->resource && bo->resource->mem_type == mem_type)
1008 return bo;
1009
1010 ttm_bo_lru_cursor_cleanup_bo(curs);
1011 if (ret && ret != -EALREADY)
1012 return ERR_PTR(error: ret);
1013
1014 spin_lock(lock: lru_lock);
1015 }
1016
1017 spin_unlock(lock: lru_lock);
1018 return res ? bo : NULL;
1019}
1020
1021/**
1022 * ttm_bo_lru_cursor_next() - Continue iterating a manager's LRU lists
1023 * to find and lock buffer object.
1024 * @curs: The cursor initialized using ttm_bo_lru_cursor_init() and
1025 * ttm_bo_lru_cursor_first().
1026 *
1027 * Return: A pointer to a locked and reference-counted buffer object,
1028 * or NULL if none could be found and looping should be terminated.
1029 */
1030struct ttm_buffer_object *ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs)
1031{
1032 return __ttm_bo_lru_cursor_next(curs);
1033}
1034EXPORT_SYMBOL(ttm_bo_lru_cursor_next);
1035
1036/**
1037 * ttm_bo_lru_cursor_first() - Start iterating a manager's LRU lists
1038 * to find and lock buffer object.
1039 * @curs: The cursor initialized using ttm_bo_lru_cursor_init().
1040 *
1041 * Return: A pointer to a locked and reference-counted buffer object,
1042 * or NULL if none could be found and looping should be terminated.
1043 */
1044struct ttm_buffer_object *ttm_bo_lru_cursor_first(struct ttm_bo_lru_cursor *curs)
1045{
1046 ttm_bo_lru_cursor_cleanup_bo(curs);
1047 return __ttm_bo_lru_cursor_next(curs);
1048}
1049EXPORT_SYMBOL(ttm_bo_lru_cursor_first);
1050
1051/**
1052 * ttm_bo_shrink() - Helper to shrink a ttm buffer object.
1053 * @ctx: The struct ttm_operation_ctx used for the shrinking operation.
1054 * @bo: The buffer object.
1055 * @flags: Flags governing the shrinking behaviour.
1056 *
1057 * The function uses the ttm_tt_back_up functionality to back up or
1058 * purge a struct ttm_tt. If the bo is not in system, it's first
1059 * moved there.
1060 *
1061 * Return: The number of pages shrunken or purged, or
1062 * negative error code on failure.
1063 */
1064long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
1065 const struct ttm_bo_shrink_flags flags)
1066{
1067 static const struct ttm_place sys_placement_flags = {
1068 .fpfn = 0,
1069 .lpfn = 0,
1070 .mem_type = TTM_PL_SYSTEM,
1071 .flags = 0,
1072 };
1073 static struct ttm_placement sys_placement = {
1074 .num_placement = 1,
1075 .placement = &sys_placement_flags,
1076 };
1077 struct ttm_tt *tt = bo->ttm;
1078 long lret;
1079
1080 dma_resv_assert_held(bo->base.resv);
1081
1082 if (flags.allow_move && bo->resource->mem_type != TTM_PL_SYSTEM) {
1083 int ret = ttm_bo_validate(bo, placement: &sys_placement, ctx);
1084
1085 /* Consider -ENOMEM and -ENOSPC non-fatal. */
1086 if (ret) {
1087 if (ret == -ENOMEM || ret == -ENOSPC)
1088 ret = -EBUSY;
1089 return ret;
1090 }
1091 }
1092
1093 ttm_bo_unmap_virtual(bo);
1094 lret = ttm_bo_wait_ctx(bo, ctx);
1095 if (lret < 0)
1096 return lret;
1097
1098 if (bo->bulk_move) {
1099 spin_lock(lock: &bo->bdev->lru_lock);
1100 ttm_resource_del_bulk_move(res: bo->resource, bo);
1101 spin_unlock(lock: &bo->bdev->lru_lock);
1102 }
1103
1104 lret = ttm_tt_backup(bdev: bo->bdev, tt, flags: (struct ttm_backup_flags)
1105 {.purge = flags.purge,
1106 .writeback = flags.writeback});
1107
1108 if (lret <= 0 && bo->bulk_move) {
1109 spin_lock(lock: &bo->bdev->lru_lock);
1110 ttm_resource_add_bulk_move(res: bo->resource, bo);
1111 spin_unlock(lock: &bo->bdev->lru_lock);
1112 }
1113
1114 if (lret < 0 && lret != -EINTR)
1115 return -EBUSY;
1116
1117 return lret;
1118}
1119EXPORT_SYMBOL(ttm_bo_shrink);
1120
1121/**
1122 * ttm_bo_shrink_suitable() - Whether a bo is suitable for shinking
1123 * @ctx: The struct ttm_operation_ctx governing the shrinking.
1124 * @bo: The candidate for shrinking.
1125 *
1126 * Check whether the object, given the information available to TTM,
1127 * is suitable for shinking, This function can and should be used
1128 * before attempting to shrink an object.
1129 *
1130 * Return: true if suitable. false if not.
1131 */
1132bool ttm_bo_shrink_suitable(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
1133{
1134 return bo->ttm && ttm_tt_is_populated(tt: bo->ttm) && !bo->pin_count &&
1135 (!ctx->no_wait_gpu ||
1136 dma_resv_test_signaled(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP));
1137}
1138EXPORT_SYMBOL(ttm_bo_shrink_suitable);
1139
1140/**
1141 * ttm_bo_shrink_avoid_wait() - Whether to avoid waiting for GPU
1142 * during shrinking
1143 *
1144 * In some situations, like direct reclaim, waiting (in particular gpu waiting)
1145 * should be avoided since it may stall a system that could otherwise make progress
1146 * shrinking something else less time consuming.
1147 *
1148 * Return: true if gpu waiting should be avoided, false if not.
1149 */
1150bool ttm_bo_shrink_avoid_wait(void)
1151{
1152 return !current_is_kswapd();
1153}
1154EXPORT_SYMBOL(ttm_bo_shrink_avoid_wait);
1155