1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include <linux/dma-buf.h>
29#include <linux/export.h>
30#include <linux/file.h>
31#include <linux/fs.h>
32#include <linux/iosys-map.h>
33#include <linux/mem_encrypt.h>
34#include <linux/mm.h>
35#include <linux/mman.h>
36#include <linux/module.h>
37#include <linux/pagemap.h>
38#include <linux/pagevec.h>
39#include <linux/shmem_fs.h>
40#include <linux/slab.h>
41#include <linux/string_helpers.h>
42#include <linux/types.h>
43#include <linux/uaccess.h>
44
45#include <drm/drm.h>
46#include <drm/drm_device.h>
47#include <drm/drm_drv.h>
48#include <drm/drm_file.h>
49#include <drm/drm_gem.h>
50#include <drm/drm_managed.h>
51#include <drm/drm_print.h>
52#include <drm/drm_vma_manager.h>
53
54#include "drm_internal.h"
55
56/** @file drm_gem.c
57 *
58 * This file provides some of the base ioctls and library routines for
59 * the graphics memory manager implemented by each device driver.
60 *
61 * Because various devices have different requirements in terms of
62 * synchronization and migration strategies, implementing that is left up to
63 * the driver, and all that the general API provides should be generic --
64 * allocating objects, reading/writing data with the cpu, freeing objects.
65 * Even there, platform-dependent optimizations for reading/writing data with
66 * the CPU mean we'll likely hook those out to driver-specific calls. However,
67 * the DRI2 implementation wants to have at least allocate/mmap be generic.
68 *
69 * The goal was to have swap-backed object allocation managed through
70 * struct file. However, file descriptors as handles to a struct file have
71 * two major failings:
72 * - Process limits prevent more than 1024 or so being used at a time by
73 * default.
74 * - Inability to allocate high fds will aggravate the X Server's select()
75 * handling, and likely that of many GL client applications as well.
76 *
77 * This led to a plan of using our own integer IDs (called handles, following
78 * DRM terminology) to mimic fds, and implement the fd syscalls we need as
79 * ioctls. The objects themselves will still include the struct file so
80 * that we can transition to fds if the required kernel infrastructure shows
81 * up at a later date, and as our interface with shmfs for memory allocation.
82 */
83
84static void
85drm_gem_init_release(struct drm_device *dev, void *ptr)
86{
87 drm_vma_offset_manager_destroy(mgr: dev->vma_offset_manager);
88}
89
90/**
91 * drm_gem_init - Initialize the GEM device fields
92 * @dev: drm_devic structure to initialize
93 */
94int
95drm_gem_init(struct drm_device *dev)
96{
97 struct drm_vma_offset_manager *vma_offset_manager;
98
99 mutex_init(&dev->object_name_lock);
100 idr_init_base(idr: &dev->object_name_idr, base: 1);
101
102 vma_offset_manager = drmm_kzalloc(dev, size: sizeof(*vma_offset_manager),
103 GFP_KERNEL);
104 if (!vma_offset_manager) {
105 DRM_ERROR("out of memory\n");
106 return -ENOMEM;
107 }
108
109 dev->vma_offset_manager = vma_offset_manager;
110 drm_vma_offset_manager_init(mgr: vma_offset_manager,
111 DRM_FILE_PAGE_OFFSET_START,
112 DRM_FILE_PAGE_OFFSET_SIZE);
113
114 return drmm_add_action(dev, drm_gem_init_release, NULL);
115}
116
117/**
118 * drm_gem_object_init_with_mnt - initialize an allocated shmem-backed GEM
119 * object in a given shmfs mountpoint
120 *
121 * @dev: drm_device the object should be initialized for
122 * @obj: drm_gem_object to initialize
123 * @size: object size
124 * @gemfs: tmpfs mount where the GEM object will be created. If NULL, use
125 * the usual tmpfs mountpoint (`shm_mnt`).
126 *
127 * Initialize an already allocated GEM object of the specified size with
128 * shmfs backing store.
129 */
130int drm_gem_object_init_with_mnt(struct drm_device *dev,
131 struct drm_gem_object *obj, size_t size,
132 struct vfsmount *gemfs)
133{
134 struct file *filp;
135
136 drm_gem_private_object_init(dev, obj, size);
137
138 if (gemfs)
139 filp = shmem_file_setup_with_mnt(mnt: gemfs, name: "drm mm object", size,
140 VM_NORESERVE);
141 else
142 filp = shmem_file_setup(name: "drm mm object", size, VM_NORESERVE);
143
144 if (IS_ERR(ptr: filp))
145 return PTR_ERR(ptr: filp);
146
147 obj->filp = filp;
148
149 return 0;
150}
151EXPORT_SYMBOL(drm_gem_object_init_with_mnt);
152
153/**
154 * drm_gem_object_init - initialize an allocated shmem-backed GEM object
155 * @dev: drm_device the object should be initialized for
156 * @obj: drm_gem_object to initialize
157 * @size: object size
158 *
159 * Initialize an already allocated GEM object of the specified size with
160 * shmfs backing store.
161 */
162int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj,
163 size_t size)
164{
165 return drm_gem_object_init_with_mnt(dev, obj, size, NULL);
166}
167EXPORT_SYMBOL(drm_gem_object_init);
168
169/**
170 * drm_gem_private_object_init - initialize an allocated private GEM object
171 * @dev: drm_device the object should be initialized for
172 * @obj: drm_gem_object to initialize
173 * @size: object size
174 *
175 * Initialize an already allocated GEM object of the specified size with
176 * no GEM provided backing store. Instead the caller is responsible for
177 * backing the object and handling it.
178 */
179void drm_gem_private_object_init(struct drm_device *dev,
180 struct drm_gem_object *obj, size_t size)
181{
182 BUG_ON((size & (PAGE_SIZE - 1)) != 0);
183
184 obj->dev = dev;
185 obj->filp = NULL;
186
187 kref_init(kref: &obj->refcount);
188 obj->handle_count = 0;
189 obj->size = size;
190 mutex_init(&obj->gpuva.lock);
191 dma_resv_init(obj: &obj->_resv);
192 if (!obj->resv)
193 obj->resv = &obj->_resv;
194
195 if (drm_core_check_feature(dev, feature: DRIVER_GEM_GPUVA))
196 drm_gem_gpuva_init(obj);
197
198 drm_vma_node_reset(node: &obj->vma_node);
199 INIT_LIST_HEAD(list: &obj->lru_node);
200}
201EXPORT_SYMBOL(drm_gem_private_object_init);
202
203/**
204 * drm_gem_private_object_fini - Finalize a failed drm_gem_object
205 * @obj: drm_gem_object
206 *
207 * Uninitialize an already allocated GEM object when it initialized failed
208 */
209void drm_gem_private_object_fini(struct drm_gem_object *obj)
210{
211 WARN_ON(obj->dma_buf);
212
213 dma_resv_fini(obj: &obj->_resv);
214 mutex_destroy(lock: &obj->gpuva.lock);
215}
216EXPORT_SYMBOL(drm_gem_private_object_fini);
217
218static void drm_gem_object_handle_get(struct drm_gem_object *obj)
219{
220 struct drm_device *dev = obj->dev;
221
222 drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock));
223
224 if (obj->handle_count++ == 0)
225 drm_gem_object_get(obj);
226}
227
228/**
229 * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any
230 * @obj: GEM object
231 *
232 * Acquires a reference on the GEM buffer object's handle. Required to keep
233 * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked()
234 * to release the reference. Does nothing if the buffer object has no handle.
235 *
236 * Returns:
237 * True if a handle exists, or false otherwise
238 */
239bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj)
240{
241 struct drm_device *dev = obj->dev;
242
243 guard(mutex)(T: &dev->object_name_lock);
244
245 /*
246 * First ref taken during GEM object creation, if any. Some
247 * drivers set up internal framebuffers with GEM objects that
248 * do not have a GEM handle. Hence, this counter can be zero.
249 */
250 if (!obj->handle_count)
251 return false;
252
253 drm_gem_object_handle_get(obj);
254
255 return true;
256}
257
258/**
259 * drm_gem_object_handle_free - release resources bound to userspace handles
260 * @obj: GEM object to clean up.
261 *
262 * Called after the last handle to the object has been closed
263 *
264 * Removes any name for the object. Note that this must be
265 * called before drm_gem_object_free or we'll be touching
266 * freed memory
267 */
268static void drm_gem_object_handle_free(struct drm_gem_object *obj)
269{
270 struct drm_device *dev = obj->dev;
271
272 /* Remove any name for this object */
273 if (obj->name) {
274 idr_remove(&dev->object_name_idr, id: obj->name);
275 obj->name = 0;
276 }
277}
278
279static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
280{
281 /* Unbreak the reference cycle if we have an exported dma_buf. */
282 if (obj->dma_buf) {
283 dma_buf_put(dmabuf: obj->dma_buf);
284 obj->dma_buf = NULL;
285 }
286}
287
288/**
289 * drm_gem_object_handle_put_unlocked - releases reference on user-space handle
290 * @obj: GEM object
291 *
292 * Releases a reference on the GEM buffer object's handle. Possibly releases
293 * the GEM buffer object and associated dma-buf objects.
294 */
295void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
296{
297 struct drm_device *dev = obj->dev;
298 bool final = false;
299
300 if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0))
301 return;
302
303 /*
304 * Must bump handle count first as this may be the last
305 * ref, in which case the object would disappear before
306 * we checked for a name.
307 */
308
309 mutex_lock(lock: &dev->object_name_lock);
310 if (--obj->handle_count == 0) {
311 drm_gem_object_handle_free(obj);
312 drm_gem_object_exported_dma_buf_free(obj);
313 final = true;
314 }
315 mutex_unlock(lock: &dev->object_name_lock);
316
317 if (final)
318 drm_gem_object_put(obj);
319}
320
321/*
322 * Called at device or object close to release the file's
323 * handle references on objects.
324 */
325static int
326drm_gem_object_release_handle(int id, void *ptr, void *data)
327{
328 struct drm_file *file_priv = data;
329 struct drm_gem_object *obj = ptr;
330
331 if (drm_WARN_ON(obj->dev, !data))
332 return 0;
333
334 if (obj->funcs->close)
335 obj->funcs->close(obj, file_priv);
336
337 mutex_lock(lock: &file_priv->prime.lock);
338
339 drm_prime_remove_buf_handle(prime_fpriv: &file_priv->prime, handle: id);
340
341 mutex_unlock(lock: &file_priv->prime.lock);
342
343 drm_vma_node_revoke(node: &obj->vma_node, tag: file_priv);
344
345 drm_gem_object_handle_put_unlocked(obj);
346
347 return 0;
348}
349
350/**
351 * drm_gem_handle_delete - deletes the given file-private handle
352 * @filp: drm file-private structure to use for the handle look up
353 * @handle: userspace handle to delete
354 *
355 * Removes the GEM handle from the @filp lookup table which has been added with
356 * drm_gem_handle_create(). If this is the last handle also cleans up linked
357 * resources like GEM names.
358 */
359int
360drm_gem_handle_delete(struct drm_file *filp, u32 handle)
361{
362 struct drm_gem_object *obj;
363
364 spin_lock(lock: &filp->table_lock);
365
366 /* Check if we currently have a reference on the object */
367 obj = idr_replace(&filp->object_idr, NULL, id: handle);
368 spin_unlock(lock: &filp->table_lock);
369 if (IS_ERR_OR_NULL(ptr: obj))
370 return -EINVAL;
371
372 /* Release driver's reference and decrement refcount. */
373 drm_gem_object_release_handle(id: handle, ptr: obj, data: filp);
374
375 /* And finally make the handle available for future allocations. */
376 spin_lock(lock: &filp->table_lock);
377 idr_remove(&filp->object_idr, id: handle);
378 spin_unlock(lock: &filp->table_lock);
379
380 return 0;
381}
382EXPORT_SYMBOL(drm_gem_handle_delete);
383
384/**
385 * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object
386 * @file: drm file-private structure containing the gem object
387 * @dev: corresponding drm_device
388 * @handle: gem object handle
389 * @offset: return location for the fake mmap offset
390 *
391 * This implements the &drm_driver.dumb_map_offset kms driver callback for
392 * drivers which use gem to manage their backing storage.
393 *
394 * Returns:
395 * 0 on success or a negative error code on failure.
396 */
397int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
398 u32 handle, u64 *offset)
399{
400 struct drm_gem_object *obj;
401 int ret;
402
403 obj = drm_gem_object_lookup(filp: file, handle);
404 if (!obj)
405 return -ENOENT;
406
407 /* Don't allow imported objects to be mapped */
408 if (drm_gem_is_imported(obj)) {
409 ret = -EINVAL;
410 goto out;
411 }
412
413 ret = drm_gem_create_mmap_offset(obj);
414 if (ret)
415 goto out;
416
417 *offset = drm_vma_node_offset_addr(node: &obj->vma_node);
418out:
419 drm_gem_object_put(obj);
420
421 return ret;
422}
423EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
424
425/**
426 * drm_gem_handle_create_tail - internal functions to create a handle
427 * @file_priv: drm file-private structure to register the handle for
428 * @obj: object to register
429 * @handlep: pointer to return the created handle to the caller
430 *
431 * This expects the &drm_device.object_name_lock to be held already and will
432 * drop it before returning. Used to avoid races in establishing new handles
433 * when importing an object from either an flink name or a dma-buf.
434 *
435 * Handles must be release again through drm_gem_handle_delete(). This is done
436 * when userspace closes @file_priv for all attached handles, or through the
437 * GEM_CLOSE ioctl for individual handles.
438 */
439int
440drm_gem_handle_create_tail(struct drm_file *file_priv,
441 struct drm_gem_object *obj,
442 u32 *handlep)
443{
444 struct drm_device *dev = obj->dev;
445 u32 handle;
446 int ret;
447
448 WARN_ON(!mutex_is_locked(&dev->object_name_lock));
449
450 drm_gem_object_handle_get(obj);
451
452 /*
453 * Get the user-visible handle using idr. Preload and perform
454 * allocation under our spinlock.
455 */
456 idr_preload(GFP_KERNEL);
457 spin_lock(lock: &file_priv->table_lock);
458
459 ret = idr_alloc(&file_priv->object_idr, NULL, start: 1, end: 0, GFP_NOWAIT);
460
461 spin_unlock(lock: &file_priv->table_lock);
462 idr_preload_end();
463
464 mutex_unlock(lock: &dev->object_name_lock);
465 if (ret < 0)
466 goto err_unref;
467
468 handle = ret;
469
470 ret = drm_vma_node_allow(node: &obj->vma_node, tag: file_priv);
471 if (ret)
472 goto err_remove;
473
474 if (obj->funcs->open) {
475 ret = obj->funcs->open(obj, file_priv);
476 if (ret)
477 goto err_revoke;
478 }
479
480 /* mirrors drm_gem_handle_delete to avoid races */
481 spin_lock(lock: &file_priv->table_lock);
482 obj = idr_replace(&file_priv->object_idr, obj, id: handle);
483 WARN_ON(obj != NULL);
484 spin_unlock(lock: &file_priv->table_lock);
485 *handlep = handle;
486 return 0;
487
488err_revoke:
489 drm_vma_node_revoke(node: &obj->vma_node, tag: file_priv);
490err_remove:
491 spin_lock(lock: &file_priv->table_lock);
492 idr_remove(&file_priv->object_idr, id: handle);
493 spin_unlock(lock: &file_priv->table_lock);
494err_unref:
495 drm_gem_object_handle_put_unlocked(obj);
496 return ret;
497}
498
499/**
500 * drm_gem_handle_create - create a gem handle for an object
501 * @file_priv: drm file-private structure to register the handle for
502 * @obj: object to register
503 * @handlep: pointer to return the created handle to the caller
504 *
505 * Create a handle for this object. This adds a handle reference to the object,
506 * which includes a regular reference count. Callers will likely want to
507 * dereference the object afterwards.
508 *
509 * Since this publishes @obj to userspace it must be fully set up by this point,
510 * drivers must call this last in their buffer object creation callbacks.
511 */
512int drm_gem_handle_create(struct drm_file *file_priv,
513 struct drm_gem_object *obj,
514 u32 *handlep)
515{
516 mutex_lock(lock: &obj->dev->object_name_lock);
517
518 return drm_gem_handle_create_tail(file_priv, obj, handlep);
519}
520EXPORT_SYMBOL(drm_gem_handle_create);
521
522
523/**
524 * drm_gem_free_mmap_offset - release a fake mmap offset for an object
525 * @obj: obj in question
526 *
527 * This routine frees fake offsets allocated by drm_gem_create_mmap_offset().
528 *
529 * Note that drm_gem_object_release() already calls this function, so drivers
530 * don't have to take care of releasing the mmap offset themselves when freeing
531 * the GEM object.
532 */
533void
534drm_gem_free_mmap_offset(struct drm_gem_object *obj)
535{
536 struct drm_device *dev = obj->dev;
537
538 drm_vma_offset_remove(mgr: dev->vma_offset_manager, node: &obj->vma_node);
539}
540EXPORT_SYMBOL(drm_gem_free_mmap_offset);
541
542/**
543 * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object
544 * @obj: obj in question
545 * @size: the virtual size
546 *
547 * GEM memory mapping works by handing back to userspace a fake mmap offset
548 * it can use in a subsequent mmap(2) call. The DRM core code then looks
549 * up the object based on the offset and sets up the various memory mapping
550 * structures.
551 *
552 * This routine allocates and attaches a fake offset for @obj, in cases where
553 * the virtual size differs from the physical size (ie. &drm_gem_object.size).
554 * Otherwise just use drm_gem_create_mmap_offset().
555 *
556 * This function is idempotent and handles an already allocated mmap offset
557 * transparently. Drivers do not need to check for this case.
558 */
559int
560drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)
561{
562 struct drm_device *dev = obj->dev;
563
564 return drm_vma_offset_add(mgr: dev->vma_offset_manager, node: &obj->vma_node,
565 pages: size / PAGE_SIZE);
566}
567EXPORT_SYMBOL(drm_gem_create_mmap_offset_size);
568
569/**
570 * drm_gem_create_mmap_offset - create a fake mmap offset for an object
571 * @obj: obj in question
572 *
573 * GEM memory mapping works by handing back to userspace a fake mmap offset
574 * it can use in a subsequent mmap(2) call. The DRM core code then looks
575 * up the object based on the offset and sets up the various memory mapping
576 * structures.
577 *
578 * This routine allocates and attaches a fake offset for @obj.
579 *
580 * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release
581 * the fake offset again.
582 */
583int drm_gem_create_mmap_offset(struct drm_gem_object *obj)
584{
585 return drm_gem_create_mmap_offset_size(obj, obj->size);
586}
587EXPORT_SYMBOL(drm_gem_create_mmap_offset);
588
589/*
590 * Move folios to appropriate lru and release the folios, decrementing the
591 * ref count of those folios.
592 */
593static void drm_gem_check_release_batch(struct folio_batch *fbatch)
594{
595 check_move_unevictable_folios(fbatch);
596 __folio_batch_release(pvec: fbatch);
597 cond_resched();
598}
599
600/**
601 * drm_gem_get_pages - helper to allocate backing pages for a GEM object
602 * from shmem
603 * @obj: obj in question
604 *
605 * This reads the page-array of the shmem-backing storage of the given gem
606 * object. An array of pages is returned. If a page is not allocated or
607 * swapped-out, this will allocate/swap-in the required pages. Note that the
608 * whole object is covered by the page-array and pinned in memory.
609 *
610 * Use drm_gem_put_pages() to release the array and unpin all pages.
611 *
612 * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()).
613 * If you require other GFP-masks, you have to do those allocations yourself.
614 *
615 * Note that you are not allowed to change gfp-zones during runtime. That is,
616 * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as
617 * set during initialization. If you have special zone constraints, set them
618 * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care
619 * to keep pages in the required zone during swap-in.
620 *
621 * This function is only valid on objects initialized with
622 * drm_gem_object_init(), but not for those initialized with
623 * drm_gem_private_object_init() only.
624 */
625struct page **drm_gem_get_pages(struct drm_gem_object *obj)
626{
627 struct address_space *mapping;
628 struct page **pages;
629 struct folio *folio;
630 struct folio_batch fbatch;
631 unsigned long i, j, npages;
632
633 if (WARN_ON(!obj->filp))
634 return ERR_PTR(error: -EINVAL);
635
636 /* This is the shared memory object that backs the GEM resource */
637 mapping = obj->filp->f_mapping;
638
639 /* We already BUG_ON() for non-page-aligned sizes in
640 * drm_gem_object_init(), so we should never hit this unless
641 * driver author is doing something really wrong:
642 */
643 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
644
645 npages = obj->size >> PAGE_SHIFT;
646
647 pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
648 if (pages == NULL)
649 return ERR_PTR(error: -ENOMEM);
650
651 mapping_set_unevictable(mapping);
652
653 i = 0;
654 while (i < npages) {
655 unsigned long nr;
656 folio = shmem_read_folio_gfp(mapping, index: i,
657 gfp: mapping_gfp_mask(mapping));
658 if (IS_ERR(ptr: folio))
659 goto fail;
660 nr = min(npages - i, folio_nr_pages(folio));
661 for (j = 0; j < nr; j++, i++)
662 pages[i] = folio_file_page(folio, index: i);
663
664 /* Make sure shmem keeps __GFP_DMA32 allocated pages in the
665 * correct region during swapin. Note that this requires
666 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping)
667 * so shmem can relocate pages during swapin if required.
668 */
669 BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) &&
670 (folio_pfn(folio) >= 0x00100000UL));
671 }
672
673 return pages;
674
675fail:
676 mapping_clear_unevictable(mapping);
677 folio_batch_init(fbatch: &fbatch);
678 j = 0;
679 while (j < i) {
680 struct folio *f = page_folio(pages[j]);
681 if (!folio_batch_add(fbatch: &fbatch, folio: f))
682 drm_gem_check_release_batch(fbatch: &fbatch);
683 j += folio_nr_pages(folio: f);
684 }
685 if (fbatch.nr)
686 drm_gem_check_release_batch(fbatch: &fbatch);
687
688 kvfree(addr: pages);
689 return ERR_CAST(ptr: folio);
690}
691EXPORT_SYMBOL(drm_gem_get_pages);
692
693/**
694 * drm_gem_put_pages - helper to free backing pages for a GEM object
695 * @obj: obj in question
696 * @pages: pages to free
697 * @dirty: if true, pages will be marked as dirty
698 * @accessed: if true, the pages will be marked as accessed
699 */
700void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
701 bool dirty, bool accessed)
702{
703 int i, npages;
704 struct address_space *mapping;
705 struct folio_batch fbatch;
706
707 mapping = file_inode(f: obj->filp)->i_mapping;
708 mapping_clear_unevictable(mapping);
709
710 /* We already BUG_ON() for non-page-aligned sizes in
711 * drm_gem_object_init(), so we should never hit this unless
712 * driver author is doing something really wrong:
713 */
714 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
715
716 npages = obj->size >> PAGE_SHIFT;
717
718 folio_batch_init(fbatch: &fbatch);
719 for (i = 0; i < npages; i++) {
720 struct folio *folio;
721
722 if (!pages[i])
723 continue;
724 folio = page_folio(pages[i]);
725
726 if (dirty)
727 folio_mark_dirty(folio);
728
729 if (accessed)
730 folio_mark_accessed(folio);
731
732 /* Undo the reference we took when populating the table */
733 if (!folio_batch_add(fbatch: &fbatch, folio))
734 drm_gem_check_release_batch(fbatch: &fbatch);
735 i += folio_nr_pages(folio) - 1;
736 }
737 if (folio_batch_count(fbatch: &fbatch))
738 drm_gem_check_release_batch(fbatch: &fbatch);
739
740 kvfree(addr: pages);
741}
742EXPORT_SYMBOL(drm_gem_put_pages);
743
744static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
745 struct drm_gem_object **objs)
746{
747 int i, ret = 0;
748 struct drm_gem_object *obj;
749
750 spin_lock(lock: &filp->table_lock);
751
752 for (i = 0; i < count; i++) {
753 /* Check if we currently have a reference on the object */
754 obj = idr_find(&filp->object_idr, id: handle[i]);
755 if (!obj) {
756 ret = -ENOENT;
757 break;
758 }
759 drm_gem_object_get(obj);
760 objs[i] = obj;
761 }
762 spin_unlock(lock: &filp->table_lock);
763
764 return ret;
765}
766
767/**
768 * drm_gem_objects_lookup - look up GEM objects from an array of handles
769 * @filp: DRM file private date
770 * @bo_handles: user pointer to array of userspace handle
771 * @count: size of handle array
772 * @objs_out: returned pointer to array of drm_gem_object pointers
773 *
774 * Takes an array of userspace handles and returns a newly allocated array of
775 * GEM objects.
776 *
777 * For a single handle lookup, use drm_gem_object_lookup().
778 *
779 * Returns:
780 * @objs filled in with GEM object pointers. Returned GEM objects need to be
781 * released with drm_gem_object_put(). -ENOENT is returned on a lookup
782 * failure. 0 is returned on success.
783 *
784 */
785int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
786 int count, struct drm_gem_object ***objs_out)
787{
788 int ret;
789 u32 *handles;
790 struct drm_gem_object **objs;
791
792 if (!count)
793 return 0;
794
795 objs = kvmalloc_array(count, sizeof(struct drm_gem_object *),
796 GFP_KERNEL | __GFP_ZERO);
797 if (!objs)
798 return -ENOMEM;
799
800 *objs_out = objs;
801
802 handles = kvmalloc_array(count, sizeof(u32), GFP_KERNEL);
803 if (!handles) {
804 ret = -ENOMEM;
805 goto out;
806 }
807
808 if (copy_from_user(to: handles, from: bo_handles, n: count * sizeof(u32))) {
809 ret = -EFAULT;
810 DRM_DEBUG("Failed to copy in GEM handles\n");
811 goto out;
812 }
813
814 ret = objects_lookup(filp, handle: handles, count, objs);
815out:
816 kvfree(addr: handles);
817 return ret;
818
819}
820EXPORT_SYMBOL(drm_gem_objects_lookup);
821
822/**
823 * drm_gem_object_lookup - look up a GEM object from its handle
824 * @filp: DRM file private date
825 * @handle: userspace handle
826 *
827 * If looking up an array of handles, use drm_gem_objects_lookup().
828 *
829 * Returns:
830 * A reference to the object named by the handle if such exists on @filp, NULL
831 * otherwise.
832 */
833struct drm_gem_object *
834drm_gem_object_lookup(struct drm_file *filp, u32 handle)
835{
836 struct drm_gem_object *obj = NULL;
837
838 objects_lookup(filp, handle: &handle, count: 1, objs: &obj);
839 return obj;
840}
841EXPORT_SYMBOL(drm_gem_object_lookup);
842
843/**
844 * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects
845 * shared and/or exclusive fences.
846 * @filep: DRM file private date
847 * @handle: userspace handle
848 * @wait_all: if true, wait on all fences, else wait on just exclusive fence
849 * @timeout: timeout value in jiffies or zero to return immediately
850 *
851 * Returns:
852 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
853 * greater than 0 on success.
854 */
855long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle,
856 bool wait_all, unsigned long timeout)
857{
858 long ret;
859 struct drm_gem_object *obj;
860
861 obj = drm_gem_object_lookup(filep, handle);
862 if (!obj) {
863 DRM_DEBUG("Failed to look up GEM BO %d\n", handle);
864 return -EINVAL;
865 }
866
867 ret = dma_resv_wait_timeout(obj: obj->resv, usage: dma_resv_usage_rw(write: wait_all),
868 intr: true, timeout);
869 if (ret == 0)
870 ret = -ETIME;
871 else if (ret > 0)
872 ret = 0;
873
874 drm_gem_object_put(obj);
875
876 return ret;
877}
878EXPORT_SYMBOL(drm_gem_dma_resv_wait);
879
880int
881drm_gem_close_ioctl(struct drm_device *dev, void *data,
882 struct drm_file *file_priv)
883{
884 struct drm_gem_close *args = data;
885 int ret;
886
887 if (!drm_core_check_feature(dev, feature: DRIVER_GEM))
888 return -EOPNOTSUPP;
889
890 ret = drm_gem_handle_delete(file_priv, args->handle);
891
892 return ret;
893}
894
895int
896drm_gem_flink_ioctl(struct drm_device *dev, void *data,
897 struct drm_file *file_priv)
898{
899 struct drm_gem_flink *args = data;
900 struct drm_gem_object *obj;
901 int ret;
902
903 if (!drm_core_check_feature(dev, feature: DRIVER_GEM))
904 return -EOPNOTSUPP;
905
906 obj = drm_gem_object_lookup(file_priv, args->handle);
907 if (obj == NULL)
908 return -ENOENT;
909
910 mutex_lock(lock: &dev->object_name_lock);
911 /* prevent races with concurrent gem_close. */
912 if (obj->handle_count == 0) {
913 ret = -ENOENT;
914 goto err;
915 }
916
917 if (!obj->name) {
918 ret = idr_alloc(&dev->object_name_idr, ptr: obj, start: 1, end: 0, GFP_KERNEL);
919 if (ret < 0)
920 goto err;
921
922 obj->name = ret;
923 }
924
925 args->name = (uint64_t) obj->name;
926 ret = 0;
927
928err:
929 mutex_unlock(lock: &dev->object_name_lock);
930 drm_gem_object_put(obj);
931 return ret;
932}
933
934int
935drm_gem_open_ioctl(struct drm_device *dev, void *data,
936 struct drm_file *file_priv)
937{
938 struct drm_gem_open *args = data;
939 struct drm_gem_object *obj;
940 int ret;
941 u32 handle;
942
943 if (!drm_core_check_feature(dev, feature: DRIVER_GEM))
944 return -EOPNOTSUPP;
945
946 mutex_lock(lock: &dev->object_name_lock);
947 obj = idr_find(&dev->object_name_idr, id: (int) args->name);
948 if (obj) {
949 drm_gem_object_get(obj);
950 } else {
951 mutex_unlock(lock: &dev->object_name_lock);
952 return -ENOENT;
953 }
954
955 /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */
956 ret = drm_gem_handle_create_tail(file_priv, obj, handlep: &handle);
957 if (ret)
958 goto err;
959
960 args->handle = handle;
961 args->size = obj->size;
962
963err:
964 drm_gem_object_put(obj);
965 return ret;
966}
967
968int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
969 struct drm_file *file_priv)
970{
971 struct drm_gem_change_handle *args = data;
972 struct drm_gem_object *obj;
973 int ret;
974
975 if (!drm_core_check_feature(dev, feature: DRIVER_GEM))
976 return -EOPNOTSUPP;
977
978 obj = drm_gem_object_lookup(file_priv, args->handle);
979 if (!obj)
980 return -ENOENT;
981
982 if (args->handle == args->new_handle)
983 return 0;
984
985 mutex_lock(lock: &file_priv->prime.lock);
986
987 spin_lock(lock: &file_priv->table_lock);
988 ret = idr_alloc(&file_priv->object_idr, ptr: obj,
989 start: args->new_handle, end: args->new_handle + 1, GFP_NOWAIT);
990 spin_unlock(lock: &file_priv->table_lock);
991
992 if (ret < 0)
993 goto out_unlock;
994
995 if (obj->dma_buf) {
996 ret = drm_prime_add_buf_handle(prime_fpriv: &file_priv->prime, dma_buf: obj->dma_buf, handle: args->new_handle);
997 if (ret < 0) {
998 spin_lock(lock: &file_priv->table_lock);
999 idr_remove(&file_priv->object_idr, id: args->new_handle);
1000 spin_unlock(lock: &file_priv->table_lock);
1001 goto out_unlock;
1002 }
1003
1004 drm_prime_remove_buf_handle(prime_fpriv: &file_priv->prime, handle: args->handle);
1005 }
1006
1007 ret = 0;
1008
1009 spin_lock(lock: &file_priv->table_lock);
1010 idr_remove(&file_priv->object_idr, id: args->handle);
1011 spin_unlock(lock: &file_priv->table_lock);
1012
1013out_unlock:
1014 mutex_unlock(lock: &file_priv->prime.lock);
1015
1016 return ret;
1017}
1018
1019/**
1020 * drm_gem_open - initializes GEM file-private structures at devnode open time
1021 * @dev: drm_device which is being opened by userspace
1022 * @file_private: drm file-private structure to set up
1023 *
1024 * Called at device open time, sets up the structure for handling refcounting
1025 * of mm objects.
1026 */
1027void
1028drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
1029{
1030 idr_init_base(idr: &file_private->object_idr, base: 1);
1031 spin_lock_init(&file_private->table_lock);
1032}
1033
1034/**
1035 * drm_gem_release - release file-private GEM resources
1036 * @dev: drm_device which is being closed by userspace
1037 * @file_private: drm file-private structure to clean up
1038 *
1039 * Called at close time when the filp is going away.
1040 *
1041 * Releases any remaining references on objects by this filp.
1042 */
1043void
1044drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
1045{
1046 idr_for_each(&file_private->object_idr,
1047 fn: &drm_gem_object_release_handle, data: file_private);
1048 idr_destroy(&file_private->object_idr);
1049}
1050
1051/**
1052 * drm_gem_object_release - release GEM buffer object resources
1053 * @obj: GEM buffer object
1054 *
1055 * This releases any structures and resources used by @obj and is the inverse of
1056 * drm_gem_object_init().
1057 */
1058void
1059drm_gem_object_release(struct drm_gem_object *obj)
1060{
1061 if (obj->filp)
1062 fput(obj->filp);
1063
1064 drm_gem_private_object_fini(obj);
1065
1066 drm_gem_free_mmap_offset(obj);
1067 drm_gem_lru_remove(obj);
1068}
1069EXPORT_SYMBOL(drm_gem_object_release);
1070
1071/**
1072 * drm_gem_object_free - free a GEM object
1073 * @kref: kref of the object to free
1074 *
1075 * Called after the last reference to the object has been lost.
1076 *
1077 * Frees the object
1078 */
1079void
1080drm_gem_object_free(struct kref *kref)
1081{
1082 struct drm_gem_object *obj =
1083 container_of(kref, struct drm_gem_object, refcount);
1084
1085 if (WARN_ON(!obj->funcs->free))
1086 return;
1087
1088 obj->funcs->free(obj);
1089}
1090EXPORT_SYMBOL(drm_gem_object_free);
1091
1092/**
1093 * drm_gem_vm_open - vma->ops->open implementation for GEM
1094 * @vma: VM area structure
1095 *
1096 * This function implements the #vm_operations_struct open() callback for GEM
1097 * drivers. This must be used together with drm_gem_vm_close().
1098 */
1099void drm_gem_vm_open(struct vm_area_struct *vma)
1100{
1101 struct drm_gem_object *obj = vma->vm_private_data;
1102
1103 drm_gem_object_get(obj);
1104}
1105EXPORT_SYMBOL(drm_gem_vm_open);
1106
1107/**
1108 * drm_gem_vm_close - vma->ops->close implementation for GEM
1109 * @vma: VM area structure
1110 *
1111 * This function implements the #vm_operations_struct close() callback for GEM
1112 * drivers. This must be used together with drm_gem_vm_open().
1113 */
1114void drm_gem_vm_close(struct vm_area_struct *vma)
1115{
1116 struct drm_gem_object *obj = vma->vm_private_data;
1117
1118 drm_gem_object_put(obj);
1119}
1120EXPORT_SYMBOL(drm_gem_vm_close);
1121
1122/**
1123 * drm_gem_mmap_obj - memory map a GEM object
1124 * @obj: the GEM object to map
1125 * @obj_size: the object size to be mapped, in bytes
1126 * @vma: VMA for the area to be mapped
1127 *
1128 * Set up the VMA to prepare mapping of the GEM object using the GEM object's
1129 * vm_ops. Depending on their requirements, GEM objects can either
1130 * provide a fault handler in their vm_ops (in which case any accesses to
1131 * the object will be trapped, to perform migration, GTT binding, surface
1132 * register allocation, or performance monitoring), or mmap the buffer memory
1133 * synchronously after calling drm_gem_mmap_obj.
1134 *
1135 * This function is mainly intended to implement the DMABUF mmap operation, when
1136 * the GEM object is not looked up based on its fake offset. To implement the
1137 * DRM mmap operation, drivers should use the drm_gem_mmap() function.
1138 *
1139 * drm_gem_mmap_obj() assumes the user is granted access to the buffer while
1140 * drm_gem_mmap() prevents unprivileged users from mapping random objects. So
1141 * callers must verify access restrictions before calling this helper.
1142 *
1143 * Return 0 or success or -EINVAL if the object size is smaller than the VMA
1144 * size, or if no vm_ops are provided.
1145 */
1146int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
1147 struct vm_area_struct *vma)
1148{
1149 int ret;
1150
1151 /* Check for valid size. */
1152 if (obj_size < vma->vm_end - vma->vm_start)
1153 return -EINVAL;
1154
1155 /* Take a ref for this mapping of the object, so that the fault
1156 * handler can dereference the mmap offset's pointer to the object.
1157 * This reference is cleaned up by the corresponding vm_close
1158 * (which should happen whether the vma was created by this call, or
1159 * by a vm_open due to mremap or partial unmap or whatever).
1160 */
1161 drm_gem_object_get(obj);
1162
1163 vma->vm_private_data = obj;
1164 vma->vm_ops = obj->funcs->vm_ops;
1165
1166 if (obj->funcs->mmap) {
1167 ret = obj->funcs->mmap(obj, vma);
1168 if (ret)
1169 goto err_drm_gem_object_put;
1170 WARN_ON(!(vma->vm_flags & VM_DONTEXPAND));
1171 } else {
1172 if (!vma->vm_ops) {
1173 ret = -EINVAL;
1174 goto err_drm_gem_object_put;
1175 }
1176
1177 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1178 vma->vm_page_prot = pgprot_writecombine(prot: vm_get_page_prot(vm_flags: vma->vm_flags));
1179 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1180 }
1181
1182 return 0;
1183
1184err_drm_gem_object_put:
1185 drm_gem_object_put(obj);
1186 return ret;
1187}
1188EXPORT_SYMBOL(drm_gem_mmap_obj);
1189
1190/**
1191 * drm_gem_mmap - memory map routine for GEM objects
1192 * @filp: DRM file pointer
1193 * @vma: VMA for the area to be mapped
1194 *
1195 * If a driver supports GEM object mapping, mmap calls on the DRM file
1196 * descriptor will end up here.
1197 *
1198 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
1199 * contain the fake offset we created when the GTT map ioctl was called on
1200 * the object) and map it with a call to drm_gem_mmap_obj().
1201 *
1202 * If the caller is not granted access to the buffer object, the mmap will fail
1203 * with EACCES. Please see the vma manager for more information.
1204 */
1205int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1206{
1207 struct drm_file *priv = filp->private_data;
1208 struct drm_device *dev = priv->minor->dev;
1209 struct drm_gem_object *obj = NULL;
1210 struct drm_vma_offset_node *node;
1211 int ret;
1212
1213 if (drm_dev_is_unplugged(dev))
1214 return -ENODEV;
1215
1216 drm_vma_offset_lock_lookup(mgr: dev->vma_offset_manager);
1217 node = drm_vma_offset_exact_lookup_locked(mgr: dev->vma_offset_manager,
1218 start: vma->vm_pgoff,
1219 pages: vma_pages(vma));
1220 if (likely(node)) {
1221 obj = container_of(node, struct drm_gem_object, vma_node);
1222 /*
1223 * When the object is being freed, after it hits 0-refcnt it
1224 * proceeds to tear down the object. In the process it will
1225 * attempt to remove the VMA offset and so acquire this
1226 * mgr->vm_lock. Therefore if we find an object with a 0-refcnt
1227 * that matches our range, we know it is in the process of being
1228 * destroyed and will be freed as soon as we release the lock -
1229 * so we have to check for the 0-refcnted object and treat it as
1230 * invalid.
1231 */
1232 if (!kref_get_unless_zero(kref: &obj->refcount))
1233 obj = NULL;
1234 }
1235 drm_vma_offset_unlock_lookup(mgr: dev->vma_offset_manager);
1236
1237 if (!obj)
1238 return -EINVAL;
1239
1240 if (!drm_vma_node_is_allowed(node, tag: priv)) {
1241 drm_gem_object_put(obj);
1242 return -EACCES;
1243 }
1244
1245 ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
1246 vma);
1247
1248 drm_gem_object_put(obj);
1249
1250 return ret;
1251}
1252EXPORT_SYMBOL(drm_gem_mmap);
1253
1254void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
1255 const struct drm_gem_object *obj)
1256{
1257 drm_printf_indent(p, indent, "name=%d\n", obj->name);
1258 drm_printf_indent(p, indent, "refcount=%u\n",
1259 kref_read(&obj->refcount));
1260 drm_printf_indent(p, indent, "start=%08lx\n",
1261 drm_vma_node_start(&obj->vma_node));
1262 drm_printf_indent(p, indent, "size=%zu\n", obj->size);
1263 drm_printf_indent(p, indent, "imported=%s\n",
1264 str_yes_no(drm_gem_is_imported(obj)));
1265
1266 if (obj->funcs->print_info)
1267 obj->funcs->print_info(p, indent, obj);
1268}
1269
1270int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map)
1271{
1272 int ret;
1273
1274 dma_resv_assert_held(obj->resv);
1275
1276 if (!obj->funcs->vmap)
1277 return -EOPNOTSUPP;
1278
1279 ret = obj->funcs->vmap(obj, map);
1280 if (ret)
1281 return ret;
1282 else if (iosys_map_is_null(map))
1283 return -ENOMEM;
1284
1285 return 0;
1286}
1287EXPORT_SYMBOL(drm_gem_vmap_locked);
1288
1289void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map)
1290{
1291 dma_resv_assert_held(obj->resv);
1292
1293 if (iosys_map_is_null(map))
1294 return;
1295
1296 if (obj->funcs->vunmap)
1297 obj->funcs->vunmap(obj, map);
1298
1299 /* Always set the mapping to NULL. Callers may rely on this. */
1300 iosys_map_clear(map);
1301}
1302EXPORT_SYMBOL(drm_gem_vunmap_locked);
1303
1304void drm_gem_lock(struct drm_gem_object *obj)
1305{
1306 dma_resv_lock(obj: obj->resv, NULL);
1307}
1308EXPORT_SYMBOL(drm_gem_lock);
1309
1310void drm_gem_unlock(struct drm_gem_object *obj)
1311{
1312 dma_resv_unlock(obj: obj->resv);
1313}
1314EXPORT_SYMBOL(drm_gem_unlock);
1315
1316int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1317{
1318 int ret;
1319
1320 dma_resv_lock(obj: obj->resv, NULL);
1321 ret = drm_gem_vmap_locked(obj, map);
1322 dma_resv_unlock(obj: obj->resv);
1323
1324 return ret;
1325}
1326EXPORT_SYMBOL(drm_gem_vmap);
1327
1328void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1329{
1330 dma_resv_lock(obj: obj->resv, NULL);
1331 drm_gem_vunmap_locked(obj, map);
1332 dma_resv_unlock(obj: obj->resv);
1333}
1334EXPORT_SYMBOL(drm_gem_vunmap);
1335
1336/**
1337 * drm_gem_lock_reservations - Sets up the ww context and acquires
1338 * the lock on an array of GEM objects.
1339 *
1340 * Once you've locked your reservations, you'll want to set up space
1341 * for your shared fences (if applicable), submit your job, then
1342 * drm_gem_unlock_reservations().
1343 *
1344 * @objs: drm_gem_objects to lock
1345 * @count: Number of objects in @objs
1346 * @acquire_ctx: struct ww_acquire_ctx that will be initialized as
1347 * part of tracking this set of locked reservations.
1348 */
1349int
1350drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
1351 struct ww_acquire_ctx *acquire_ctx)
1352{
1353 int contended = -1;
1354 int i, ret;
1355
1356 ww_acquire_init(ctx: acquire_ctx, ww_class: &reservation_ww_class);
1357
1358retry:
1359 if (contended != -1) {
1360 struct drm_gem_object *obj = objs[contended];
1361
1362 ret = dma_resv_lock_slow_interruptible(obj: obj->resv,
1363 ctx: acquire_ctx);
1364 if (ret) {
1365 ww_acquire_fini(ctx: acquire_ctx);
1366 return ret;
1367 }
1368 }
1369
1370 for (i = 0; i < count; i++) {
1371 if (i == contended)
1372 continue;
1373
1374 ret = dma_resv_lock_interruptible(obj: objs[i]->resv,
1375 ctx: acquire_ctx);
1376 if (ret) {
1377 int j;
1378
1379 for (j = 0; j < i; j++)
1380 dma_resv_unlock(obj: objs[j]->resv);
1381
1382 if (contended != -1 && contended >= i)
1383 dma_resv_unlock(obj: objs[contended]->resv);
1384
1385 if (ret == -EDEADLK) {
1386 contended = i;
1387 goto retry;
1388 }
1389
1390 ww_acquire_fini(ctx: acquire_ctx);
1391 return ret;
1392 }
1393 }
1394
1395 ww_acquire_done(ctx: acquire_ctx);
1396
1397 return 0;
1398}
1399EXPORT_SYMBOL(drm_gem_lock_reservations);
1400
1401void
1402drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
1403 struct ww_acquire_ctx *acquire_ctx)
1404{
1405 int i;
1406
1407 for (i = 0; i < count; i++)
1408 dma_resv_unlock(obj: objs[i]->resv);
1409
1410 ww_acquire_fini(ctx: acquire_ctx);
1411}
1412EXPORT_SYMBOL(drm_gem_unlock_reservations);
1413
1414/**
1415 * drm_gem_lru_init - initialize a LRU
1416 *
1417 * @lru: The LRU to initialize
1418 * @lock: The lock protecting the LRU
1419 */
1420void
1421drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock)
1422{
1423 lru->lock = lock;
1424 lru->count = 0;
1425 INIT_LIST_HEAD(list: &lru->list);
1426}
1427EXPORT_SYMBOL(drm_gem_lru_init);
1428
1429static void
1430drm_gem_lru_remove_locked(struct drm_gem_object *obj)
1431{
1432 obj->lru->count -= obj->size >> PAGE_SHIFT;
1433 WARN_ON(obj->lru->count < 0);
1434 list_del(entry: &obj->lru_node);
1435 obj->lru = NULL;
1436}
1437
1438/**
1439 * drm_gem_lru_remove - remove object from whatever LRU it is in
1440 *
1441 * If the object is currently in any LRU, remove it.
1442 *
1443 * @obj: The GEM object to remove from current LRU
1444 */
1445void
1446drm_gem_lru_remove(struct drm_gem_object *obj)
1447{
1448 struct drm_gem_lru *lru = obj->lru;
1449
1450 if (!lru)
1451 return;
1452
1453 mutex_lock(lock: lru->lock);
1454 drm_gem_lru_remove_locked(obj);
1455 mutex_unlock(lock: lru->lock);
1456}
1457EXPORT_SYMBOL(drm_gem_lru_remove);
1458
1459/**
1460 * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU
1461 *
1462 * Like &drm_gem_lru_move_tail but lru lock must be held
1463 *
1464 * @lru: The LRU to move the object into.
1465 * @obj: The GEM object to move into this LRU
1466 */
1467void
1468drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1469{
1470 lockdep_assert_held_once(lru->lock);
1471
1472 if (obj->lru)
1473 drm_gem_lru_remove_locked(obj);
1474
1475 lru->count += obj->size >> PAGE_SHIFT;
1476 list_add_tail(new: &obj->lru_node, head: &lru->list);
1477 obj->lru = lru;
1478}
1479EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
1480
1481/**
1482 * drm_gem_lru_move_tail - move the object to the tail of the LRU
1483 *
1484 * If the object is already in this LRU it will be moved to the
1485 * tail. Otherwise it will be removed from whichever other LRU
1486 * it is in (if any) and moved into this LRU.
1487 *
1488 * @lru: The LRU to move the object into.
1489 * @obj: The GEM object to move into this LRU
1490 */
1491void
1492drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1493{
1494 mutex_lock(lock: lru->lock);
1495 drm_gem_lru_move_tail_locked(lru, obj);
1496 mutex_unlock(lock: lru->lock);
1497}
1498EXPORT_SYMBOL(drm_gem_lru_move_tail);
1499
1500/**
1501 * drm_gem_lru_scan - helper to implement shrinker.scan_objects
1502 *
1503 * If the shrink callback succeeds, it is expected that the driver
1504 * move the object out of this LRU.
1505 *
1506 * If the LRU possibly contain active buffers, it is the responsibility
1507 * of the shrink callback to check for this (ie. dma_resv_test_signaled())
1508 * or if necessary block until the buffer becomes idle.
1509 *
1510 * @lru: The LRU to scan
1511 * @nr_to_scan: The number of pages to try to reclaim
1512 * @remaining: The number of pages left to reclaim, should be initialized by caller
1513 * @shrink: Callback to try to shrink/reclaim the object.
1514 * @ticket: Optional ww_acquire_ctx context to use for locking
1515 */
1516unsigned long
1517drm_gem_lru_scan(struct drm_gem_lru *lru,
1518 unsigned int nr_to_scan,
1519 unsigned long *remaining,
1520 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
1521 struct ww_acquire_ctx *ticket)
1522{
1523 struct drm_gem_lru still_in_lru;
1524 struct drm_gem_object *obj;
1525 unsigned freed = 0;
1526
1527 drm_gem_lru_init(&still_in_lru, lru->lock);
1528
1529 mutex_lock(lock: lru->lock);
1530
1531 while (freed < nr_to_scan) {
1532 obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
1533
1534 if (!obj)
1535 break;
1536
1537 drm_gem_lru_move_tail_locked(&still_in_lru, obj);
1538
1539 /*
1540 * If it's in the process of being freed, gem_object->free()
1541 * may be blocked on lock waiting to remove it. So just
1542 * skip it.
1543 */
1544 if (!kref_get_unless_zero(kref: &obj->refcount))
1545 continue;
1546
1547 /*
1548 * Now that we own a reference, we can drop the lock for the
1549 * rest of the loop body, to reduce contention with other
1550 * code paths that need the LRU lock
1551 */
1552 mutex_unlock(lock: lru->lock);
1553
1554 if (ticket)
1555 ww_acquire_init(ctx: ticket, ww_class: &reservation_ww_class);
1556
1557 /*
1558 * Note that this still needs to be trylock, since we can
1559 * hit shrinker in response to trying to get backing pages
1560 * for this obj (ie. while it's lock is already held)
1561 */
1562 if (!ww_mutex_trylock(lock: &obj->resv->lock, ctx: ticket)) {
1563 *remaining += obj->size >> PAGE_SHIFT;
1564 goto tail;
1565 }
1566
1567 if (shrink(obj, ticket)) {
1568 freed += obj->size >> PAGE_SHIFT;
1569
1570 /*
1571 * If we succeeded in releasing the object's backing
1572 * pages, we expect the driver to have moved the object
1573 * out of this LRU
1574 */
1575 WARN_ON(obj->lru == &still_in_lru);
1576 WARN_ON(obj->lru == lru);
1577 }
1578
1579 dma_resv_unlock(obj: obj->resv);
1580
1581 if (ticket)
1582 ww_acquire_fini(ctx: ticket);
1583
1584tail:
1585 drm_gem_object_put(obj);
1586 mutex_lock(lock: lru->lock);
1587 }
1588
1589 /*
1590 * Move objects we've skipped over out of the temporary still_in_lru
1591 * back into this LRU
1592 */
1593 list_for_each_entry (obj, &still_in_lru.list, lru_node)
1594 obj->lru = lru;
1595 list_splice_tail(list: &still_in_lru.list, head: &lru->list);
1596 lru->count += still_in_lru.count;
1597
1598 mutex_unlock(lock: lru->lock);
1599
1600 return freed;
1601}
1602EXPORT_SYMBOL(drm_gem_lru_scan);
1603
1604/**
1605 * drm_gem_evict_locked - helper to evict backing pages for a GEM object
1606 * @obj: obj in question
1607 */
1608int drm_gem_evict_locked(struct drm_gem_object *obj)
1609{
1610 dma_resv_assert_held(obj->resv);
1611
1612 if (!dma_resv_test_signaled(obj: obj->resv, usage: DMA_RESV_USAGE_READ))
1613 return -EBUSY;
1614
1615 if (obj->funcs->evict)
1616 return obj->funcs->evict(obj);
1617
1618 return 0;
1619}
1620EXPORT_SYMBOL(drm_gem_evict_locked);
1621