1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2014-2016 Intel Corporation
4 */
5
6#include <drm/drm_cache.h>
7#include <drm/drm_panic.h>
8#include <linux/vmalloc.h>
9
10#include "display/intel_fb.h"
11#include "display/intel_display_types.h"
12#include "gt/intel_gt.h"
13#include "gt/intel_tlb.h"
14
15#include "i915_drv.h"
16#include "i915_gem_object.h"
17#include "i915_scatterlist.h"
18#include "i915_gem_lmem.h"
19#include "i915_gem_mman.h"
20
21void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
22 struct sg_table *pages)
23{
24 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
25 unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
26 bool shrinkable;
27 int i;
28
29 assert_object_held_shared(obj);
30
31 if (i915_gem_object_is_volatile(obj))
32 obj->mm.madv = I915_MADV_DONTNEED;
33
34 /* Make the pages coherent with the GPU (flushing any swapin). */
35 if (obj->cache_dirty) {
36 WARN_ON_ONCE(IS_DGFX(i915));
37 obj->write_domain = 0;
38 if (i915_gem_object_has_struct_page(obj))
39 drm_clflush_sg(st: pages);
40 obj->cache_dirty = false;
41 }
42
43 obj->mm.get_page.sg_pos = pages->sgl;
44 obj->mm.get_page.sg_idx = 0;
45 obj->mm.get_dma_page.sg_pos = pages->sgl;
46 obj->mm.get_dma_page.sg_idx = 0;
47
48 obj->mm.pages = pages;
49
50 obj->mm.page_sizes.phys = i915_sg_dma_sizes(sg: pages->sgl);
51 GEM_BUG_ON(!obj->mm.page_sizes.phys);
52
53 /*
54 * Calculate the supported page-sizes which fit into the given
55 * sg_page_sizes. This will give us the page-sizes which we may be able
56 * to use opportunistically when later inserting into the GTT. For
57 * example if phys=2G, then in theory we should be able to use 1G, 2M,
58 * 64K or 4K pages, although in practice this will depend on a number of
59 * other factors.
60 */
61 obj->mm.page_sizes.sg = 0;
62 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
63 if (obj->mm.page_sizes.phys & ~0u << i)
64 obj->mm.page_sizes.sg |= BIT(i);
65 }
66 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
67
68 shrinkable = i915_gem_object_is_shrinkable(obj);
69
70 if (i915_gem_object_is_tiled(obj) &&
71 i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
72 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
73 i915_gem_object_set_tiling_quirk(obj);
74 GEM_BUG_ON(!list_empty(&obj->mm.link));
75 atomic_inc(v: &obj->mm.shrink_pin);
76 shrinkable = false;
77 }
78
79 if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) {
80 struct list_head *list;
81 unsigned long flags;
82
83 assert_object_held(obj);
84 spin_lock_irqsave(&i915->mm.obj_lock, flags);
85
86 i915->mm.shrink_count++;
87 i915->mm.shrink_memory += obj->base.size;
88
89 if (obj->mm.madv != I915_MADV_WILLNEED)
90 list = &i915->mm.purge_list;
91 else
92 list = &i915->mm.shrink_list;
93 list_add_tail(new: &obj->mm.link, head: list);
94
95 atomic_set(v: &obj->mm.shrink_pin, i: 0);
96 spin_unlock_irqrestore(lock: &i915->mm.obj_lock, flags);
97 }
98}
99
100int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
101{
102 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
103 int err;
104
105 assert_object_held_shared(obj);
106
107 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
108 drm_dbg(&i915->drm,
109 "Attempting to obtain a purgeable object\n");
110 return -EFAULT;
111 }
112
113 err = obj->ops->get_pages(obj);
114 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
115
116 return err;
117}
118
119/* Ensure that the associated pages are gathered from the backing storage
120 * and pinned into our object. i915_gem_object_pin_pages() may be called
121 * multiple times before they are released by a single call to
122 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
123 * either as a result of memory pressure (reaping pages under the shrinker)
124 * or as the object is itself released.
125 */
126int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
127{
128 int err;
129
130 assert_object_held(obj);
131
132 assert_object_held_shared(obj);
133
134 if (unlikely(!i915_gem_object_has_pages(obj))) {
135 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
136
137 err = ____i915_gem_object_get_pages(obj);
138 if (err)
139 return err;
140
141 smp_mb__before_atomic();
142 }
143 atomic_inc(v: &obj->mm.pages_pin_count);
144
145 return 0;
146}
147
148int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
149{
150 struct i915_gem_ww_ctx ww;
151 int err;
152
153 i915_gem_ww_ctx_init(ctx: &ww, intr: true);
154retry:
155 err = i915_gem_object_lock(obj, ww: &ww);
156 if (!err)
157 err = i915_gem_object_pin_pages(obj);
158
159 if (err == -EDEADLK) {
160 err = i915_gem_ww_ctx_backoff(ctx: &ww);
161 if (!err)
162 goto retry;
163 }
164 i915_gem_ww_ctx_fini(ctx: &ww);
165 return err;
166}
167
168/* Immediately discard the backing storage */
169int i915_gem_object_truncate(struct drm_i915_gem_object *obj)
170{
171 if (obj->ops->truncate)
172 return obj->ops->truncate(obj);
173
174 return 0;
175}
176
177static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
178{
179 struct radix_tree_iter iter;
180 void __rcu **slot;
181
182 rcu_read_lock();
183 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
184 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
185 radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)
186 radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
187 rcu_read_unlock();
188}
189
190static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
191{
192 if (is_vmalloc_addr(x: ptr))
193 vunmap(addr: ptr);
194}
195
196static void flush_tlb_invalidate(struct drm_i915_gem_object *obj)
197{
198 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
199 struct intel_gt *gt;
200 int id;
201
202 for_each_gt(gt, i915, id) {
203 if (!obj->mm.tlb[id])
204 continue;
205
206 intel_gt_invalidate_tlb_full(gt, seqno: obj->mm.tlb[id]);
207 obj->mm.tlb[id] = 0;
208 }
209}
210
211struct sg_table *
212__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
213{
214 struct sg_table *pages;
215
216 assert_object_held_shared(obj);
217
218 pages = fetch_and_zero(&obj->mm.pages);
219 if (IS_ERR_OR_NULL(ptr: pages))
220 return pages;
221
222 if (i915_gem_object_is_volatile(obj))
223 obj->mm.madv = I915_MADV_WILLNEED;
224
225 if (!i915_gem_object_has_self_managed_shrink_list(obj))
226 i915_gem_object_make_unshrinkable(obj);
227
228 if (obj->mm.mapping) {
229 unmap_object(obj, page_mask_bits(obj->mm.mapping));
230 obj->mm.mapping = NULL;
231 }
232
233 __i915_gem_object_reset_page_iter(obj);
234 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
235
236 flush_tlb_invalidate(obj);
237
238 return pages;
239}
240
241int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
242{
243 struct sg_table *pages;
244
245 if (i915_gem_object_has_pinned_pages(obj))
246 return -EBUSY;
247
248 /* May be called by shrinker from within get_pages() (on another bo) */
249 assert_object_held_shared(obj);
250
251 i915_gem_object_release_mmap_offset(obj);
252
253 /*
254 * ->put_pages might need to allocate memory for the bit17 swizzle
255 * array, hence protect them from being reaped by removing them from gtt
256 * lists early.
257 */
258 pages = __i915_gem_object_unset_pages(obj);
259
260 /*
261 * XXX Temporary hijinx to avoid updating all backends to handle
262 * NULL pages. In the future, when we have more asynchronous
263 * get_pages backends we should be better able to handle the
264 * cancellation of the async task in a more uniform manner.
265 */
266 if (!IS_ERR_OR_NULL(ptr: pages))
267 obj->ops->put_pages(obj, pages);
268
269 return 0;
270}
271
272/* The 'mapping' part of i915_gem_object_pin_map() below */
273static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
274 enum i915_map_type type)
275{
276 unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
277 struct page *stack[32], **pages = stack, *page;
278 struct sgt_iter iter;
279 pgprot_t pgprot;
280 void *vaddr;
281
282 switch (type) {
283 default:
284 MISSING_CASE(type);
285 fallthrough; /* to use PAGE_KERNEL anyway */
286 case I915_MAP_WB:
287 /*
288 * On 32b, highmem using a finite set of indirect PTE (i.e.
289 * vmap) to provide virtual mappings of the high pages.
290 * As these are finite, map_new_virtual() must wait for some
291 * other kmap() to finish when it runs out. If we map a large
292 * number of objects, there is no method for it to tell us
293 * to release the mappings, and we deadlock.
294 *
295 * However, if we make an explicit vmap of the page, that
296 * uses a larger vmalloc arena, and also has the ability
297 * to tell us to release unwanted mappings. Most importantly,
298 * it will fail and propagate an error instead of waiting
299 * forever.
300 *
301 * So if the page is beyond the 32b boundary, make an explicit
302 * vmap.
303 */
304 if (n_pages == 1 && !PageHighMem(page: sg_page(sg: obj->mm.pages->sgl)))
305 return page_address(sg_page(obj->mm.pages->sgl));
306 pgprot = PAGE_KERNEL;
307 break;
308 case I915_MAP_WC:
309 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
310 break;
311 }
312
313 if (n_pages > ARRAY_SIZE(stack)) {
314 /* Too big for stack -- allocate temporary array instead */
315 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
316 if (!pages)
317 return ERR_PTR(error: -ENOMEM);
318 }
319
320 i = 0;
321 for_each_sgt_page(page, iter, obj->mm.pages)
322 pages[i++] = page;
323 vaddr = vmap(pages, count: n_pages, flags: 0, prot: pgprot);
324 if (pages != stack)
325 kvfree(addr: pages);
326
327 return vaddr ?: ERR_PTR(error: -ENOMEM);
328}
329
330static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
331 enum i915_map_type type)
332{
333 resource_size_t iomap = obj->mm.region->iomap.base -
334 obj->mm.region->region.start;
335 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
336 unsigned long stack[32], *pfns = stack, i;
337 struct sgt_iter iter;
338 dma_addr_t addr;
339 void *vaddr;
340
341 GEM_BUG_ON(type != I915_MAP_WC);
342
343 if (n_pfn > ARRAY_SIZE(stack)) {
344 /* Too big for stack -- allocate temporary array instead */
345 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
346 if (!pfns)
347 return ERR_PTR(error: -ENOMEM);
348 }
349
350 i = 0;
351 for_each_sgt_daddr(addr, iter, obj->mm.pages)
352 pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
353 vaddr = vmap_pfn(pfns, count: n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
354 if (pfns != stack)
355 kvfree(addr: pfns);
356
357 return vaddr ?: ERR_PTR(error: -ENOMEM);
358}
359
360struct intel_panic {
361 struct page **pages;
362 int page;
363 void *vaddr;
364};
365
366static void i915_panic_kunmap(struct intel_panic *panic)
367{
368 if (panic->vaddr) {
369 drm_clflush_virt_range(addr: panic->vaddr, PAGE_SIZE);
370 kunmap_local(panic->vaddr);
371 panic->vaddr = NULL;
372 }
373}
374
375static struct page **i915_gem_object_panic_pages(struct drm_i915_gem_object *obj)
376{
377 unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
378 struct page *page;
379 struct page **pages;
380 struct sgt_iter iter;
381
382 /* For a 3840x2160 32 bits Framebuffer, this should require ~64K */
383 pages = kmalloc_array(n_pages, sizeof(*pages), GFP_ATOMIC);
384 if (!pages)
385 return NULL;
386
387 i = 0;
388 for_each_sgt_page(page, iter, obj->mm.pages)
389 pages[i++] = page;
390 return pages;
391}
392
393static void i915_gem_object_panic_map_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
394 unsigned int y, u32 color)
395{
396 struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
397 unsigned int offset = fb->panic_tiling(sb->width, x, y);
398
399 iosys_map_wr(&sb->map[0], offset, u32, color);
400}
401
402/*
403 * The scanout buffer pages are not mapped, so for each pixel,
404 * use kmap_local_page_try_from_panic() to map the page, and write the pixel.
405 * Try to keep the map from the previous pixel, to avoid too much map/unmap.
406 */
407static void i915_gem_object_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
408 unsigned int y, u32 color)
409{
410 unsigned int new_page;
411 unsigned int offset;
412 struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
413 struct intel_panic *panic = fb->panic;
414
415 if (fb->panic_tiling)
416 offset = fb->panic_tiling(sb->width, x, y);
417 else
418 offset = y * sb->pitch[0] + x * sb->format->cpp[0];
419
420 new_page = offset >> PAGE_SHIFT;
421 offset = offset % PAGE_SIZE;
422 if (new_page != panic->page) {
423 i915_panic_kunmap(panic);
424 panic->page = new_page;
425 panic->vaddr =
426 kmap_local_page_try_from_panic(page: panic->pages[panic->page]);
427 }
428 if (panic->vaddr) {
429 u32 *pix = panic->vaddr + offset;
430 *pix = color;
431 }
432}
433
434struct intel_panic *i915_gem_object_alloc_panic(void)
435{
436 struct intel_panic *panic;
437
438 panic = kzalloc(sizeof(*panic), GFP_KERNEL);
439
440 return panic;
441}
442
443/*
444 * Setup the gem framebuffer for drm_panic access.
445 * Use current vaddr if it exists, or setup a list of pages.
446 * pfn is not supported yet.
447 */
448int i915_gem_object_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb,
449 struct drm_gem_object *_obj, bool panic_tiling)
450{
451 enum i915_map_type has_type;
452 struct drm_i915_gem_object *obj = to_intel_bo(gem: _obj);
453 void *ptr;
454
455 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
456 if (ptr) {
457 if (i915_gem_object_has_iomem(obj))
458 iosys_map_set_vaddr_iomem(map: &sb->map[0], vaddr_iomem: (void __iomem *)ptr);
459 else
460 iosys_map_set_vaddr(map: &sb->map[0], vaddr: ptr);
461
462 if (panic_tiling)
463 sb->set_pixel = i915_gem_object_panic_map_set_pixel;
464 return 0;
465 }
466 if (i915_gem_object_has_struct_page(obj)) {
467 panic->pages = i915_gem_object_panic_pages(obj);
468 if (!panic->pages)
469 return -ENOMEM;
470 panic->page = -1;
471 sb->set_pixel = i915_gem_object_panic_page_set_pixel;
472 return 0;
473 }
474 return -EOPNOTSUPP;
475}
476
477void i915_gem_object_panic_finish(struct intel_panic *panic)
478{
479 i915_panic_kunmap(panic);
480 panic->page = -1;
481 kfree(objp: panic->pages);
482 panic->pages = NULL;
483}
484
485/* get, pin, and map the pages of the object into kernel space */
486void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
487 enum i915_map_type type)
488{
489 enum i915_map_type has_type;
490 bool pinned;
491 void *ptr;
492 int err;
493
494 if (!i915_gem_object_has_struct_page(obj) &&
495 !i915_gem_object_has_iomem(obj))
496 return ERR_PTR(error: -ENXIO);
497
498 if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY))
499 return ERR_PTR(error: -EINVAL);
500
501 assert_object_held(obj);
502
503 pinned = !(type & I915_MAP_OVERRIDE);
504 type &= ~I915_MAP_OVERRIDE;
505
506 if (!atomic_inc_not_zero(v: &obj->mm.pages_pin_count)) {
507 if (unlikely(!i915_gem_object_has_pages(obj))) {
508 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
509
510 err = ____i915_gem_object_get_pages(obj);
511 if (err)
512 return ERR_PTR(error: err);
513
514 smp_mb__before_atomic();
515 }
516 atomic_inc(v: &obj->mm.pages_pin_count);
517 pinned = false;
518 }
519 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
520
521 /*
522 * For discrete our CPU mappings needs to be consistent in order to
523 * function correctly on !x86. When mapping things through TTM, we use
524 * the same rules to determine the caching type.
525 *
526 * The caching rules, starting from DG1:
527 *
528 * - If the object can be placed in device local-memory, then the
529 * pages should be allocated and mapped as write-combined only.
530 *
531 * - Everything else is always allocated and mapped as write-back,
532 * with the guarantee that everything is also coherent with the
533 * GPU.
534 *
535 * Internal users of lmem are already expected to get this right, so no
536 * fudging needed there.
537 */
538 if (i915_gem_object_placement_possible(obj, type: INTEL_MEMORY_LOCAL)) {
539 if (type != I915_MAP_WC && !obj->mm.n_placements) {
540 ptr = ERR_PTR(error: -ENODEV);
541 goto err_unpin;
542 }
543
544 type = I915_MAP_WC;
545 } else if (IS_DGFX(to_i915(obj->base.dev))) {
546 type = I915_MAP_WB;
547 }
548
549 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
550 if (ptr && has_type != type) {
551 if (pinned) {
552 ptr = ERR_PTR(error: -EBUSY);
553 goto err_unpin;
554 }
555
556 unmap_object(obj, ptr);
557
558 ptr = obj->mm.mapping = NULL;
559 }
560
561 if (!ptr) {
562 err = i915_gem_object_wait_moving_fence(obj, intr: true);
563 if (err) {
564 ptr = ERR_PTR(error: err);
565 goto err_unpin;
566 }
567
568 if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled()))
569 ptr = ERR_PTR(error: -ENODEV);
570 else if (i915_gem_object_has_struct_page(obj))
571 ptr = i915_gem_object_map_page(obj, type);
572 else
573 ptr = i915_gem_object_map_pfn(obj, type);
574 if (IS_ERR(ptr))
575 goto err_unpin;
576
577 obj->mm.mapping = page_pack_bits(ptr, type);
578 }
579
580 return ptr;
581
582err_unpin:
583 atomic_dec(v: &obj->mm.pages_pin_count);
584 return ptr;
585}
586
587void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
588 enum i915_map_type type)
589{
590 void *ret;
591
592 i915_gem_object_lock(obj, NULL);
593 ret = i915_gem_object_pin_map(obj, type);
594 i915_gem_object_unlock(obj);
595
596 return ret;
597}
598
599void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
600 unsigned long offset,
601 unsigned long size)
602{
603 enum i915_map_type has_type;
604 void *ptr;
605
606 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
607 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
608 offset, size, obj->base.size));
609
610 wmb(); /* let all previous writes be visible to coherent partners */
611 obj->mm.dirty = true;
612
613 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
614 return;
615
616 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
617 if (has_type == I915_MAP_WC)
618 return;
619
620 drm_clflush_virt_range(addr: ptr + offset, length: size);
621 if (size == obj->base.size) {
622 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
623 obj->cache_dirty = false;
624 }
625}
626
627void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
628{
629 GEM_BUG_ON(!obj->mm.mapping);
630
631 /*
632 * We allow removing the mapping from underneath pinned pages!
633 *
634 * Furthermore, since this is an unsafe operation reserved only
635 * for construction time manipulation, we ignore locking prudence.
636 */
637 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
638
639 i915_gem_object_unpin_map(obj);
640}
641
642struct scatterlist *
643__i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj,
644 struct i915_gem_object_page_iter *iter,
645 pgoff_t n,
646 unsigned int *offset)
647
648{
649 const bool dma = iter == &obj->mm.get_dma_page ||
650 iter == &obj->ttm.get_io_page;
651 unsigned int idx, count;
652 struct scatterlist *sg;
653
654 might_sleep();
655 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
656 if (!i915_gem_object_has_pinned_pages(obj))
657 assert_object_held(obj);
658
659 /* As we iterate forward through the sg, we record each entry in a
660 * radixtree for quick repeated (backwards) lookups. If we have seen
661 * this index previously, we will have an entry for it.
662 *
663 * Initial lookup is O(N), but this is amortized to O(1) for
664 * sequential page access (where each new request is consecutive
665 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
666 * i.e. O(1) with a large constant!
667 */
668 if (n < READ_ONCE(iter->sg_idx))
669 goto lookup;
670
671 mutex_lock(lock: &iter->lock);
672
673 /* We prefer to reuse the last sg so that repeated lookup of this
674 * (or the subsequent) sg are fast - comparing against the last
675 * sg is faster than going through the radixtree.
676 */
677
678 sg = iter->sg_pos;
679 idx = iter->sg_idx;
680 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
681
682 while (idx + count <= n) {
683 void *entry;
684 unsigned long i;
685 int ret;
686
687 /* If we cannot allocate and insert this entry, or the
688 * individual pages from this range, cancel updating the
689 * sg_idx so that on this lookup we are forced to linearly
690 * scan onwards, but on future lookups we will try the
691 * insertion again (in which case we need to be careful of
692 * the error return reporting that we have already inserted
693 * this index).
694 */
695 ret = radix_tree_insert(&iter->radix, index: idx, sg);
696 if (ret && ret != -EEXIST)
697 goto scan;
698
699 entry = xa_mk_value(v: idx);
700 for (i = 1; i < count; i++) {
701 ret = radix_tree_insert(&iter->radix, index: idx + i, entry);
702 if (ret && ret != -EEXIST)
703 goto scan;
704 }
705
706 idx += count;
707 sg = ____sg_next(sg);
708 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
709 }
710
711scan:
712 iter->sg_pos = sg;
713 iter->sg_idx = idx;
714
715 mutex_unlock(lock: &iter->lock);
716
717 if (unlikely(n < idx)) /* insertion completed by another thread */
718 goto lookup;
719
720 /* In case we failed to insert the entry into the radixtree, we need
721 * to look beyond the current sg.
722 */
723 while (idx + count <= n) {
724 idx += count;
725 sg = ____sg_next(sg);
726 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
727 }
728
729 *offset = n - idx;
730 return sg;
731
732lookup:
733 rcu_read_lock();
734
735 sg = radix_tree_lookup(&iter->radix, n);
736 GEM_BUG_ON(!sg);
737
738 /* If this index is in the middle of multi-page sg entry,
739 * the radix tree will contain a value entry that points
740 * to the start of that range. We will return the pointer to
741 * the base page and the offset of this page within the
742 * sg entry's range.
743 */
744 *offset = 0;
745 if (unlikely(xa_is_value(sg))) {
746 unsigned long base = xa_to_value(entry: sg);
747
748 sg = radix_tree_lookup(&iter->radix, base);
749 GEM_BUG_ON(!sg);
750
751 *offset = n - base;
752 }
753
754 rcu_read_unlock();
755
756 return sg;
757}
758
759struct page *
760__i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n)
761{
762 struct scatterlist *sg;
763 unsigned int offset;
764
765 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
766
767 sg = i915_gem_object_get_sg(obj, n, &offset);
768 return sg_page(sg) + offset;
769}
770
771/* Like i915_gem_object_get_page(), but mark the returned page dirty */
772struct page *
773__i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n)
774{
775 struct page *page;
776
777 page = i915_gem_object_get_page(obj, n);
778 if (!obj->mm.dirty)
779 set_page_dirty(page);
780
781 return page;
782}
783
784dma_addr_t
785__i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
786 pgoff_t n, unsigned int *len)
787{
788 struct scatterlist *sg;
789 unsigned int offset;
790
791 sg = i915_gem_object_get_sg_dma(obj, n, &offset);
792
793 if (len)
794 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
795
796 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
797}
798
799dma_addr_t
800__i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n)
801{
802 return i915_gem_object_get_dma_address_len(obj, n, NULL);
803}
804