1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5#include <drm/ttm/ttm_device.h>
6#include <drm/ttm/ttm_range_manager.h>
7
8#include "i915_drv.h"
9#include "i915_scatterlist.h"
10#include "i915_ttm_buddy_manager.h"
11
12#include "intel_region_ttm.h"
13
14#include "gem/i915_gem_region.h"
15#include "gem/i915_gem_ttm.h" /* For the funcs/ops export only */
16/**
17 * DOC: TTM support structure
18 *
19 * The code in this file deals with setting up memory managers for TTM
20 * LMEM and MOCK regions and converting the output from
21 * the managers to struct sg_table, Basically providing the mapping from
22 * i915 GEM regions to TTM memory types and resource managers.
23 */
24
25/**
26 * intel_region_ttm_device_init - Initialize a TTM device
27 * @dev_priv: Pointer to an i915 device private structure.
28 *
29 * Return: 0 on success, negative error code on failure.
30 */
31int intel_region_ttm_device_init(struct drm_i915_private *dev_priv)
32{
33 struct drm_device *drm = &dev_priv->drm;
34
35 return ttm_device_init(bdev: &dev_priv->bdev, funcs: i915_ttm_driver(),
36 dev: drm->dev, mapping: drm->anon_inode->i_mapping,
37 vma_manager: drm->vma_offset_manager, use_dma_alloc: false, use_dma32: false);
38}
39
40/**
41 * intel_region_ttm_device_fini - Finalize a TTM device
42 * @dev_priv: Pointer to an i915 device private structure.
43 */
44void intel_region_ttm_device_fini(struct drm_i915_private *dev_priv)
45{
46 ttm_device_fini(bdev: &dev_priv->bdev);
47}
48
49/*
50 * Map the i915 memory regions to TTM memory types. We use the
51 * driver-private types for now, reserving TTM_PL_VRAM for stolen
52 * memory and TTM_PL_TT for GGTT use if decided to implement this.
53 */
54int intel_region_to_ttm_type(const struct intel_memory_region *mem)
55{
56 int type;
57
58 GEM_BUG_ON(mem->type != INTEL_MEMORY_LOCAL &&
59 mem->type != INTEL_MEMORY_MOCK &&
60 mem->type != INTEL_MEMORY_SYSTEM);
61
62 if (mem->type == INTEL_MEMORY_SYSTEM)
63 return TTM_PL_SYSTEM;
64
65 type = mem->instance + TTM_PL_PRIV;
66 GEM_BUG_ON(type >= TTM_NUM_MEM_TYPES);
67
68 return type;
69}
70
71/**
72 * intel_region_ttm_init - Initialize a memory region for TTM.
73 * @mem: The region to initialize.
74 *
75 * This function initializes a suitable TTM resource manager for the
76 * region, and if it's a LMEM region type, attaches it to the TTM
77 * device. MOCK regions are NOT attached to the TTM device, since we don't
78 * have one for the mock selftests.
79 *
80 * Return: 0 on success, negative error code on failure.
81 */
82int intel_region_ttm_init(struct intel_memory_region *mem)
83{
84 struct ttm_device *bdev = &mem->i915->bdev;
85 int mem_type = intel_region_to_ttm_type(mem);
86 int ret;
87
88 ret = i915_ttm_buddy_man_init(bdev, type: mem_type, use_tt: false,
89 size: resource_size(res: &mem->region),
90 visible_size: resource_size(res: &mem->io),
91 default_page_size: mem->min_page_size, PAGE_SIZE);
92 if (ret)
93 return ret;
94
95 mem->region_private = ttm_manager_type(bdev, mem_type);
96
97 return 0;
98}
99
100/**
101 * intel_region_ttm_fini - Finalize a TTM region.
102 * @mem: The memory region
103 *
104 * This functions takes down the TTM resource manager associated with the
105 * memory region, and if it was registered with the TTM device,
106 * removes that registration.
107 */
108int intel_region_ttm_fini(struct intel_memory_region *mem)
109{
110 struct ttm_resource_manager *man = mem->region_private;
111 int ret = -EBUSY;
112 int count;
113
114 /*
115 * Put the region's move fences. This releases requests that
116 * may hold on to contexts and vms that may hold on to buffer
117 * objects placed in this region.
118 */
119 if (man)
120 ttm_resource_manager_cleanup(man);
121
122 /* Flush objects from region. */
123 for (count = 0; count < 10; ++count) {
124 i915_gem_flush_free_objects(i915: mem->i915);
125
126 mutex_lock(lock: &mem->objects.lock);
127 if (list_empty(head: &mem->objects.list))
128 ret = 0;
129 mutex_unlock(lock: &mem->objects.lock);
130 if (!ret)
131 break;
132
133 msleep(msecs: 20);
134 drain_workqueue(wq: mem->i915->bdev.wq);
135 }
136
137 /* If we leaked objects, Don't free the region causing use after free */
138 if (ret || !man)
139 return ret;
140
141 ret = i915_ttm_buddy_man_fini(bdev: &mem->i915->bdev,
142 type: intel_region_to_ttm_type(mem));
143 GEM_WARN_ON(ret);
144 mem->region_private = NULL;
145
146 return ret;
147}
148
149/**
150 * intel_region_ttm_resource_to_rsgt -
151 * Convert an opaque TTM resource manager resource to a refcounted sg_table.
152 * @mem: The memory region.
153 * @res: The resource manager resource obtained from the TTM resource manager.
154 * @page_alignment: Required page alignment for each sg entry. Power of two.
155 *
156 * The gem backends typically use sg-tables for operations on the underlying
157 * io_memory. So provide a way for the backends to translate the
158 * nodes they are handed from TTM to sg-tables.
159 *
160 * Return: A malloced sg_table on success, an error pointer on failure.
161 */
162struct i915_refct_sgt *
163intel_region_ttm_resource_to_rsgt(struct intel_memory_region *mem,
164 struct ttm_resource *res,
165 u32 page_alignment)
166{
167 if (mem->is_range_manager) {
168 struct ttm_range_mgr_node *range_node =
169 to_ttm_range_mgr_node(res);
170
171 return i915_rsgt_from_mm_node(node: &range_node->mm_nodes[0],
172 region_start: mem->region.start,
173 page_alignment);
174 } else {
175 return i915_rsgt_from_buddy_resource(res, region_start: mem->region.start,
176 page_alignment);
177 }
178}
179
180#ifdef CONFIG_DRM_I915_SELFTEST
181/**
182 * intel_region_ttm_resource_alloc - Allocate memory resources from a region
183 * @mem: The memory region,
184 * @offset: BO offset
185 * @size: The requested size in bytes
186 * @flags: Allocation flags
187 *
188 * This functionality is provided only for callers that need to allocate
189 * memory from standalone TTM range managers, without the TTM eviction
190 * functionality. Don't use if you are not completely sure that's the
191 * case. The returned opaque node can be converted to an sg_table using
192 * intel_region_ttm_resource_to_st(), and can be freed using
193 * intel_region_ttm_resource_free().
194 *
195 * Return: A valid pointer on success, an error pointer on failure.
196 */
197struct ttm_resource *
198intel_region_ttm_resource_alloc(struct intel_memory_region *mem,
199 resource_size_t offset,
200 resource_size_t size,
201 unsigned int flags)
202{
203 struct ttm_resource_manager *man = mem->region_private;
204 struct ttm_place place = {};
205 struct ttm_buffer_object mock_bo = {};
206 struct ttm_resource *res;
207 int ret;
208
209 if (flags & I915_BO_ALLOC_CONTIGUOUS)
210 place.flags |= TTM_PL_FLAG_CONTIGUOUS;
211 if (offset != I915_BO_INVALID_OFFSET) {
212 if (WARN_ON(overflows_type(offset >> PAGE_SHIFT, place.fpfn))) {
213 ret = -E2BIG;
214 goto out;
215 }
216 place.fpfn = offset >> PAGE_SHIFT;
217 if (WARN_ON(overflows_type(place.fpfn + (size >> PAGE_SHIFT), place.lpfn))) {
218 ret = -E2BIG;
219 goto out;
220 }
221 place.lpfn = place.fpfn + (size >> PAGE_SHIFT);
222 } else if (resource_size(&mem->io) && resource_size(&mem->io) < mem->total) {
223 if (flags & I915_BO_ALLOC_GPU_ONLY) {
224 place.flags |= TTM_PL_FLAG_TOPDOWN;
225 } else {
226 place.fpfn = 0;
227 if (WARN_ON(overflows_type(resource_size(&mem->io) >> PAGE_SHIFT, place.lpfn))) {
228 ret = -E2BIG;
229 goto out;
230 }
231 place.lpfn = resource_size(&mem->io) >> PAGE_SHIFT;
232 }
233 }
234
235 mock_bo.base.size = size;
236 mock_bo.bdev = &mem->i915->bdev;
237
238 ret = man->func->alloc(man, &mock_bo, &place, &res);
239
240out:
241 if (ret == -ENOSPC)
242 ret = -ENXIO;
243 if (!ret)
244 res->bo = NULL; /* Rather blow up, then some uaf */
245 return ret ? ERR_PTR(ret) : res;
246}
247
248#endif
249
250/**
251 * intel_region_ttm_resource_free - Free a resource allocated from a resource manager
252 * @mem: The region the resource was allocated from.
253 * @res: The opaque resource representing an allocation.
254 */
255void intel_region_ttm_resource_free(struct intel_memory_region *mem,
256 struct ttm_resource *res)
257{
258 struct ttm_resource_manager *man = mem->region_private;
259 struct ttm_buffer_object mock_bo = {};
260
261 mock_bo.base.size = res->size;
262 mock_bo.bdev = &mem->i915->bdev;
263 res->bo = &mock_bo;
264
265 man->func->free(man, res);
266}
267