1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * dma-fence-util: misc functions for dma_fence objects
4 *
5 * Copyright (C) 2022 Advanced Micro Devices, Inc.
6 * Authors:
7 * Christian König <christian.koenig@amd.com>
8 */
9
10#include <linux/dma-fence.h>
11#include <linux/dma-fence-array.h>
12#include <linux/dma-fence-chain.h>
13#include <linux/dma-fence-unwrap.h>
14#include <linux/slab.h>
15#include <linux/sort.h>
16
17/* Internal helper to start new array iteration, don't use directly */
18static struct dma_fence *
19__dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
20{
21 cursor->array = dma_fence_chain_contained(fence: cursor->chain);
22 cursor->index = 0;
23 return dma_fence_array_first(head: cursor->array);
24}
25
26/**
27 * dma_fence_unwrap_first - return the first fence from fence containers
28 * @head: the entrypoint into the containers
29 * @cursor: current position inside the containers
30 *
31 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
32 * first fence.
33 */
34struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
35 struct dma_fence_unwrap *cursor)
36{
37 cursor->chain = dma_fence_get(fence: head);
38 return __dma_fence_unwrap_array(cursor);
39}
40EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
41
42/**
43 * dma_fence_unwrap_next - return the next fence from a fence containers
44 * @cursor: current position inside the containers
45 *
46 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
47 * the next fence from them.
48 */
49struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
50{
51 struct dma_fence *tmp;
52
53 ++cursor->index;
54 tmp = dma_fence_array_next(head: cursor->array, index: cursor->index);
55 if (tmp)
56 return tmp;
57
58 cursor->chain = dma_fence_chain_walk(fence: cursor->chain);
59 return __dma_fence_unwrap_array(cursor);
60}
61EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
62
63
64static int fence_cmp(const void *_a, const void *_b)
65{
66 struct dma_fence *a = *(struct dma_fence **)_a;
67 struct dma_fence *b = *(struct dma_fence **)_b;
68
69 if (a->context < b->context)
70 return -1;
71 else if (a->context > b->context)
72 return 1;
73
74 if (dma_fence_is_later(f1: b, f2: a))
75 return 1;
76 else if (dma_fence_is_later(f1: a, f2: b))
77 return -1;
78
79 return 0;
80}
81
82/**
83 * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers
84 * @fences: Array of dma_fence pointers to be deduplicated
85 * @num_fences: Number of entries in the @fences array
86 *
87 * Sorts the input array by context, then removes duplicate
88 * fences with the same context, keeping only the most recent one.
89 *
90 * The array is modified in-place and unreferenced duplicate fences are released
91 * via dma_fence_put(). The function returns the new number of fences after
92 * deduplication.
93 *
94 * Return: Number of unique fences remaining in the array.
95 */
96int dma_fence_dedup_array(struct dma_fence **fences, int num_fences)
97{
98 int i, j;
99
100 sort(base: fences, num: num_fences, size: sizeof(*fences), cmp_func: fence_cmp, NULL);
101
102 /*
103 * Only keep the most recent fence for each context.
104 */
105 j = 0;
106 for (i = 1; i < num_fences; i++) {
107 if (fences[i]->context == fences[j]->context)
108 dma_fence_put(fence: fences[i]);
109 else
110 fences[++j] = fences[i];
111 }
112
113 return ++j;
114}
115EXPORT_SYMBOL_GPL(dma_fence_dedup_array);
116
117/* Implementation for the dma_fence_merge() marco, don't use directly */
118struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
119 struct dma_fence **fences,
120 struct dma_fence_unwrap *iter)
121{
122 struct dma_fence *tmp, *unsignaled = NULL, **array;
123 struct dma_fence_array *result;
124 ktime_t timestamp;
125 int i, count;
126
127 count = 0;
128 timestamp = ns_to_ktime(ns: 0);
129 for (i = 0; i < num_fences; ++i) {
130 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
131 if (!dma_fence_is_signaled(fence: tmp)) {
132 dma_fence_put(fence: unsignaled);
133 unsignaled = dma_fence_get(fence: tmp);
134 ++count;
135 } else {
136 ktime_t t = dma_fence_timestamp(fence: tmp);
137
138 if (ktime_after(cmp1: t, cmp2: timestamp))
139 timestamp = t;
140 }
141 }
142 }
143
144 /*
145 * If we couldn't find a pending fence just return a private signaled
146 * fence with the timestamp of the last signaled one.
147 *
148 * Or if there was a single unsignaled fence left we can return it
149 * directly and early since that is a major path on many workloads.
150 */
151 if (count == 0)
152 return dma_fence_allocate_private_stub(timestamp);
153 else if (count == 1)
154 return unsignaled;
155
156 dma_fence_put(fence: unsignaled);
157
158 array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
159 if (!array)
160 return NULL;
161
162 count = 0;
163 for (i = 0; i < num_fences; ++i) {
164 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
165 if (!dma_fence_is_signaled(fence: tmp)) {
166 array[count++] = dma_fence_get(fence: tmp);
167 } else {
168 ktime_t t = dma_fence_timestamp(fence: tmp);
169
170 if (ktime_after(cmp1: t, cmp2: timestamp))
171 timestamp = t;
172 }
173 }
174 }
175
176 if (count == 0 || count == 1)
177 goto return_fastpath;
178
179 count = dma_fence_dedup_array(array, count);
180
181 if (count > 1) {
182 result = dma_fence_array_create(num_fences: count, fences: array,
183 context: dma_fence_context_alloc(num: 1),
184 seqno: 1, signal_on_any: false);
185 if (!result) {
186 for (i = 0; i < count; i++)
187 dma_fence_put(fence: array[i]);
188 tmp = NULL;
189 goto return_tmp;
190 }
191 return &result->base;
192 }
193
194return_fastpath:
195 if (count == 0)
196 tmp = dma_fence_allocate_private_stub(timestamp);
197 else
198 tmp = array[0];
199
200return_tmp:
201 kfree(objp: array);
202 return tmp;
203}
204EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);
205