1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * dma-fence-util: misc functions for dma_fence objects |
4 | * |
5 | * Copyright (C) 2022 Advanced Micro Devices, Inc. |
6 | * Authors: |
7 | * Christian König <christian.koenig@amd.com> |
8 | */ |
9 | |
10 | #include <linux/dma-fence.h> |
11 | #include <linux/dma-fence-array.h> |
12 | #include <linux/dma-fence-chain.h> |
13 | #include <linux/dma-fence-unwrap.h> |
14 | #include <linux/slab.h> |
15 | |
16 | /* Internal helper to start new array iteration, don't use directly */ |
17 | static struct dma_fence * |
18 | __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor) |
19 | { |
20 | cursor->array = dma_fence_chain_contained(fence: cursor->chain); |
21 | cursor->index = 0; |
22 | return dma_fence_array_first(head: cursor->array); |
23 | } |
24 | |
25 | /** |
26 | * dma_fence_unwrap_first - return the first fence from fence containers |
27 | * @head: the entrypoint into the containers |
28 | * @cursor: current position inside the containers |
29 | * |
30 | * Unwraps potential dma_fence_chain/dma_fence_array containers and return the |
31 | * first fence. |
32 | */ |
33 | struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head, |
34 | struct dma_fence_unwrap *cursor) |
35 | { |
36 | cursor->chain = dma_fence_get(fence: head); |
37 | return __dma_fence_unwrap_array(cursor); |
38 | } |
39 | EXPORT_SYMBOL_GPL(dma_fence_unwrap_first); |
40 | |
41 | /** |
42 | * dma_fence_unwrap_next - return the next fence from a fence containers |
43 | * @cursor: current position inside the containers |
44 | * |
45 | * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return |
46 | * the next fence from them. |
47 | */ |
48 | struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) |
49 | { |
50 | struct dma_fence *tmp; |
51 | |
52 | ++cursor->index; |
53 | tmp = dma_fence_array_next(head: cursor->array, index: cursor->index); |
54 | if (tmp) |
55 | return tmp; |
56 | |
57 | cursor->chain = dma_fence_chain_walk(fence: cursor->chain); |
58 | return __dma_fence_unwrap_array(cursor); |
59 | } |
60 | EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); |
61 | |
62 | /* Implementation for the dma_fence_merge() marco, don't use directly */ |
63 | struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, |
64 | struct dma_fence **fences, |
65 | struct dma_fence_unwrap *iter) |
66 | { |
67 | struct dma_fence_array *result; |
68 | struct dma_fence *tmp, **array; |
69 | ktime_t timestamp; |
70 | unsigned int i; |
71 | size_t count; |
72 | |
73 | count = 0; |
74 | timestamp = ns_to_ktime(ns: 0); |
75 | for (i = 0; i < num_fences; ++i) { |
76 | dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { |
77 | if (!dma_fence_is_signaled(fence: tmp)) { |
78 | ++count; |
79 | } else { |
80 | ktime_t t = dma_fence_timestamp(fence: tmp); |
81 | |
82 | if (ktime_after(cmp1: t, cmp2: timestamp)) |
83 | timestamp = t; |
84 | } |
85 | } |
86 | } |
87 | |
88 | /* |
89 | * If we couldn't find a pending fence just return a private signaled |
90 | * fence with the timestamp of the last signaled one. |
91 | */ |
92 | if (count == 0) |
93 | return dma_fence_allocate_private_stub(timestamp); |
94 | |
95 | array = kmalloc_array(n: count, size: sizeof(*array), GFP_KERNEL); |
96 | if (!array) |
97 | return NULL; |
98 | |
99 | /* |
100 | * This trashes the input fence array and uses it as position for the |
101 | * following merge loop. This works because the dma_fence_merge() |
102 | * wrapper macro is creating this temporary array on the stack together |
103 | * with the iterators. |
104 | */ |
105 | for (i = 0; i < num_fences; ++i) |
106 | fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]); |
107 | |
108 | count = 0; |
109 | do { |
110 | unsigned int sel; |
111 | |
112 | restart: |
113 | tmp = NULL; |
114 | for (i = 0; i < num_fences; ++i) { |
115 | struct dma_fence *next; |
116 | |
117 | while (fences[i] && dma_fence_is_signaled(fence: fences[i])) |
118 | fences[i] = dma_fence_unwrap_next(&iter[i]); |
119 | |
120 | next = fences[i]; |
121 | if (!next) |
122 | continue; |
123 | |
124 | /* |
125 | * We can't guarantee that inpute fences are ordered by |
126 | * context, but it is still quite likely when this |
127 | * function is used multiple times. So attempt to order |
128 | * the fences by context as we pass over them and merge |
129 | * fences with the same context. |
130 | */ |
131 | if (!tmp || tmp->context > next->context) { |
132 | tmp = next; |
133 | sel = i; |
134 | |
135 | } else if (tmp->context < next->context) { |
136 | continue; |
137 | |
138 | } else if (dma_fence_is_later(f1: tmp, f2: next)) { |
139 | fences[i] = dma_fence_unwrap_next(&iter[i]); |
140 | goto restart; |
141 | } else { |
142 | fences[sel] = dma_fence_unwrap_next(&iter[sel]); |
143 | goto restart; |
144 | } |
145 | } |
146 | |
147 | if (tmp) { |
148 | array[count++] = dma_fence_get(fence: tmp); |
149 | fences[sel] = dma_fence_unwrap_next(&iter[sel]); |
150 | } |
151 | } while (tmp); |
152 | |
153 | if (count == 0) { |
154 | tmp = dma_fence_allocate_private_stub(timestamp: ktime_get()); |
155 | goto return_tmp; |
156 | } |
157 | |
158 | if (count == 1) { |
159 | tmp = array[0]; |
160 | goto return_tmp; |
161 | } |
162 | |
163 | result = dma_fence_array_create(num_fences: count, fences: array, |
164 | context: dma_fence_context_alloc(num: 1), |
165 | seqno: 1, signal_on_any: false); |
166 | if (!result) { |
167 | tmp = NULL; |
168 | goto return_tmp; |
169 | } |
170 | return &result->base; |
171 | |
172 | return_tmp: |
173 | kfree(objp: array); |
174 | return tmp; |
175 | } |
176 | EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge); |
177 | |