1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * Copyright © 2016 Intel Corporation |
5 | */ |
6 | |
7 | #include "i915_scatterlist.h" |
8 | #include "i915_ttm_buddy_manager.h" |
9 | |
10 | #include <drm/drm_buddy.h> |
11 | #include <drm/drm_mm.h> |
12 | |
13 | #include <linux/slab.h> |
14 | |
15 | bool i915_sg_trim(struct sg_table *orig_st) |
16 | { |
17 | struct sg_table new_st; |
18 | struct scatterlist *sg, *new_sg; |
19 | unsigned int i; |
20 | |
21 | if (orig_st->nents == orig_st->orig_nents) |
22 | return false; |
23 | |
24 | if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN)) |
25 | return false; |
26 | |
27 | new_sg = new_st.sgl; |
28 | for_each_sg(orig_st->sgl, sg, orig_st->nents, i) { |
29 | sg_set_page(sg: new_sg, page: sg_page(sg), len: sg->length, offset: 0); |
30 | sg_dma_address(new_sg) = sg_dma_address(sg); |
31 | sg_dma_len(new_sg) = sg_dma_len(sg); |
32 | |
33 | new_sg = sg_next(new_sg); |
34 | } |
35 | GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */ |
36 | |
37 | sg_free_table(orig_st); |
38 | |
39 | *orig_st = new_st; |
40 | return true; |
41 | } |
42 | |
43 | static void i915_refct_sgt_release(struct kref *ref) |
44 | { |
45 | struct i915_refct_sgt *rsgt = |
46 | container_of(ref, typeof(*rsgt), kref); |
47 | |
48 | sg_free_table(&rsgt->table); |
49 | kfree(objp: rsgt); |
50 | } |
51 | |
52 | static const struct i915_refct_sgt_ops rsgt_ops = { |
53 | .release = i915_refct_sgt_release |
54 | }; |
55 | |
56 | /** |
57 | * i915_refct_sgt_init - Initialize a struct i915_refct_sgt with default ops |
58 | * @rsgt: The struct i915_refct_sgt to initialize. |
59 | * @size: The size of the underlying memory buffer. |
60 | */ |
61 | void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size) |
62 | { |
63 | __i915_refct_sgt_init(rsgt, size, ops: &rsgt_ops); |
64 | } |
65 | |
66 | /** |
67 | * i915_rsgt_from_mm_node - Create a refcounted sg_table from a struct |
68 | * drm_mm_node |
69 | * @node: The drm_mm_node. |
70 | * @region_start: An offset to add to the dma addresses of the sg list. |
71 | * @page_alignment: Required page alignment for each sg entry. Power of two. |
72 | * |
73 | * Create a struct sg_table, initializing it from a struct drm_mm_node, |
74 | * taking a maximum segment length into account, splitting into segments |
75 | * if necessary. |
76 | * |
77 | * Return: A pointer to a kmalloced struct i915_refct_sgt on success, negative |
78 | * error code cast to an error pointer on failure. |
79 | */ |
80 | struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, |
81 | u64 region_start, |
82 | u32 page_alignment) |
83 | { |
84 | const u32 max_segment = round_down(UINT_MAX, page_alignment); |
85 | const u32 segment_pages = max_segment >> PAGE_SHIFT; |
86 | u64 block_size, offset, prev_end; |
87 | struct i915_refct_sgt *rsgt; |
88 | struct sg_table *st; |
89 | struct scatterlist *sg; |
90 | |
91 | GEM_BUG_ON(!max_segment); |
92 | |
93 | rsgt = kmalloc(size: sizeof(*rsgt), GFP_KERNEL); |
94 | if (!rsgt) |
95 | return ERR_PTR(error: -ENOMEM); |
96 | |
97 | i915_refct_sgt_init(rsgt, size: node->size << PAGE_SHIFT); |
98 | st = &rsgt->table; |
99 | /* restricted by sg_alloc_table */ |
100 | if (WARN_ON(overflows_type(DIV_ROUND_UP_ULL(node->size, segment_pages), |
101 | unsigned int))) { |
102 | i915_refct_sgt_put(rsgt); |
103 | return ERR_PTR(error: -E2BIG); |
104 | } |
105 | |
106 | if (sg_alloc_table(st, DIV_ROUND_UP_ULL(node->size, segment_pages), |
107 | GFP_KERNEL)) { |
108 | i915_refct_sgt_put(rsgt); |
109 | return ERR_PTR(error: -ENOMEM); |
110 | } |
111 | |
112 | sg = st->sgl; |
113 | st->nents = 0; |
114 | prev_end = (resource_size_t)-1; |
115 | block_size = node->size << PAGE_SHIFT; |
116 | offset = node->start << PAGE_SHIFT; |
117 | |
118 | while (block_size) { |
119 | u64 len; |
120 | |
121 | if (offset != prev_end || sg->length >= max_segment) { |
122 | if (st->nents) |
123 | sg = __sg_next(sg); |
124 | |
125 | sg_dma_address(sg) = region_start + offset; |
126 | GEM_BUG_ON(!IS_ALIGNED(sg_dma_address(sg), |
127 | page_alignment)); |
128 | sg_dma_len(sg) = 0; |
129 | sg->length = 0; |
130 | st->nents++; |
131 | } |
132 | |
133 | len = min_t(u64, block_size, max_segment - sg->length); |
134 | sg->length += len; |
135 | sg_dma_len(sg) += len; |
136 | |
137 | offset += len; |
138 | block_size -= len; |
139 | |
140 | prev_end = offset; |
141 | } |
142 | |
143 | sg_mark_end(sg); |
144 | i915_sg_trim(orig_st: st); |
145 | |
146 | return rsgt; |
147 | } |
148 | |
149 | /** |
150 | * i915_rsgt_from_buddy_resource - Create a refcounted sg_table from a struct |
151 | * i915_buddy_block list |
152 | * @res: The struct i915_ttm_buddy_resource. |
153 | * @region_start: An offset to add to the dma addresses of the sg list. |
154 | * @page_alignment: Required page alignment for each sg entry. Power of two. |
155 | * |
156 | * Create a struct sg_table, initializing it from struct i915_buddy_block list, |
157 | * taking a maximum segment length into account, splitting into segments |
158 | * if necessary. |
159 | * |
160 | * Return: A pointer to a kmalloced struct i915_refct_sgts on success, negative |
161 | * error code cast to an error pointer on failure. |
162 | */ |
163 | struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, |
164 | u64 region_start, |
165 | u32 page_alignment) |
166 | { |
167 | struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); |
168 | const u64 size = res->size; |
169 | const u32 max_segment = round_down(UINT_MAX, page_alignment); |
170 | struct drm_buddy *mm = bman_res->mm; |
171 | struct list_head *blocks = &bman_res->blocks; |
172 | struct drm_buddy_block *block; |
173 | struct i915_refct_sgt *rsgt; |
174 | struct scatterlist *sg; |
175 | struct sg_table *st; |
176 | resource_size_t prev_end; |
177 | |
178 | GEM_BUG_ON(list_empty(blocks)); |
179 | GEM_BUG_ON(!max_segment); |
180 | |
181 | rsgt = kmalloc(size: sizeof(*rsgt), GFP_KERNEL); |
182 | if (!rsgt) |
183 | return ERR_PTR(error: -ENOMEM); |
184 | |
185 | i915_refct_sgt_init(rsgt, size); |
186 | st = &rsgt->table; |
187 | /* restricted by sg_alloc_table */ |
188 | if (WARN_ON(overflows_type(PFN_UP(res->size), unsigned int))) { |
189 | i915_refct_sgt_put(rsgt); |
190 | return ERR_PTR(error: -E2BIG); |
191 | } |
192 | |
193 | if (sg_alloc_table(st, PFN_UP(res->size), GFP_KERNEL)) { |
194 | i915_refct_sgt_put(rsgt); |
195 | return ERR_PTR(error: -ENOMEM); |
196 | } |
197 | |
198 | sg = st->sgl; |
199 | st->nents = 0; |
200 | prev_end = (resource_size_t)-1; |
201 | |
202 | list_for_each_entry(block, blocks, link) { |
203 | u64 block_size, offset; |
204 | |
205 | block_size = min_t(u64, size, drm_buddy_block_size(mm, block)); |
206 | offset = drm_buddy_block_offset(block); |
207 | |
208 | while (block_size) { |
209 | u64 len; |
210 | |
211 | if (offset != prev_end || sg->length >= max_segment) { |
212 | if (st->nents) |
213 | sg = __sg_next(sg); |
214 | |
215 | sg_dma_address(sg) = region_start + offset; |
216 | GEM_BUG_ON(!IS_ALIGNED(sg_dma_address(sg), |
217 | page_alignment)); |
218 | sg_dma_len(sg) = 0; |
219 | sg->length = 0; |
220 | st->nents++; |
221 | } |
222 | |
223 | len = min_t(u64, block_size, max_segment - sg->length); |
224 | sg->length += len; |
225 | sg_dma_len(sg) += len; |
226 | |
227 | offset += len; |
228 | block_size -= len; |
229 | |
230 | prev_end = offset; |
231 | } |
232 | } |
233 | |
234 | sg_mark_end(sg); |
235 | i915_sg_trim(orig_st: st); |
236 | |
237 | return rsgt; |
238 | } |
239 | |
240 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
241 | #include "selftests/scatterlist.c" |
242 | #endif |
243 | |