1/* SPDX-License-Identifier: MIT */
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#ifndef _XE_BO_H_
7#define _XE_BO_H_
8
9#include <drm/ttm/ttm_tt.h>
10
11#include "xe_bo_types.h"
12#include "xe_macros.h"
13#include "xe_vm_types.h"
14#include "xe_vm.h"
15
16#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */
17
18#define XE_BO_FLAG_USER BIT(0)
19/* The bits below need to be contiguous, or things break */
20#define XE_BO_FLAG_SYSTEM BIT(1)
21#define XE_BO_FLAG_VRAM0 BIT(2)
22#define XE_BO_FLAG_VRAM1 BIT(3)
23#define XE_BO_FLAG_VRAM_MASK (XE_BO_FLAG_VRAM0 | XE_BO_FLAG_VRAM1)
24/* -- */
25#define XE_BO_FLAG_STOLEN BIT(4)
26#define XE_BO_FLAG_VRAM_IF_DGFX(tile) (IS_DGFX(tile_to_xe(tile)) ? \
27 XE_BO_FLAG_VRAM0 << (tile)->id : \
28 XE_BO_FLAG_SYSTEM)
29#define XE_BO_FLAG_GGTT BIT(5)
30#define XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE BIT(6)
31#define XE_BO_FLAG_PINNED BIT(7)
32#define XE_BO_FLAG_NO_RESV_EVICT BIT(8)
33#define XE_BO_FLAG_DEFER_BACKING BIT(9)
34#define XE_BO_FLAG_SCANOUT BIT(10)
35#define XE_BO_FLAG_FIXED_PLACEMENT BIT(11)
36#define XE_BO_FLAG_PAGETABLE BIT(12)
37#define XE_BO_FLAG_NEEDS_CPU_ACCESS BIT(13)
38#define XE_BO_FLAG_NEEDS_UC BIT(14)
39#define XE_BO_FLAG_NEEDS_64K BIT(15)
40#define XE_BO_FLAG_NEEDS_2M BIT(16)
41#define XE_BO_FLAG_GGTT_INVALIDATE BIT(17)
42#define XE_BO_FLAG_PINNED_NORESTORE BIT(18)
43#define XE_BO_FLAG_PINNED_LATE_RESTORE BIT(19)
44#define XE_BO_FLAG_GGTT0 BIT(20)
45#define XE_BO_FLAG_GGTT1 BIT(21)
46#define XE_BO_FLAG_GGTT2 BIT(22)
47#define XE_BO_FLAG_GGTT3 BIT(23)
48#define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24)
49
50/* this one is trigger internally only */
51#define XE_BO_FLAG_INTERNAL_TEST BIT(30)
52#define XE_BO_FLAG_INTERNAL_64K BIT(31)
53
54#define XE_BO_FLAG_GGTT_ALL (XE_BO_FLAG_GGTT0 | \
55 XE_BO_FLAG_GGTT1 | \
56 XE_BO_FLAG_GGTT2 | \
57 XE_BO_FLAG_GGTT3)
58
59#define XE_BO_FLAG_GGTTx(tile) \
60 (XE_BO_FLAG_GGTT0 << (tile)->id)
61
62#define XE_PTE_SHIFT 12
63#define XE_PAGE_SIZE (1 << XE_PTE_SHIFT)
64#define XE_PTE_MASK (XE_PAGE_SIZE - 1)
65#define XE_PDE_SHIFT (XE_PTE_SHIFT - 3)
66#define XE_PDES (1 << XE_PDE_SHIFT)
67#define XE_PDE_MASK (XE_PDES - 1)
68
69#define XE_64K_PTE_SHIFT 16
70#define XE_64K_PAGE_SIZE (1 << XE_64K_PTE_SHIFT)
71#define XE_64K_PTE_MASK (XE_64K_PAGE_SIZE - 1)
72#define XE_64K_PDE_MASK (XE_PDE_MASK >> 4)
73
74#define XE_PL_SYSTEM TTM_PL_SYSTEM
75#define XE_PL_TT TTM_PL_TT
76#define XE_PL_VRAM0 TTM_PL_VRAM
77#define XE_PL_VRAM1 (XE_PL_VRAM0 + 1)
78#define XE_PL_STOLEN (TTM_NUM_MEM_TYPES - 1)
79
80#define XE_BO_PROPS_INVALID (-1)
81
82#define XE_PCI_BARRIER_MMAP_OFFSET (0x50 << XE_PTE_SHIFT)
83
84struct sg_table;
85
86struct xe_bo *xe_bo_alloc(void);
87void xe_bo_free(struct xe_bo *bo);
88
89struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
90 struct xe_tile *tile, struct dma_resv *resv,
91 struct ttm_lru_bulk_move *bulk, size_t size,
92 u16 cpu_caching, enum ttm_bo_type type,
93 u32 flags);
94struct xe_bo *
95xe_bo_create_locked_range(struct xe_device *xe,
96 struct xe_tile *tile, struct xe_vm *vm,
97 size_t size, u64 start, u64 end,
98 enum ttm_bo_type type, u32 flags, u64 alignment);
99struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,
100 struct xe_vm *vm, size_t size,
101 enum ttm_bo_type type, u32 flags);
102struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile,
103 struct xe_vm *vm, size_t size,
104 enum ttm_bo_type type, u32 flags);
105struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile,
106 struct xe_vm *vm, size_t size,
107 u16 cpu_caching,
108 u32 flags);
109struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
110 struct xe_vm *vm, size_t size,
111 enum ttm_bo_type type, u32 flags);
112struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile,
113 struct xe_vm *vm, size_t size, u64 offset,
114 enum ttm_bo_type type, u32 flags);
115struct xe_bo *xe_bo_create_pin_map_at_aligned(struct xe_device *xe,
116 struct xe_tile *tile,
117 struct xe_vm *vm,
118 size_t size, u64 offset,
119 enum ttm_bo_type type, u32 flags,
120 u64 alignment);
121struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
122 const void *data, size_t size,
123 enum ttm_bo_type type, u32 flags);
124struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
125 size_t size, u32 flags);
126struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
127 const void *data, size_t size, u32 flags);
128int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src);
129
130int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
131 u32 bo_flags);
132
133static inline struct xe_bo *ttm_to_xe_bo(const struct ttm_buffer_object *bo)
134{
135 return container_of(bo, struct xe_bo, ttm);
136}
137
138static inline struct xe_bo *gem_to_xe_bo(const struct drm_gem_object *obj)
139{
140 return container_of(obj, struct xe_bo, ttm.base);
141}
142
143#define xe_bo_device(bo) ttm_to_xe_device((bo)->ttm.bdev)
144
145static inline struct xe_bo *xe_bo_get(struct xe_bo *bo)
146{
147 if (bo)
148 drm_gem_object_get(obj: &bo->ttm.base);
149
150 return bo;
151}
152
153void xe_bo_put(struct xe_bo *bo);
154
155/*
156 * xe_bo_get_unless_zero() - Conditionally obtain a GEM object refcount on an
157 * xe bo
158 * @bo: The bo for which we want to obtain a refcount.
159 *
160 * There is a short window between where the bo's GEM object refcount reaches
161 * zero and where we put the final ttm_bo reference. Code in the eviction- and
162 * shrinking path should therefore attempt to grab a gem object reference before
163 * trying to use members outside of the base class ttm object. This function is
164 * intended for that purpose. On successful return, this function must be paired
165 * with an xe_bo_put().
166 *
167 * Return: @bo on success, NULL on failure.
168 */
169static inline __must_check struct xe_bo *xe_bo_get_unless_zero(struct xe_bo *bo)
170{
171 if (!bo || !kref_get_unless_zero(kref: &bo->ttm.base.refcount))
172 return NULL;
173
174 return bo;
175}
176
177static inline void __xe_bo_unset_bulk_move(struct xe_bo *bo)
178{
179 if (bo)
180 ttm_bo_set_bulk_move(bo: &bo->ttm, NULL);
181}
182
183static inline void xe_bo_assert_held(struct xe_bo *bo)
184{
185 if (bo)
186 dma_resv_assert_held((bo)->ttm.base.resv);
187}
188
189int xe_bo_lock(struct xe_bo *bo, bool intr);
190
191void xe_bo_unlock(struct xe_bo *bo);
192
193static inline void xe_bo_unlock_vm_held(struct xe_bo *bo)
194{
195 if (bo) {
196 XE_WARN_ON(bo->vm && bo->ttm.base.resv != xe_vm_resv(bo->vm));
197 if (bo->vm)
198 xe_vm_assert_held(bo->vm);
199 else
200 dma_resv_unlock(obj: bo->ttm.base.resv);
201 }
202}
203
204int xe_bo_pin_external(struct xe_bo *bo);
205int xe_bo_pin(struct xe_bo *bo);
206void xe_bo_unpin_external(struct xe_bo *bo);
207void xe_bo_unpin(struct xe_bo *bo);
208int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict);
209
210static inline bool xe_bo_is_pinned(struct xe_bo *bo)
211{
212 return bo->ttm.pin_count;
213}
214
215static inline bool xe_bo_is_protected(const struct xe_bo *bo)
216{
217 return bo->pxp_key_instance;
218}
219
220static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo)
221{
222 if (likely(bo)) {
223 xe_bo_lock(bo, intr: false);
224 xe_bo_unpin(bo);
225 xe_bo_unlock(bo);
226
227 xe_bo_put(bo);
228 }
229}
230
231bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo);
232dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size);
233dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size);
234
235static inline dma_addr_t
236xe_bo_main_addr(struct xe_bo *bo, size_t page_size)
237{
238 return xe_bo_addr(bo, offset: 0, page_size);
239}
240
241static inline u32
242__xe_bo_ggtt_addr(struct xe_bo *bo, u8 tile_id)
243{
244 struct xe_ggtt_node *ggtt_node = bo->ggtt_node[tile_id];
245
246 if (XE_WARN_ON(!ggtt_node))
247 return 0;
248
249 XE_WARN_ON(ggtt_node->base.size > bo->size);
250 XE_WARN_ON(ggtt_node->base.start + ggtt_node->base.size > (1ull << 32));
251 return ggtt_node->base.start;
252}
253
254static inline u32
255xe_bo_ggtt_addr(struct xe_bo *bo)
256{
257 xe_assert(xe_bo_device(bo), bo->tile);
258
259 return __xe_bo_ggtt_addr(bo, tile_id: bo->tile->id);
260}
261
262int xe_bo_vmap(struct xe_bo *bo);
263void xe_bo_vunmap(struct xe_bo *bo);
264int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size);
265
266bool mem_type_is_vram(u32 mem_type);
267bool xe_bo_is_vram(struct xe_bo *bo);
268bool xe_bo_is_stolen(struct xe_bo *bo);
269bool xe_bo_is_stolen_devmem(struct xe_bo *bo);
270bool xe_bo_is_vm_bound(struct xe_bo *bo);
271bool xe_bo_has_single_placement(struct xe_bo *bo);
272uint64_t vram_region_gpu_offset(struct ttm_resource *res);
273
274bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type);
275
276int xe_bo_migrate(struct xe_bo *bo, u32 mem_type);
277int xe_bo_evict(struct xe_bo *bo);
278
279int xe_bo_evict_pinned(struct xe_bo *bo);
280int xe_bo_notifier_prepare_pinned(struct xe_bo *bo);
281int xe_bo_notifier_unprepare_pinned(struct xe_bo *bo);
282int xe_bo_restore_pinned(struct xe_bo *bo);
283
284int xe_bo_dma_unmap_pinned(struct xe_bo *bo);
285
286extern const struct ttm_device_funcs xe_ttm_funcs;
287extern const char *const xe_mem_type_to_name[];
288
289int xe_gem_create_ioctl(struct drm_device *dev, void *data,
290 struct drm_file *file);
291int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
292 struct drm_file *file);
293void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo);
294
295int xe_bo_dumb_create(struct drm_file *file_priv,
296 struct drm_device *dev,
297 struct drm_mode_create_dumb *args);
298
299bool xe_bo_needs_ccs_pages(struct xe_bo *bo);
300
301static inline size_t xe_bo_ccs_pages_start(struct xe_bo *bo)
302{
303 return PAGE_ALIGN(bo->ttm.base.size);
304}
305
306static inline bool xe_bo_has_pages(struct xe_bo *bo)
307{
308 if ((bo->ttm.ttm && ttm_tt_is_populated(tt: bo->ttm.ttm)) ||
309 xe_bo_is_vram(bo))
310 return true;
311
312 return false;
313}
314
315void __xe_bo_release_dummy(struct kref *kref);
316
317/**
318 * xe_bo_put_deferred() - Put a buffer object with delayed final freeing
319 * @bo: The bo to put.
320 * @deferred: List to which to add the buffer object if we cannot put, or
321 * NULL if the function is to put unconditionally.
322 *
323 * Since the final freeing of an object includes both sleeping and (!)
324 * memory allocation in the dma_resv individualization, it's not ok
325 * to put an object from atomic context nor from within a held lock
326 * tainted by reclaim. In such situations we want to defer the final
327 * freeing until we've exited the restricting context, or in the worst
328 * case to a workqueue.
329 * This function either puts the object if possible without the refcount
330 * reaching zero, or adds it to the @deferred list if that was not possible.
331 * The caller needs to follow up with a call to xe_bo_put_commit() to actually
332 * put the bo iff this function returns true. It's safe to always
333 * follow up with a call to xe_bo_put_commit().
334 * TODO: It's TTM that is the villain here. Perhaps TTM should add an
335 * interface like this.
336 *
337 * Return: true if @bo was the first object put on the @freed list,
338 * false otherwise.
339 */
340static inline bool
341xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
342{
343 if (!deferred) {
344 xe_bo_put(bo);
345 return false;
346 }
347
348 if (!kref_put(kref: &bo->ttm.base.refcount, release: __xe_bo_release_dummy))
349 return false;
350
351 return llist_add(new: &bo->freed, head: deferred);
352}
353
354void xe_bo_put_commit(struct llist_head *deferred);
355
356/**
357 * xe_bo_put_async() - Put BO async
358 * @bo: The bo to put.
359 *
360 * Put BO async, the final put is deferred to a worker to exit an IRQ context.
361 */
362static inline void
363xe_bo_put_async(struct xe_bo *bo)
364{
365 struct xe_bo_dev *bo_device = &xe_bo_device(bo)->bo_device;
366
367 if (xe_bo_put_deferred(bo, deferred: &bo_device->async_list))
368 schedule_work(work: &bo_device->async_free);
369}
370
371void xe_bo_dev_init(struct xe_bo_dev *bo_device);
372
373void xe_bo_dev_fini(struct xe_bo_dev *bo_device);
374
375struct sg_table *xe_bo_sg(struct xe_bo *bo);
376
377/*
378 * xe_sg_segment_size() - Provides upper limit for sg segment size.
379 * @dev: device pointer
380 *
381 * Returns the maximum segment size for the 'struct scatterlist'
382 * elements.
383 */
384static inline unsigned int xe_sg_segment_size(struct device *dev)
385{
386 struct scatterlist __maybe_unused sg;
387 size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1;
388
389 max = min_t(size_t, max, dma_max_mapping_size(dev));
390
391 /*
392 * The iommu_dma_map_sg() function ensures iova allocation doesn't
393 * cross dma segment boundary. It does so by padding some sg elements.
394 * This can cause overflow, ending up with sg->length being set to 0.
395 * Avoid this by ensuring maximum segment size is half of 'max'
396 * rounded down to PAGE_SIZE.
397 */
398 return round_down(max / 2, PAGE_SIZE);
399}
400
401/**
402 * struct xe_bo_shrink_flags - flags governing the shrink behaviour.
403 * @purge: Only purging allowed. Don't shrink if bo not purgeable.
404 * @writeback: Attempt to immediately move content to swap.
405 */
406struct xe_bo_shrink_flags {
407 u32 purge : 1;
408 u32 writeback : 1;
409};
410
411long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
412 const struct xe_bo_shrink_flags flags,
413 unsigned long *scanned);
414
415/**
416 * xe_bo_is_mem_type - Whether the bo currently resides in the given
417 * TTM memory type
418 * @bo: The bo to check.
419 * @mem_type: The TTM memory type.
420 *
421 * Return: true iff the bo resides in @mem_type, false otherwise.
422 */
423static inline bool xe_bo_is_mem_type(struct xe_bo *bo, u32 mem_type)
424{
425 xe_bo_assert_held(bo);
426 return bo->ttm.resource->mem_type == mem_type;
427}
428#endif
429

source code of linux/drivers/gpu/drm/xe/xe_bo.h