1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/bitfield.h>
7#include <linux/highmem.h>
8#include <linux/set_memory.h>
9
10#include <drm/drm_cache.h>
11
12#include "ivpu_drv.h"
13#include "ivpu_hw.h"
14#include "ivpu_mmu.h"
15#include "ivpu_mmu_context.h"
16
17#define IVPU_MMU_VPU_ADDRESS_MASK GENMASK(47, 12)
18#define IVPU_MMU_PGD_INDEX_MASK GENMASK(47, 39)
19#define IVPU_MMU_PUD_INDEX_MASK GENMASK(38, 30)
20#define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21)
21#define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12)
22#define IVPU_MMU_ENTRY_FLAGS_MASK (BIT(52) | GENMASK(11, 0))
23#define IVPU_MMU_ENTRY_FLAG_CONT BIT(52)
24#define IVPU_MMU_ENTRY_FLAG_NG BIT(11)
25#define IVPU_MMU_ENTRY_FLAG_AF BIT(10)
26#define IVPU_MMU_ENTRY_FLAG_USER BIT(6)
27#define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
28#define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1)
29#define IVPU_MMU_ENTRY_FLAG_VALID BIT(0)
30
31#define IVPU_MMU_PAGE_SIZE SZ_4K
32#define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16)
33#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
34#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
35#define IVPU_MMU_PUD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE)
36#define IVPU_MMU_PGD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE)
37#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
38
39#define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
40#define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)
41#define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
42#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
43 IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
44
45static void *ivpu_pgtable_alloc_page(struct ivpu_device *vdev, dma_addr_t *dma)
46{
47 dma_addr_t dma_addr;
48 struct page *page;
49 void *cpu;
50
51 page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
52 if (!page)
53 return NULL;
54
55 set_pages_array_wc(pages: &page, addrinarray: 1);
56
57 dma_addr = dma_map_page(vdev->drm.dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
58 if (dma_mapping_error(dev: vdev->drm.dev, dma_addr))
59 goto err_free_page;
60
61 cpu = vmap(pages: &page, count: 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL));
62 if (!cpu)
63 goto err_dma_unmap_page;
64
65
66 *dma = dma_addr;
67 return cpu;
68
69err_dma_unmap_page:
70 dma_unmap_page(vdev->drm.dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
71
72err_free_page:
73 put_page(page);
74 return NULL;
75}
76
77static void ivpu_pgtable_free_page(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
78{
79 struct page *page;
80
81 if (cpu_addr) {
82 page = vmalloc_to_page(addr: cpu_addr);
83 vunmap(addr: cpu_addr);
84 dma_unmap_page(vdev->drm.dev, dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK, PAGE_SIZE,
85 DMA_BIDIRECTIONAL);
86 set_pages_array_wb(pages: &page, addrinarray: 1);
87 put_page(page);
88 }
89}
90
91static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
92{
93 dma_addr_t pgd_dma;
94
95 pgtable->pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, dma: &pgd_dma);
96 if (!pgtable->pgd_dma_ptr)
97 return -ENOMEM;
98
99 pgtable->pgd_dma = pgd_dma;
100
101 return 0;
102}
103
104static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
105{
106 int pgd_idx, pud_idx, pmd_idx;
107 dma_addr_t pud_dma, pmd_dma, pte_dma;
108 u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr;
109
110 for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) {
111 pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
112 pud_dma = pgtable->pgd_dma_ptr[pgd_idx];
113
114 if (!pud_dma_ptr)
115 continue;
116
117 for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {
118 pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
119 pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];
120
121 if (!pmd_dma_ptr)
122 continue;
123
124 for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) {
125 pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
126 pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
127
128 ivpu_pgtable_free_page(vdev, cpu_addr: pte_dma_ptr, dma_addr: pte_dma);
129 }
130
131 kfree(objp: pgtable->pte_ptrs[pgd_idx][pud_idx]);
132 ivpu_pgtable_free_page(vdev, cpu_addr: pmd_dma_ptr, dma_addr: pmd_dma);
133 }
134
135 kfree(objp: pgtable->pmd_ptrs[pgd_idx]);
136 kfree(objp: pgtable->pte_ptrs[pgd_idx]);
137 ivpu_pgtable_free_page(vdev, cpu_addr: pud_dma_ptr, dma_addr: pud_dma);
138 }
139
140 ivpu_pgtable_free_page(vdev, cpu_addr: pgtable->pgd_dma_ptr, dma_addr: pgtable->pgd_dma);
141}
142
143static u64*
144ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)
145{
146 u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
147 dma_addr_t pud_dma;
148
149 if (pud_dma_ptr)
150 return pud_dma_ptr;
151
152 pud_dma_ptr = ivpu_pgtable_alloc_page(vdev, dma: &pud_dma);
153 if (!pud_dma_ptr)
154 return NULL;
155
156 drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);
157 pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
158 if (!pgtable->pmd_ptrs[pgd_idx])
159 goto err_free_pud_dma_ptr;
160
161 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);
162 pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
163 if (!pgtable->pte_ptrs[pgd_idx])
164 goto err_free_pmd_ptrs;
165
166 pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;
167 pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;
168
169 return pud_dma_ptr;
170
171err_free_pmd_ptrs:
172 kfree(objp: pgtable->pmd_ptrs[pgd_idx]);
173
174err_free_pud_dma_ptr:
175 ivpu_pgtable_free_page(vdev, cpu_addr: pud_dma_ptr, dma_addr: pud_dma);
176 return NULL;
177}
178
179static u64*
180ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,
181 int pud_idx)
182{
183 u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
184 dma_addr_t pmd_dma;
185
186 if (pmd_dma_ptr)
187 return pmd_dma_ptr;
188
189 pmd_dma_ptr = ivpu_pgtable_alloc_page(vdev, dma: &pmd_dma);
190 if (!pmd_dma_ptr)
191 return NULL;
192
193 drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);
194 pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
195 if (!pgtable->pte_ptrs[pgd_idx][pud_idx])
196 goto err_free_pmd_dma_ptr;
197
198 pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;
199 pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;
200
201 return pmd_dma_ptr;
202
203err_free_pmd_dma_ptr:
204 ivpu_pgtable_free_page(vdev, cpu_addr: pmd_dma_ptr, dma_addr: pmd_dma);
205 return NULL;
206}
207
208static u64*
209ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
210 int pgd_idx, int pud_idx, int pmd_idx)
211{
212 u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
213 dma_addr_t pte_dma;
214
215 if (pte_dma_ptr)
216 return pte_dma_ptr;
217
218 pte_dma_ptr = ivpu_pgtable_alloc_page(vdev, dma: &pte_dma);
219 if (!pte_dma_ptr)
220 return NULL;
221
222 pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;
223 pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;
224
225 return pte_dma_ptr;
226}
227
228static int
229ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
230 u64 vpu_addr, dma_addr_t dma_addr, u64 prot)
231{
232 u64 *pte;
233 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
234 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
235 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
236 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
237
238 /* Allocate PUD - second level page table if needed */
239 if (!ivpu_mmu_ensure_pud(vdev, pgtable: &ctx->pgtable, pgd_idx))
240 return -ENOMEM;
241
242 /* Allocate PMD - third level page table if needed */
243 if (!ivpu_mmu_ensure_pmd(vdev, pgtable: &ctx->pgtable, pgd_idx, pud_idx))
244 return -ENOMEM;
245
246 /* Allocate PTE - fourth level page table if needed */
247 pte = ivpu_mmu_ensure_pte(vdev, pgtable: &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);
248 if (!pte)
249 return -ENOMEM;
250
251 /* Update PTE */
252 pte[pte_idx] = dma_addr | prot;
253
254 return 0;
255}
256
257static int
258ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
259 dma_addr_t dma_addr, u64 prot)
260{
261 size_t size = IVPU_MMU_CONT_PAGES_SIZE;
262
263 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size));
264 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size));
265
266 prot |= IVPU_MMU_ENTRY_FLAG_CONT;
267
268 while (size) {
269 int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
270
271 if (ret)
272 return ret;
273
274 size -= IVPU_MMU_PAGE_SIZE;
275 vpu_addr += IVPU_MMU_PAGE_SIZE;
276 dma_addr += IVPU_MMU_PAGE_SIZE;
277 }
278
279 return 0;
280}
281
282static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
283{
284 int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
285 int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
286 int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
287 int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
288
289 /* Update PTE with dummy physical address and clear flags */
290 ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
291}
292
293static int
294ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
295 u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)
296{
297 int map_size;
298 int ret;
299
300 while (size) {
301 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE &&
302 IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) {
303 ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot);
304 map_size = IVPU_MMU_CONT_PAGES_SIZE;
305 } else {
306 ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
307 map_size = IVPU_MMU_PAGE_SIZE;
308 }
309
310 if (ret)
311 return ret;
312
313 vpu_addr += map_size;
314 dma_addr += map_size;
315 size -= map_size;
316 }
317
318 return 0;
319}
320
321static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
322{
323 while (size) {
324 ivpu_mmu_context_unmap_page(ctx, vpu_addr);
325 vpu_addr += IVPU_MMU_PAGE_SIZE;
326 size -= IVPU_MMU_PAGE_SIZE;
327 }
328}
329
330int
331ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
332 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent)
333{
334 struct scatterlist *sg;
335 int ret;
336 u64 prot;
337 u64 i;
338
339 if (drm_WARN_ON(&vdev->drm, !ctx))
340 return -EINVAL;
341
342 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
343 return -EINVAL;
344
345 if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK)
346 return -EINVAL;
347
348 prot = IVPU_MMU_ENTRY_MAPPED;
349 if (llc_coherent)
350 prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
351
352 mutex_lock(&ctx->lock);
353
354 for_each_sgtable_dma_sg(sgt, sg, i) {
355 dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
356 size_t size = sg_dma_len(sg) + sg->offset;
357
358 ivpu_dbg(vdev, MMU_MAP, "Map ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",
359 ctx->id, dma_addr, vpu_addr, size);
360
361 ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
362 if (ret) {
363 ivpu_err(vdev, "Failed to map context pages\n");
364 mutex_unlock(lock: &ctx->lock);
365 return ret;
366 }
367 vpu_addr += size;
368 }
369
370 /* Ensure page table modifications are flushed from wc buffers to memory */
371 wmb();
372
373 mutex_unlock(lock: &ctx->lock);
374
375 ret = ivpu_mmu_invalidate_tlb(vdev, ssid: ctx->id);
376 if (ret)
377 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
378 return ret;
379}
380
381void
382ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
383 u64 vpu_addr, struct sg_table *sgt)
384{
385 struct scatterlist *sg;
386 int ret;
387 u64 i;
388
389 if (drm_WARN_ON(&vdev->drm, !ctx))
390 return;
391
392 mutex_lock(&ctx->lock);
393
394 for_each_sgtable_dma_sg(sgt, sg, i) {
395 dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
396 size_t size = sg_dma_len(sg) + sg->offset;
397
398 ivpu_dbg(vdev, MMU_MAP, "Unmap ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",
399 ctx->id, dma_addr, vpu_addr, size);
400
401 ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
402 vpu_addr += size;
403 }
404
405 /* Ensure page table modifications are flushed from wc buffers to memory */
406 wmb();
407
408 mutex_unlock(lock: &ctx->lock);
409
410 ret = ivpu_mmu_invalidate_tlb(vdev, ssid: ctx->id);
411 if (ret)
412 ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
413}
414
415int
416ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,
417 u64 size, struct drm_mm_node *node)
418{
419 int ret;
420
421 WARN_ON(!range);
422
423 mutex_lock(&ctx->lock);
424 if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {
425 ret = drm_mm_insert_node_in_range(mm: &ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, color: 0,
426 start: range->start, end: range->end, mode: DRM_MM_INSERT_BEST);
427 if (!ret)
428 goto unlock;
429 }
430
431 ret = drm_mm_insert_node_in_range(mm: &ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, color: 0,
432 start: range->start, end: range->end, mode: DRM_MM_INSERT_BEST);
433unlock:
434 mutex_unlock(lock: &ctx->lock);
435 return ret;
436}
437
438void
439ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
440{
441 mutex_lock(&ctx->lock);
442 drm_mm_remove_node(node);
443 mutex_unlock(lock: &ctx->lock);
444}
445
446static int
447ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)
448{
449 u64 start, end;
450 int ret;
451
452 mutex_init(&ctx->lock);
453
454 ret = ivpu_mmu_pgtable_init(vdev, pgtable: &ctx->pgtable);
455 if (ret) {
456 ivpu_err(vdev, "Failed to initialize pgtable for ctx %u: %d\n", context_id, ret);
457 return ret;
458 }
459
460 if (!context_id) {
461 start = vdev->hw->ranges.global.start;
462 end = vdev->hw->ranges.shave.end;
463 } else {
464 start = vdev->hw->ranges.user.start;
465 end = vdev->hw->ranges.dma.end;
466 }
467
468 drm_mm_init(mm: &ctx->mm, start, size: end - start);
469 ctx->id = context_id;
470
471 return 0;
472}
473
474static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
475{
476 if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))
477 return;
478
479 mutex_destroy(lock: &ctx->lock);
480 ivpu_mmu_pgtables_free(vdev, pgtable: &ctx->pgtable);
481 drm_mm_takedown(mm: &ctx->mm);
482
483 ctx->pgtable.pgd_dma_ptr = NULL;
484 ctx->pgtable.pgd_dma = 0;
485}
486
487int ivpu_mmu_global_context_init(struct ivpu_device *vdev)
488{
489 return ivpu_mmu_context_init(vdev, ctx: &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
490}
491
492void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
493{
494 return ivpu_mmu_context_fini(vdev, ctx: &vdev->gctx);
495}
496
497int ivpu_mmu_reserved_context_init(struct ivpu_device *vdev)
498{
499 return ivpu_mmu_user_context_init(vdev, ctx: &vdev->rctx, IVPU_RESERVED_CONTEXT_MMU_SSID);
500}
501
502void ivpu_mmu_reserved_context_fini(struct ivpu_device *vdev)
503{
504 return ivpu_mmu_user_context_fini(vdev, ctx: &vdev->rctx);
505}
506
507void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)
508{
509 struct ivpu_file_priv *file_priv;
510
511 xa_lock(&vdev->context_xa);
512
513 file_priv = xa_load(&vdev->context_xa, index: ssid);
514 if (file_priv)
515 file_priv->has_mmu_faults = true;
516
517 xa_unlock(&vdev->context_xa);
518}
519
520int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id)
521{
522 int ret;
523
524 drm_WARN_ON(&vdev->drm, !ctx_id);
525
526 ret = ivpu_mmu_context_init(vdev, ctx, context_id: ctx_id);
527 if (ret) {
528 ivpu_err(vdev, "Failed to initialize context %u: %d\n", ctx_id, ret);
529 return ret;
530 }
531
532 ret = ivpu_mmu_set_pgtable(vdev, ssid: ctx_id, pgtable: &ctx->pgtable);
533 if (ret) {
534 ivpu_err(vdev, "Failed to set page table for context %u: %d\n", ctx_id, ret);
535 goto err_context_fini;
536 }
537
538 return 0;
539
540err_context_fini:
541 ivpu_mmu_context_fini(vdev, ctx);
542 return ret;
543}
544
545void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
546{
547 drm_WARN_ON(&vdev->drm, !ctx->id);
548
549 ivpu_mmu_clear_pgtable(vdev, ssid: ctx->id);
550 ivpu_mmu_context_fini(vdev, ctx);
551}
552

source code of linux/drivers/accel/ivpu/ivpu_mmu_context.c