1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2025, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_device.h>
8#include <drm/drm_print.h>
9#include <linux/dma-buf.h>
10#include <linux/pagemap.h>
11#include <linux/vmalloc.h>
12
13#include "amdxdna_pci_drv.h"
14#include "amdxdna_ubuf.h"
15
16struct amdxdna_ubuf_priv {
17 struct page **pages;
18 u64 nr_pages;
19 enum amdxdna_ubuf_flag flags;
20 struct mm_struct *mm;
21};
22
23static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach,
24 enum dma_data_direction direction)
25{
26 struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv;
27 struct sg_table *sg;
28 int ret;
29
30 sg = kzalloc(sizeof(*sg), GFP_KERNEL);
31 if (!sg)
32 return ERR_PTR(error: -ENOMEM);
33
34 ret = sg_alloc_table_from_pages(sgt: sg, pages: ubuf->pages, n_pages: ubuf->nr_pages, offset: 0,
35 size: ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL);
36 if (ret)
37 return ERR_PTR(error: ret);
38
39 if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) {
40 ret = dma_map_sgtable(dev: attach->dev, sgt: sg, dir: direction, attrs: 0);
41 if (ret)
42 return ERR_PTR(error: ret);
43 }
44
45 return sg;
46}
47
48static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach,
49 struct sg_table *sg,
50 enum dma_data_direction direction)
51{
52 struct amdxdna_ubuf_priv *ubuf = attach->dmabuf->priv;
53
54 if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA)
55 dma_unmap_sgtable(dev: attach->dev, sgt: sg, dir: direction, attrs: 0);
56
57 sg_free_table(sg);
58 kfree(objp: sg);
59}
60
61static void amdxdna_ubuf_release(struct dma_buf *dbuf)
62{
63 struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
64
65 unpin_user_pages(pages: ubuf->pages, npages: ubuf->nr_pages);
66 kvfree(addr: ubuf->pages);
67 atomic64_sub(i: ubuf->nr_pages, v: &ubuf->mm->pinned_vm);
68 mmdrop(mm: ubuf->mm);
69 kfree(objp: ubuf);
70}
71
72static vm_fault_t amdxdna_ubuf_vm_fault(struct vm_fault *vmf)
73{
74 struct vm_area_struct *vma = vmf->vma;
75 struct amdxdna_ubuf_priv *ubuf;
76 unsigned long pfn;
77 pgoff_t pgoff;
78
79 ubuf = vma->vm_private_data;
80 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
81
82 pfn = page_to_pfn(ubuf->pages[pgoff]);
83 return vmf_insert_pfn(vma, addr: vmf->address, pfn);
84}
85
86static const struct vm_operations_struct amdxdna_ubuf_vm_ops = {
87 .fault = amdxdna_ubuf_vm_fault,
88};
89
90static int amdxdna_ubuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma)
91{
92 struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
93
94 vma->vm_ops = &amdxdna_ubuf_vm_ops;
95 vma->vm_private_data = ubuf;
96 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
97
98 return 0;
99}
100
101static int amdxdna_ubuf_vmap(struct dma_buf *dbuf, struct iosys_map *map)
102{
103 struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
104 void *kva;
105
106 kva = vmap(pages: ubuf->pages, count: ubuf->nr_pages, VM_MAP, PAGE_KERNEL);
107 if (!kva)
108 return -EINVAL;
109
110 iosys_map_set_vaddr(map, vaddr: kva);
111 return 0;
112}
113
114static void amdxdna_ubuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map)
115{
116 vunmap(addr: map->vaddr);
117}
118
119static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = {
120 .map_dma_buf = amdxdna_ubuf_map,
121 .unmap_dma_buf = amdxdna_ubuf_unmap,
122 .release = amdxdna_ubuf_release,
123 .mmap = amdxdna_ubuf_mmap,
124 .vmap = amdxdna_ubuf_vmap,
125 .vunmap = amdxdna_ubuf_vunmap,
126};
127
128struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
129 enum amdxdna_ubuf_flag flags,
130 u32 num_entries, void __user *va_entries)
131{
132 struct amdxdna_dev *xdna = to_xdna_dev(dev);
133 unsigned long lock_limit, new_pinned;
134 struct amdxdna_drm_va_entry *va_ent;
135 struct amdxdna_ubuf_priv *ubuf;
136 u32 npages, start = 0;
137 struct dma_buf *dbuf;
138 int i, ret;
139 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
140
141 if (!can_do_mlock())
142 return ERR_PTR(error: -EPERM);
143
144 ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
145 if (!ubuf)
146 return ERR_PTR(error: -ENOMEM);
147
148 ubuf->flags = flags;
149 ubuf->mm = current->mm;
150 mmgrab(mm: ubuf->mm);
151
152 va_ent = kvcalloc(num_entries, sizeof(*va_ent), GFP_KERNEL);
153 if (!va_ent) {
154 ret = -ENOMEM;
155 goto free_ubuf;
156 }
157
158 if (copy_from_user(to: va_ent, from: va_entries, n: sizeof(*va_ent) * num_entries)) {
159 XDNA_DBG(xdna, "Access va entries failed");
160 ret = -EINVAL;
161 goto free_ent;
162 }
163
164 for (i = 0, exp_info.size = 0; i < num_entries; i++) {
165 if (!IS_ALIGNED(va_ent[i].vaddr, PAGE_SIZE) ||
166 !IS_ALIGNED(va_ent[i].len, PAGE_SIZE)) {
167 XDNA_ERR(xdna, "Invalid address or len %llx, %llx",
168 va_ent[i].vaddr, va_ent[i].len);
169 ret = -EINVAL;
170 goto free_ent;
171 }
172
173 exp_info.size += va_ent[i].len;
174 }
175
176 ubuf->nr_pages = exp_info.size >> PAGE_SHIFT;
177 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
178 new_pinned = atomic64_add_return(i: ubuf->nr_pages, v: &ubuf->mm->pinned_vm);
179 if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) {
180 XDNA_DBG(xdna, "New pin %ld, limit %ld, cap %d",
181 new_pinned, lock_limit, capable(CAP_IPC_LOCK));
182 ret = -ENOMEM;
183 goto sub_pin_cnt;
184 }
185
186 ubuf->pages = kvmalloc_array(ubuf->nr_pages, sizeof(*ubuf->pages), GFP_KERNEL);
187 if (!ubuf->pages) {
188 ret = -ENOMEM;
189 goto sub_pin_cnt;
190 }
191
192 for (i = 0; i < num_entries; i++) {
193 npages = va_ent[i].len >> PAGE_SHIFT;
194
195 ret = pin_user_pages_fast(start: va_ent[i].vaddr, nr_pages: npages,
196 gup_flags: FOLL_WRITE | FOLL_LONGTERM,
197 pages: &ubuf->pages[start]);
198 if (ret < 0 || ret != npages) {
199 ret = -ENOMEM;
200 XDNA_ERR(xdna, "Failed to pin pages ret %d", ret);
201 goto destroy_pages;
202 }
203
204 start += ret;
205 }
206
207 exp_info.ops = &amdxdna_ubuf_dmabuf_ops;
208 exp_info.priv = ubuf;
209 exp_info.flags = O_RDWR | O_CLOEXEC;
210
211 dbuf = dma_buf_export(exp_info: &exp_info);
212 if (IS_ERR(ptr: dbuf)) {
213 ret = PTR_ERR(ptr: dbuf);
214 goto destroy_pages;
215 }
216 kvfree(addr: va_ent);
217
218 return dbuf;
219
220destroy_pages:
221 if (start)
222 unpin_user_pages(pages: ubuf->pages, npages: start);
223 kvfree(addr: ubuf->pages);
224sub_pin_cnt:
225 atomic64_sub(i: ubuf->nr_pages, v: &ubuf->mm->pinned_vm);
226free_ent:
227 kvfree(addr: va_ent);
228free_ubuf:
229 mmdrop(mm: ubuf->mm);
230 kfree(objp: ubuf);
231 return ERR_PTR(error: ret);
232}
233

source code of linux/drivers/accel/amdxdna/amdxdna_ubuf.c