1// SPDX-License-Identifier: GPL-2.0-only or MIT
2// Copyright (C) 2025 Arm, Ltd.
3
4#include <linux/bitfield.h>
5#include <linux/clk.h>
6#include <linux/genalloc.h>
7#include <linux/io.h>
8#include <linux/iopoll.h>
9#include <linux/module.h>
10#include <linux/mod_devicetable.h>
11#include <linux/platform_device.h>
12#include <linux/pm_runtime.h>
13
14#include <drm/drm_drv.h>
15#include <drm/drm_ioctl.h>
16#include <drm/drm_utils.h>
17#include <drm/drm_gem.h>
18#include <drm/drm_accel.h>
19#include <drm/ethosu_accel.h>
20
21#include "ethosu_drv.h"
22#include "ethosu_device.h"
23#include "ethosu_gem.h"
24#include "ethosu_job.h"
25
26static int ethosu_ioctl_dev_query(struct drm_device *ddev, void *data,
27 struct drm_file *file)
28{
29 struct ethosu_device *ethosudev = to_ethosu_device(ddev);
30 struct drm_ethosu_dev_query *args = data;
31
32 if (!args->pointer) {
33 switch (args->type) {
34 case DRM_ETHOSU_DEV_QUERY_NPU_INFO:
35 args->size = sizeof(ethosudev->npu_info);
36 return 0;
37 default:
38 return -EINVAL;
39 }
40 }
41
42 switch (args->type) {
43 case DRM_ETHOSU_DEV_QUERY_NPU_INFO:
44 if (args->size < offsetofend(struct drm_ethosu_npu_info, sram_size))
45 return -EINVAL;
46 return copy_struct_to_user(u64_to_user_ptr(args->pointer),
47 usize: args->size,
48 src: &ethosudev->npu_info,
49 ksize: sizeof(ethosudev->npu_info), NULL);
50 default:
51 return -EINVAL;
52 }
53}
54
55#define ETHOSU_BO_FLAGS DRM_ETHOSU_BO_NO_MMAP
56
57static int ethosu_ioctl_bo_create(struct drm_device *ddev, void *data,
58 struct drm_file *file)
59{
60 struct drm_ethosu_bo_create *args = data;
61 int cookie, ret;
62
63 if (!drm_dev_enter(dev: ddev, idx: &cookie))
64 return -ENODEV;
65
66 if (!args->size || (args->flags & ~ETHOSU_BO_FLAGS)) {
67 ret = -EINVAL;
68 goto out_dev_exit;
69 }
70
71 ret = ethosu_gem_create_with_handle(file, ddev, size: &args->size,
72 flags: args->flags, handle: &args->handle);
73
74out_dev_exit:
75 drm_dev_exit(idx: cookie);
76 return ret;
77}
78
79static int ethosu_ioctl_bo_wait(struct drm_device *ddev, void *data,
80 struct drm_file *file)
81{
82 struct drm_ethosu_bo_wait *args = data;
83 int cookie, ret;
84 unsigned long timeout = drm_timeout_abs_to_jiffies(timeout_nsec: args->timeout_ns);
85
86 if (args->pad)
87 return -EINVAL;
88
89 if (!drm_dev_enter(dev: ddev, idx: &cookie))
90 return -ENODEV;
91
92 ret = drm_gem_dma_resv_wait(filep: file, handle: args->handle, wait_all: true, timeout);
93
94 drm_dev_exit(idx: cookie);
95 return ret;
96}
97
98static int ethosu_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data,
99 struct drm_file *file)
100{
101 struct drm_ethosu_bo_mmap_offset *args = data;
102 struct drm_gem_object *obj;
103
104 if (args->pad)
105 return -EINVAL;
106
107 obj = drm_gem_object_lookup(filp: file, handle: args->handle);
108 if (!obj)
109 return -ENOENT;
110
111 args->offset = drm_vma_node_offset_addr(node: &obj->vma_node);
112 drm_gem_object_put(obj);
113 return 0;
114}
115
116static int ethosu_ioctl_cmdstream_bo_create(struct drm_device *ddev, void *data,
117 struct drm_file *file)
118{
119 struct drm_ethosu_cmdstream_bo_create *args = data;
120 int cookie, ret;
121
122 if (!drm_dev_enter(dev: ddev, idx: &cookie))
123 return -ENODEV;
124
125 if (!args->size || !args->data || args->pad || args->flags) {
126 ret = -EINVAL;
127 goto out_dev_exit;
128 }
129
130 args->flags |= DRM_ETHOSU_BO_NO_MMAP;
131
132 ret = ethosu_gem_cmdstream_create(file, ddev, size: args->size, data: args->data,
133 flags: args->flags, handle: &args->handle);
134
135out_dev_exit:
136 drm_dev_exit(idx: cookie);
137 return ret;
138}
139
140static int ethosu_open(struct drm_device *ddev, struct drm_file *file)
141{
142 int ret = 0;
143
144 if (!try_module_get(THIS_MODULE))
145 return -EINVAL;
146
147 struct ethosu_file_priv __free(kfree) *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
148 if (!priv) {
149 ret = -ENOMEM;
150 goto err_put_mod;
151 }
152 priv->edev = to_ethosu_device(ddev);
153
154 ret = ethosu_job_open(ethosu_priv: priv);
155 if (ret)
156 goto err_put_mod;
157
158 file->driver_priv = no_free_ptr(priv);
159 return 0;
160
161err_put_mod:
162 module_put(THIS_MODULE);
163 return ret;
164}
165
166static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file)
167{
168 ethosu_job_close(ethosu_priv: file->driver_priv);
169 kfree(objp: file->driver_priv);
170 module_put(THIS_MODULE);
171}
172
173static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = {
174#define ETHOSU_IOCTL(n, func, flags) \
175 DRM_IOCTL_DEF_DRV(ETHOSU_##n, ethosu_ioctl_##func, flags)
176
177 ETHOSU_IOCTL(DEV_QUERY, dev_query, 0),
178 ETHOSU_IOCTL(BO_CREATE, bo_create, 0),
179 ETHOSU_IOCTL(BO_WAIT, bo_wait, 0),
180 ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0),
181 ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0),
182 ETHOSU_IOCTL(SUBMIT, submit, 0),
183};
184
185DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops);
186
187/*
188 * Ethosu driver version:
189 * - 1.0 - initial interface
190 */
191static const struct drm_driver ethosu_drm_driver = {
192 .driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM,
193 .open = ethosu_open,
194 .postclose = ethosu_postclose,
195 .ioctls = ethosu_drm_driver_ioctls,
196 .num_ioctls = ARRAY_SIZE(ethosu_drm_driver_ioctls),
197 .fops = &ethosu_drm_driver_fops,
198 .name = "ethosu",
199 .desc = "Arm Ethos-U Accel driver",
200 .major = 1,
201 .minor = 0,
202
203 .gem_create_object = ethosu_gem_create_object,
204};
205
206#define U65_DRAM_AXI_LIMIT_CFG 0x1f3f0002
207#define U65_SRAM_AXI_LIMIT_CFG 0x1f3f00b0
208#define U85_AXI_EXT_CFG 0x00021f3f
209#define U85_AXI_SRAM_CFG 0x00021f3f
210#define U85_MEM_ATTR0_CFG 0x00000000
211#define U85_MEM_ATTR2_CFG 0x000000b7
212
213static int ethosu_reset(struct ethosu_device *ethosudev)
214{
215 int ret;
216 u32 reg;
217
218 writel_relaxed(RESET_PENDING_CSL, ethosudev->regs + NPU_REG_RESET);
219 ret = readl_poll_timeout(ethosudev->regs + NPU_REG_STATUS, reg,
220 !FIELD_GET(STATUS_RESET_STATUS, reg),
221 USEC_PER_MSEC, USEC_PER_SEC);
222 if (ret)
223 return ret;
224
225 if (!FIELD_GET(PROT_ACTIVE_CSL, readl_relaxed(ethosudev->regs + NPU_REG_PROT))) {
226 dev_warn(ethosudev->base.dev, "Could not reset to non-secure mode (PROT = %x)\n",
227 readl_relaxed(ethosudev->regs + NPU_REG_PROT));
228 }
229
230 /*
231 * Assign region 2 (SRAM) to AXI M0 (AXILIMIT0),
232 * everything else to AXI M1 (AXILIMIT2)
233 */
234 writel_relaxed(0x0000aa8a, ethosudev->regs + NPU_REG_REGIONCFG);
235 if (ethosu_is_u65(ethosudev)) {
236 writel_relaxed(U65_SRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT0);
237 writel_relaxed(U65_DRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT2);
238 } else {
239 writel_relaxed(U85_AXI_SRAM_CFG, ethosudev->regs + NPU_REG_AXI_SRAM);
240 writel_relaxed(U85_AXI_EXT_CFG, ethosudev->regs + NPU_REG_AXI_EXT);
241 writel_relaxed(U85_MEM_ATTR0_CFG, ethosudev->regs + NPU_REG_MEM_ATTR0); // SRAM
242 writel_relaxed(U85_MEM_ATTR2_CFG, ethosudev->regs + NPU_REG_MEM_ATTR2); // DRAM
243 }
244
245 if (ethosudev->sram)
246 memset_io(ethosudev->sram, 0, ethosudev->npu_info.sram_size);
247
248 return 0;
249}
250
251static int ethosu_device_resume(struct device *dev)
252{
253 struct ethosu_device *ethosudev = dev_get_drvdata(dev);
254 int ret;
255
256 ret = clk_bulk_prepare_enable(num_clks: ethosudev->num_clks, clks: ethosudev->clks);
257 if (ret)
258 return ret;
259
260 ret = ethosu_reset(ethosudev);
261 if (!ret)
262 return 0;
263
264 clk_bulk_disable_unprepare(num_clks: ethosudev->num_clks, clks: ethosudev->clks);
265 return ret;
266}
267
268static int ethosu_device_suspend(struct device *dev)
269{
270 struct ethosu_device *ethosudev = dev_get_drvdata(dev);
271
272 clk_bulk_disable_unprepare(num_clks: ethosudev->num_clks, clks: ethosudev->clks);
273 return 0;
274}
275
276static int ethosu_sram_init(struct ethosu_device *ethosudev)
277{
278 ethosudev->npu_info.sram_size = 0;
279
280 ethosudev->srampool = of_gen_pool_get(np: ethosudev->base.dev->of_node, propname: "sram", index: 0);
281 if (!ethosudev->srampool)
282 return 0;
283
284 ethosudev->npu_info.sram_size = gen_pool_size(ethosudev->srampool);
285
286 ethosudev->sram = (void __iomem *)gen_pool_dma_alloc(pool: ethosudev->srampool,
287 size: ethosudev->npu_info.sram_size,
288 dma: &ethosudev->sramphys);
289 if (!ethosudev->sram) {
290 dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n");
291 return -ENOMEM;
292 }
293
294 return 0;
295}
296
297static int ethosu_init(struct ethosu_device *ethosudev)
298{
299 int ret;
300 u32 id, config;
301
302 ret = ethosu_device_resume(dev: ethosudev->base.dev);
303 if (ret)
304 return ret;
305
306 pm_runtime_set_autosuspend_delay(dev: ethosudev->base.dev, delay: 50);
307 pm_runtime_use_autosuspend(dev: ethosudev->base.dev);
308 ret = devm_pm_runtime_set_active_enabled(dev: ethosudev->base.dev);
309 if (ret)
310 return ret;
311 pm_runtime_get_noresume(dev: ethosudev->base.dev);
312
313 ethosudev->npu_info.id = id = readl_relaxed(ethosudev->regs + NPU_REG_ID);
314 ethosudev->npu_info.config = config = readl_relaxed(ethosudev->regs + NPU_REG_CONFIG);
315
316 ethosu_sram_init(ethosudev);
317
318 dev_info(ethosudev->base.dev,
319 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM\n",
320 FIELD_GET(ID_ARCH_MAJOR_MASK, id),
321 FIELD_GET(ID_ARCH_MINOR_MASK, id),
322 FIELD_GET(ID_ARCH_PATCH_MASK, id),
323 FIELD_GET(ID_VER_MAJOR_MASK, id),
324 FIELD_GET(ID_VER_MINOR_MASK, id),
325 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config),
326 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config),
327 ethosudev->npu_info.sram_size / 1024);
328
329 return 0;
330}
331
332static int ethosu_probe(struct platform_device *pdev)
333{
334 int ret;
335 struct ethosu_device *ethosudev;
336
337 ethosudev = devm_drm_dev_alloc(&pdev->dev, &ethosu_drm_driver,
338 struct ethosu_device, base);
339 if (IS_ERR(ptr: ethosudev))
340 return -ENOMEM;
341 platform_set_drvdata(pdev, data: ethosudev);
342
343 dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(40));
344
345 ethosudev->regs = devm_platform_ioremap_resource(pdev, index: 0);
346
347 ethosudev->num_clks = devm_clk_bulk_get_all(dev: &pdev->dev, clks: &ethosudev->clks);
348 if (ethosudev->num_clks < 0)
349 return ethosudev->num_clks;
350
351 ret = ethosu_job_init(dev: ethosudev);
352 if (ret)
353 return ret;
354
355 ret = ethosu_init(ethosudev);
356 if (ret)
357 return ret;
358
359 ret = drm_dev_register(dev: &ethosudev->base, flags: 0);
360 if (ret)
361 pm_runtime_dont_use_autosuspend(dev: ethosudev->base.dev);
362
363 pm_runtime_put_autosuspend(dev: ethosudev->base.dev);
364 return ret;
365}
366
367static void ethosu_remove(struct platform_device *pdev)
368{
369 struct ethosu_device *ethosudev = dev_get_drvdata(dev: &pdev->dev);
370
371 drm_dev_unregister(dev: &ethosudev->base);
372 ethosu_job_fini(dev: ethosudev);
373 if (ethosudev->sram)
374 gen_pool_free(pool: ethosudev->srampool, addr: (unsigned long)ethosudev->sram,
375 size: ethosudev->npu_info.sram_size);
376}
377
378static const struct of_device_id dt_match[] = {
379 { .compatible = "arm,ethos-u65" },
380 { .compatible = "arm,ethos-u85" },
381 {}
382};
383MODULE_DEVICE_TABLE(of, dt_match);
384
385static DEFINE_RUNTIME_DEV_PM_OPS(ethosu_pm_ops,
386 ethosu_device_suspend,
387 ethosu_device_resume,
388 NULL);
389
390static struct platform_driver ethosu_driver = {
391 .probe = ethosu_probe,
392 .remove = ethosu_remove,
393 .driver = {
394 .name = "ethosu",
395 .pm = pm_ptr(&ethosu_pm_ops),
396 .of_match_table = dt_match,
397 },
398};
399module_platform_driver(ethosu_driver);
400
401MODULE_AUTHOR("Rob Herring <robh@kernel.org>");
402MODULE_DESCRIPTION("Arm Ethos-U Accel Driver");
403MODULE_LICENSE("Dual MIT/GPL");
404

source code of linux/drivers/accel/ethosu/ethosu_drv.c