1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_device.h>
8#include <drm/drm_gem.h>
9#include <drm/drm_gem_shmem_helper.h>
10#include <drm/drm_print.h>
11#include <drm/drm_syncobj.h>
12#include <linux/hmm.h>
13#include <linux/types.h>
14#include <linux/xarray.h>
15#include <trace/events/amdxdna.h>
16
17#include "aie2_msg_priv.h"
18#include "aie2_pci.h"
19#include "aie2_solver.h"
20#include "amdxdna_ctx.h"
21#include "amdxdna_gem.h"
22#include "amdxdna_mailbox.h"
23#include "amdxdna_pci_drv.h"
24#include "amdxdna_pm.h"
25
26static bool force_cmdlist;
27module_param(force_cmdlist, bool, 0600);
28MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default false)");
29
30#define HWCTX_MAX_TIMEOUT 60000 /* milliseconds */
31
32static void aie2_job_release(struct kref *ref)
33{
34 struct amdxdna_sched_job *job;
35
36 job = container_of(ref, struct amdxdna_sched_job, refcnt);
37 amdxdna_sched_job_cleanup(job);
38 atomic64_inc(v: &job->hwctx->job_free_cnt);
39 wake_up(&job->hwctx->priv->job_free_wq);
40 if (job->out_fence)
41 dma_fence_put(fence: job->out_fence);
42 kfree(objp: job);
43}
44
45static void aie2_job_put(struct amdxdna_sched_job *job)
46{
47 kref_put(kref: &job->refcnt, release: aie2_job_release);
48}
49
50static void aie2_hwctx_status_shift_stop(struct amdxdna_hwctx *hwctx)
51{
52 hwctx->old_status = hwctx->status;
53 hwctx->status = HWCTX_STAT_STOP;
54}
55
56static void aie2_hwctx_status_restore(struct amdxdna_hwctx *hwctx)
57{
58 hwctx->status = hwctx->old_status;
59}
60
61/* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */
62static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx,
63 struct drm_sched_job *bad_job)
64{
65 drm_sched_stop(sched: &hwctx->priv->sched, bad: bad_job);
66 aie2_destroy_context(ndev: xdna->dev_handle, hwctx);
67}
68
69static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx)
70{
71 struct amdxdna_gem_obj *heap = hwctx->priv->heap;
72 int ret;
73
74 ret = aie2_create_context(ndev: xdna->dev_handle, hwctx);
75 if (ret) {
76 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret);
77 goto out;
78 }
79
80 ret = aie2_map_host_buf(ndev: xdna->dev_handle, context_id: hwctx->fw_ctx_id,
81 addr: heap->mem.userptr, size: heap->mem.size);
82 if (ret) {
83 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret);
84 goto out;
85 }
86
87 if (hwctx->status != HWCTX_STAT_READY) {
88 XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status);
89 goto out;
90 }
91
92 ret = aie2_config_cu(hwctx, NULL);
93 if (ret) {
94 XDNA_ERR(xdna, "Config cu failed, ret %d", ret);
95 goto out;
96 }
97
98out:
99 drm_sched_start(sched: &hwctx->priv->sched, errno: 0);
100 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret);
101 return ret;
102}
103
104static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq)
105{
106 struct dma_fence *fence, *out_fence = NULL;
107 int ret;
108
109 fence = drm_syncobj_fence_get(syncobj: hwctx->priv->syncobj);
110 if (!fence)
111 return NULL;
112
113 ret = dma_fence_chain_find_seqno(pfence: &fence, seqno: seq);
114 if (ret)
115 goto out;
116
117 out_fence = dma_fence_get(fence: dma_fence_chain_contained(fence));
118
119out:
120 dma_fence_put(fence);
121 return out_fence;
122}
123
124static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx)
125{
126 struct dma_fence *fence;
127
128 fence = aie2_cmd_get_out_fence(hwctx, seq: hwctx->priv->seq - 1);
129 if (!fence)
130 return;
131
132 /* Wait up to 2 seconds for fw to finish all pending requests */
133 dma_fence_wait_timeout(fence, intr: false, timeout: msecs_to_jiffies(m: 2000));
134 dma_fence_put(fence);
135}
136
137static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg)
138{
139 struct amdxdna_dev *xdna = hwctx->client->xdna;
140
141 aie2_hwctx_wait_for_idle(hwctx);
142 aie2_hwctx_stop(xdna, hwctx, NULL);
143 aie2_hwctx_status_shift_stop(hwctx);
144
145 return 0;
146}
147
148void aie2_hwctx_suspend(struct amdxdna_client *client)
149{
150 struct amdxdna_dev *xdna = client->xdna;
151
152 /*
153 * Command timeout is unlikely. But if it happens, it doesn't
154 * break the system. aie2_hwctx_stop() will destroy mailbox
155 * and abort all commands.
156 */
157 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
158 amdxdna_hwctx_walk(client, NULL, walk: aie2_hwctx_suspend_cb);
159}
160
161static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg)
162{
163 struct amdxdna_dev *xdna = hwctx->client->xdna;
164
165 aie2_hwctx_status_restore(hwctx);
166 return aie2_hwctx_restart(xdna, hwctx);
167}
168
169int aie2_hwctx_resume(struct amdxdna_client *client)
170{
171 /*
172 * The resume path cannot guarantee that mailbox channel can be
173 * regenerated. If this happen, when submit message to this
174 * mailbox channel, error will return.
175 */
176 return amdxdna_hwctx_walk(client, NULL, walk: aie2_hwctx_resume_cb);
177}
178
179static void
180aie2_sched_notify(struct amdxdna_sched_job *job)
181{
182 struct dma_fence *fence = job->fence;
183
184 trace_xdna_job(sched_job: &job->base, name: job->hwctx->name, str: "signaled fence", seq: job->seq);
185
186 amdxdna_pm_suspend_put(xdna: job->hwctx->client->xdna);
187 job->hwctx->priv->completed++;
188 dma_fence_signal(fence);
189
190 up(sem: &job->hwctx->priv->job_sem);
191 job->job_done = true;
192 mmput_async(job->mm);
193 aie2_job_put(job);
194}
195
196static int
197aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size)
198{
199 struct amdxdna_sched_job *job = handle;
200 struct amdxdna_gem_obj *cmd_abo;
201 int ret = 0;
202 u32 status;
203
204 cmd_abo = job->cmd_bo;
205
206 if (unlikely(job->job_timeout)) {
207 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_TIMEOUT);
208 ret = -EINVAL;
209 goto out;
210 }
211
212 if (unlikely(!data) || unlikely(size != sizeof(u32))) {
213 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_ABORT);
214 ret = -EINVAL;
215 goto out;
216 }
217
218 status = readl(addr: data);
219 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status);
220 if (status == AIE2_STATUS_SUCCESS)
221 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_COMPLETED);
222 else
223 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_ERROR);
224
225out:
226 aie2_sched_notify(job);
227 return ret;
228}
229
230static int
231aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size)
232{
233 struct amdxdna_sched_job *job = handle;
234 int ret = 0;
235
236 if (unlikely(!data))
237 goto out;
238
239 if (unlikely(size != sizeof(u32))) {
240 ret = -EINVAL;
241 goto out;
242 }
243
244 job->drv_cmd->result = readl(addr: data);
245
246out:
247 aie2_sched_notify(job);
248 return ret;
249}
250
251static int
252aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size)
253{
254 struct amdxdna_sched_job *job = handle;
255 struct amdxdna_gem_obj *cmd_abo;
256 struct amdxdna_dev *xdna;
257 u32 fail_cmd_status;
258 u32 fail_cmd_idx;
259 u32 cmd_status;
260 int ret = 0;
261
262 cmd_abo = job->cmd_bo;
263
264 if (unlikely(job->job_timeout)) {
265 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_TIMEOUT);
266 ret = -EINVAL;
267 goto out;
268 }
269
270 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) {
271 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_ABORT);
272 ret = -EINVAL;
273 goto out;
274 }
275
276 cmd_status = readl(addr: data + offsetof(struct cmd_chain_resp, status));
277 xdna = job->hwctx->client->xdna;
278 XDNA_DBG(xdna, "Status 0x%x", cmd_status);
279 if (cmd_status == AIE2_STATUS_SUCCESS) {
280 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_COMPLETED);
281 goto out;
282 }
283
284 /* Slow path to handle error, read from ringbuf on BAR */
285 fail_cmd_idx = readl(addr: data + offsetof(struct cmd_chain_resp, fail_cmd_idx));
286 fail_cmd_status = readl(addr: data + offsetof(struct cmd_chain_resp, fail_cmd_status));
287 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x",
288 fail_cmd_idx, fail_cmd_status);
289
290 if (fail_cmd_status == AIE2_STATUS_SUCCESS) {
291 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_ABORT);
292 ret = -EINVAL;
293 goto out;
294 }
295 amdxdna_cmd_set_state(abo: cmd_abo, s: fail_cmd_status);
296
297 if (amdxdna_cmd_get_op(abo: cmd_abo) == ERT_CMD_CHAIN) {
298 struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(abo: cmd_abo, NULL);
299
300 cc->error_index = fail_cmd_idx;
301 if (cc->error_index >= cc->command_count)
302 cc->error_index = 0;
303 }
304out:
305 aie2_sched_notify(job);
306 return ret;
307}
308
309static struct dma_fence *
310aie2_sched_job_run(struct drm_sched_job *sched_job)
311{
312 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
313 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
314 struct amdxdna_hwctx *hwctx = job->hwctx;
315 struct dma_fence *fence;
316 int ret;
317
318 if (!mmget_not_zero(mm: job->mm))
319 return ERR_PTR(error: -ESRCH);
320
321 kref_get(kref: &job->refcnt);
322 fence = dma_fence_get(fence: job->fence);
323
324 if (job->drv_cmd) {
325 switch (job->drv_cmd->opcode) {
326 case SYNC_DEBUG_BO:
327 ret = aie2_sync_bo(hwctx, job, notify_cb: aie2_sched_drvcmd_resp_handler);
328 break;
329 case ATTACH_DEBUG_BO:
330 ret = aie2_config_debug_bo(hwctx, job, notify_cb: aie2_sched_drvcmd_resp_handler);
331 break;
332 default:
333 ret = -EINVAL;
334 break;
335 }
336 goto out;
337 }
338
339 amdxdna_cmd_set_state(abo: cmd_abo, s: ERT_CMD_STATE_NEW);
340
341 if (amdxdna_cmd_get_op(abo: cmd_abo) == ERT_CMD_CHAIN)
342 ret = aie2_cmdlist_multi_execbuf(hwctx, job, notify_cb: aie2_sched_cmdlist_resp_handler);
343 else if (force_cmdlist)
344 ret = aie2_cmdlist_single_execbuf(hwctx, job, notify_cb: aie2_sched_cmdlist_resp_handler);
345 else
346 ret = aie2_execbuf(hwctx, job, notify_cb: aie2_sched_resp_handler);
347
348out:
349 if (ret) {
350 dma_fence_put(fence: job->fence);
351 aie2_job_put(job);
352 mmput(job->mm);
353 fence = ERR_PTR(error: ret);
354 }
355 trace_xdna_job(sched_job, name: hwctx->name, str: "sent to device", seq: job->seq);
356
357 return fence;
358}
359
360static void aie2_sched_job_free(struct drm_sched_job *sched_job)
361{
362 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
363 struct amdxdna_hwctx *hwctx = job->hwctx;
364
365 trace_xdna_job(sched_job, name: hwctx->name, str: "job free", seq: job->seq);
366 if (!job->job_done)
367 up(sem: &hwctx->priv->job_sem);
368
369 drm_sched_job_cleanup(job: sched_job);
370 aie2_job_put(job);
371}
372
373static enum drm_gpu_sched_stat
374aie2_sched_job_timedout(struct drm_sched_job *sched_job)
375{
376 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
377 struct amdxdna_hwctx *hwctx = job->hwctx;
378 struct amdxdna_dev *xdna;
379
380 xdna = hwctx->client->xdna;
381 trace_xdna_job(sched_job, name: hwctx->name, str: "job timedout", seq: job->seq);
382 job->job_timeout = true;
383 mutex_lock(&xdna->dev_lock);
384 aie2_hwctx_stop(xdna, hwctx, bad_job: sched_job);
385
386 aie2_hwctx_restart(xdna, hwctx);
387 mutex_unlock(lock: &xdna->dev_lock);
388
389 return DRM_GPU_SCHED_STAT_RESET;
390}
391
392static const struct drm_sched_backend_ops sched_ops = {
393 .run_job = aie2_sched_job_run,
394 .free_job = aie2_sched_job_free,
395 .timedout_job = aie2_sched_job_timedout,
396};
397
398static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx)
399{
400 struct amdxdna_dev *xdna = hwctx->client->xdna;
401 struct amdxdna_dev_hdl *ndev;
402 int start, end, first, last;
403 u32 width = 1, entries = 0;
404 int i;
405
406 if (!hwctx->num_tiles) {
407 XDNA_ERR(xdna, "Number of tiles is zero");
408 return -EINVAL;
409 }
410
411 ndev = xdna->dev_handle;
412 if (unlikely(!ndev->metadata.core.row_count)) {
413 XDNA_WARN(xdna, "Core tile row count is zero");
414 return -EINVAL;
415 }
416
417 hwctx->num_col = hwctx->num_tiles / ndev->metadata.core.row_count;
418 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) {
419 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col);
420 return -EINVAL;
421 }
422
423 if (ndev->priv->col_align == COL_ALIGN_NATURE)
424 width = hwctx->num_col;
425
426 /*
427 * In range [start, end], find out columns that is multiple of width.
428 * 'first' is the first column,
429 * 'last' is the last column,
430 * 'entries' is the total number of columns.
431 */
432 start = xdna->dev_info->first_col;
433 end = ndev->total_col - hwctx->num_col;
434 if (start > 0 && end == 0) {
435 XDNA_DBG(xdna, "Force start from col 0");
436 start = 0;
437 }
438 first = start + (width - start % width) % width;
439 last = end - end % width;
440 if (last >= first)
441 entries = (last - first) / width + 1;
442 XDNA_DBG(xdna, "start %d end %d first %d last %d",
443 start, end, first, last);
444
445 if (unlikely(!entries)) {
446 XDNA_ERR(xdna, "Start %d end %d width %d",
447 start, end, width);
448 return -EINVAL;
449 }
450
451 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL);
452 if (!hwctx->col_list)
453 return -ENOMEM;
454
455 hwctx->col_list_len = entries;
456 hwctx->col_list[0] = first;
457 for (i = 1; i < entries; i++)
458 hwctx->col_list[i] = hwctx->col_list[i - 1] + width;
459
460 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list,
461 entries * sizeof(*hwctx->col_list), false);
462 return 0;
463}
464
465static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx)
466{
467 struct amdxdna_dev *xdna = hwctx->client->xdna;
468 struct alloc_requests *xrs_req;
469 int ret;
470
471 xrs_req = kzalloc(sizeof(*xrs_req), GFP_KERNEL);
472 if (!xrs_req)
473 return -ENOMEM;
474
475 xrs_req->cdo.start_cols = hwctx->col_list;
476 xrs_req->cdo.cols_len = hwctx->col_list_len;
477 xrs_req->cdo.ncols = hwctx->num_col;
478 xrs_req->cdo.qos_cap.opc = hwctx->max_opc;
479
480 xrs_req->rqos.gops = hwctx->qos.gops;
481 xrs_req->rqos.fps = hwctx->qos.fps;
482 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth;
483 xrs_req->rqos.latency = hwctx->qos.latency;
484 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time;
485 xrs_req->rqos.priority = hwctx->qos.priority;
486
487 xrs_req->rid = (uintptr_t)hwctx;
488
489 ret = xrs_allocate_resource(hdl: xdna->xrs_hdl, req: xrs_req, cb_arg: hwctx);
490 if (ret)
491 XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret);
492
493 kfree(objp: xrs_req);
494 return ret;
495}
496
497static void aie2_release_resource(struct amdxdna_hwctx *hwctx)
498{
499 struct amdxdna_dev *xdna = hwctx->client->xdna;
500 int ret;
501
502 ret = xrs_release_resource(hdl: xdna->xrs_hdl, rid: (uintptr_t)hwctx);
503 if (ret)
504 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
505}
506
507static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx)
508{
509 struct amdxdna_dev *xdna = hwctx->client->xdna;
510 struct drm_file *filp = hwctx->client->filp;
511 struct drm_syncobj *syncobj;
512 u32 hdl;
513 int ret;
514
515 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE;
516
517 ret = drm_syncobj_create(out_syncobj: &syncobj, flags: 0, NULL);
518 if (ret) {
519 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret);
520 return ret;
521 }
522 ret = drm_syncobj_get_handle(file_private: filp, syncobj, handle: &hdl);
523 if (ret) {
524 drm_syncobj_put(obj: syncobj);
525 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret);
526 return ret;
527 }
528 hwctx->priv->syncobj = syncobj;
529 hwctx->syncobj_hdl = hdl;
530
531 return 0;
532}
533
534static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx)
535{
536 /*
537 * The syncobj_hdl is owned by user space and will be cleaned up
538 * separately.
539 */
540 drm_syncobj_put(obj: hwctx->priv->syncobj);
541}
542
543int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
544{
545 struct amdxdna_client *client = hwctx->client;
546 struct amdxdna_dev *xdna = client->xdna;
547 const struct drm_sched_init_args args = {
548 .ops = &sched_ops,
549 .num_rqs = DRM_SCHED_PRIORITY_COUNT,
550 .credit_limit = HWCTX_MAX_CMDS,
551 .timeout = msecs_to_jiffies(HWCTX_MAX_TIMEOUT),
552 .name = "amdxdna_js",
553 .dev = xdna->ddev.dev,
554 };
555 struct drm_gpu_scheduler *sched;
556 struct amdxdna_hwctx_priv *priv;
557 struct amdxdna_gem_obj *heap;
558 int i, ret;
559
560 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL);
561 if (!priv)
562 return -ENOMEM;
563 hwctx->priv = priv;
564
565 mutex_lock(&client->mm_lock);
566 heap = client->dev_heap;
567 if (!heap) {
568 XDNA_ERR(xdna, "The client dev heap object not exist");
569 mutex_unlock(lock: &client->mm_lock);
570 ret = -ENOENT;
571 goto free_priv;
572 }
573 drm_gem_object_get(to_gobj(heap));
574 mutex_unlock(lock: &client->mm_lock);
575 priv->heap = heap;
576 sema_init(sem: &priv->job_sem, HWCTX_MAX_CMDS);
577
578 ret = amdxdna_gem_pin(abo: heap);
579 if (ret) {
580 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret);
581 goto put_heap;
582 }
583
584 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {
585 struct amdxdna_gem_obj *abo;
586 struct amdxdna_drm_create_bo args = {
587 .flags = 0,
588 .type = AMDXDNA_BO_DEV,
589 .vaddr = 0,
590 .size = MAX_CHAIN_CMDBUF_SIZE,
591 };
592
593 abo = amdxdna_drm_alloc_dev_bo(dev: &xdna->ddev, args: &args, filp: client->filp);
594 if (IS_ERR(ptr: abo)) {
595 ret = PTR_ERR(ptr: abo);
596 goto free_cmd_bufs;
597 }
598
599 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx",
600 i, abo->mem.dev_addr, abo->mem.size);
601 priv->cmd_buf[i] = abo;
602 }
603
604 sched = &priv->sched;
605 mutex_init(&priv->io_lock);
606
607 fs_reclaim_acquire(GFP_KERNEL);
608 might_lock(&priv->io_lock);
609 fs_reclaim_release(GFP_KERNEL);
610
611 ret = drm_sched_init(sched, args: &args);
612 if (ret) {
613 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);
614 goto free_cmd_bufs;
615 }
616
617 ret = drm_sched_entity_init(entity: &priv->entity, priority: DRM_SCHED_PRIORITY_NORMAL,
618 sched_list: &sched, num_sched_list: 1, NULL);
619 if (ret) {
620 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret);
621 goto free_sched;
622 }
623
624 ret = aie2_hwctx_col_list(hwctx);
625 if (ret) {
626 XDNA_ERR(xdna, "Create col list failed, ret %d", ret);
627 goto free_entity;
628 }
629
630 ret = amdxdna_pm_resume_get(xdna);
631 if (ret)
632 goto free_col_list;
633
634 ret = aie2_alloc_resource(hwctx);
635 if (ret) {
636 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret);
637 goto suspend_put;
638 }
639
640 ret = aie2_map_host_buf(ndev: xdna->dev_handle, context_id: hwctx->fw_ctx_id,
641 addr: heap->mem.userptr, size: heap->mem.size);
642 if (ret) {
643 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret);
644 goto release_resource;
645 }
646
647 ret = aie2_ctx_syncobj_create(hwctx);
648 if (ret) {
649 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret);
650 goto release_resource;
651 }
652 amdxdna_pm_suspend_put(xdna);
653
654 hwctx->status = HWCTX_STAT_INIT;
655 init_waitqueue_head(&priv->job_free_wq);
656
657 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name);
658
659 return 0;
660
661release_resource:
662 aie2_release_resource(hwctx);
663suspend_put:
664 amdxdna_pm_suspend_put(xdna);
665free_col_list:
666 kfree(objp: hwctx->col_list);
667free_entity:
668 drm_sched_entity_destroy(entity: &priv->entity);
669free_sched:
670 drm_sched_fini(sched: &priv->sched);
671free_cmd_bufs:
672 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {
673 if (!priv->cmd_buf[i])
674 continue;
675 drm_gem_object_put(to_gobj(priv->cmd_buf[i]));
676 }
677 amdxdna_gem_unpin(abo: heap);
678put_heap:
679 drm_gem_object_put(to_gobj(heap));
680free_priv:
681 kfree(objp: priv);
682 return ret;
683}
684
685void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
686{
687 struct amdxdna_dev *xdna;
688 int idx;
689
690 xdna = hwctx->client->xdna;
691
692 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq);
693 aie2_hwctx_wait_for_idle(hwctx);
694
695 /* Request fw to destroy hwctx and cancel the rest pending requests */
696 aie2_release_resource(hwctx);
697
698 mutex_unlock(lock: &xdna->dev_lock);
699 drm_sched_entity_destroy(entity: &hwctx->priv->entity);
700
701 /* Wait for all submitted jobs to be completed or canceled */
702 wait_event(hwctx->priv->job_free_wq,
703 atomic64_read(&hwctx->job_submit_cnt) ==
704 atomic64_read(&hwctx->job_free_cnt));
705 mutex_lock(&xdna->dev_lock);
706
707 drm_sched_fini(sched: &hwctx->priv->sched);
708 aie2_ctx_syncobj_destroy(hwctx);
709
710 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++)
711 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx]));
712 amdxdna_gem_unpin(abo: hwctx->priv->heap);
713 drm_gem_object_put(to_gobj(hwctx->priv->heap));
714
715 mutex_destroy(lock: &hwctx->priv->io_lock);
716 kfree(objp: hwctx->col_list);
717 kfree(objp: hwctx->priv);
718 kfree(objp: hwctx->cus);
719}
720
721static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size)
722{
723 struct amdxdna_hwctx *hwctx = handle;
724
725 amdxdna_pm_suspend_put(xdna: hwctx->client->xdna);
726 return 0;
727}
728
729static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size)
730{
731 struct amdxdna_hwctx_param_config_cu *config = buf;
732 struct amdxdna_dev *xdna = hwctx->client->xdna;
733 u32 total_size;
734 int ret;
735
736 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name);
737 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad)))
738 return -EINVAL;
739
740 if (hwctx->status != HWCTX_STAT_INIT) {
741 XDNA_ERR(xdna, "Not support re-config CU");
742 return -EINVAL;
743 }
744
745 if (!config->num_cus) {
746 XDNA_ERR(xdna, "Number of CU is zero");
747 return -EINVAL;
748 }
749
750 total_size = struct_size(config, cu_configs, config->num_cus);
751 if (total_size > size) {
752 XDNA_ERR(xdna, "CU config larger than size");
753 return -EINVAL;
754 }
755
756 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL);
757 if (!hwctx->cus)
758 return -ENOMEM;
759
760 ret = amdxdna_pm_resume_get(xdna);
761 if (ret)
762 goto free_cus;
763
764 ret = aie2_config_cu(hwctx, notify_cb: aie2_config_cu_resp_handler);
765 if (ret) {
766 XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret);
767 goto pm_suspend_put;
768 }
769
770 wmb(); /* To avoid locking in command submit when check status */
771 hwctx->status = HWCTX_STAT_READY;
772
773 return 0;
774
775pm_suspend_put:
776 amdxdna_pm_suspend_put(xdna);
777free_cus:
778 kfree(objp: hwctx->cus);
779 hwctx->cus = NULL;
780 return ret;
781}
782
783static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq)
784{
785 struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq);
786
787 if (!out_fence) {
788 XDNA_ERR(hwctx->client->xdna, "Failed to get fence");
789 return;
790 }
791
792 dma_fence_wait_timeout(out_fence, intr: false, MAX_SCHEDULE_TIMEOUT);
793 dma_fence_put(fence: out_fence);
794}
795
796static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl,
797 bool attach)
798{
799 struct amdxdna_client *client = hwctx->client;
800 struct amdxdna_dev *xdna = client->xdna;
801 struct amdxdna_drv_cmd cmd = { 0 };
802 struct amdxdna_gem_obj *abo;
803 u64 seq;
804 int ret;
805
806 abo = amdxdna_gem_get_obj(client, bo_hdl, bo_type: AMDXDNA_BO_DEV);
807 if (!abo) {
808 XDNA_ERR(xdna, "Get bo %d failed", bo_hdl);
809 return -EINVAL;
810 }
811
812 if (attach) {
813 if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) {
814 ret = -EBUSY;
815 goto put_obj;
816 }
817 cmd.opcode = ATTACH_DEBUG_BO;
818 } else {
819 if (abo->assigned_hwctx != hwctx->id) {
820 ret = -EINVAL;
821 goto put_obj;
822 }
823 cmd.opcode = DETACH_DEBUG_BO;
824 }
825
826 ret = amdxdna_cmd_submit(client, drv_cmd: &cmd, AMDXDNA_INVALID_BO_HANDLE,
827 arg_bo_hdls: &bo_hdl, arg_bo_cnt: 1, hwctx_hdl: hwctx->id, seq: &seq);
828 if (ret) {
829 XDNA_ERR(xdna, "Submit command failed");
830 goto put_obj;
831 }
832
833 aie2_cmd_wait(hwctx, seq);
834 if (cmd.result) {
835 XDNA_ERR(xdna, "Response failure 0x%x", cmd.result);
836 goto put_obj;
837 }
838
839 if (attach)
840 abo->assigned_hwctx = hwctx->id;
841 else
842 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
843
844 XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name);
845
846put_obj:
847 amdxdna_gem_put_obj(abo);
848 return ret;
849}
850
851int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size)
852{
853 struct amdxdna_dev *xdna = hwctx->client->xdna;
854
855 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
856 switch (type) {
857 case DRM_AMDXDNA_HWCTX_CONFIG_CU:
858 return aie2_hwctx_cu_config(hwctx, buf, size);
859 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
860 return aie2_hwctx_cfg_debug_bo(hwctx, bo_hdl: (u32)value, attach: true);
861 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
862 return aie2_hwctx_cfg_debug_bo(hwctx, bo_hdl: (u32)value, attach: false);
863 default:
864 XDNA_DBG(xdna, "Not supported type %d", type);
865 return -EOPNOTSUPP;
866 }
867}
868
869int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl)
870{
871 struct amdxdna_client *client = hwctx->client;
872 struct amdxdna_dev *xdna = client->xdna;
873 struct amdxdna_drv_cmd cmd = { 0 };
874 u64 seq;
875 int ret;
876
877 cmd.opcode = SYNC_DEBUG_BO;
878 ret = amdxdna_cmd_submit(client, drv_cmd: &cmd, AMDXDNA_INVALID_BO_HANDLE,
879 arg_bo_hdls: &debug_bo_hdl, arg_bo_cnt: 1, hwctx_hdl: hwctx->id, seq: &seq);
880 if (ret) {
881 XDNA_ERR(xdna, "Submit command failed");
882 return ret;
883 }
884
885 aie2_cmd_wait(hwctx, seq);
886 if (cmd.result) {
887 XDNA_ERR(xdna, "Response failure 0x%x", cmd.result);
888 return -EINVAL;
889 }
890
891 return 0;
892}
893
894static int aie2_populate_range(struct amdxdna_gem_obj *abo)
895{
896 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
897 struct amdxdna_umap *mapp;
898 unsigned long timeout;
899 struct mm_struct *mm;
900 bool found;
901 int ret;
902
903 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
904again:
905 found = false;
906 down_write(sem: &xdna->notifier_lock);
907 list_for_each_entry(mapp, &abo->mem.umap_list, node) {
908 if (mapp->invalid) {
909 found = true;
910 break;
911 }
912 }
913
914 if (!found) {
915 abo->mem.map_invalid = false;
916 up_write(sem: &xdna->notifier_lock);
917 return 0;
918 }
919 kref_get(kref: &mapp->refcnt);
920 up_write(sem: &xdna->notifier_lock);
921
922 XDNA_DBG(xdna, "populate memory range %lx %lx",
923 mapp->vma->vm_start, mapp->vma->vm_end);
924 mm = mapp->notifier.mm;
925 if (!mmget_not_zero(mm)) {
926 amdxdna_umap_put(mapp);
927 return -EFAULT;
928 }
929
930 mapp->range.notifier_seq = mmu_interval_read_begin(interval_sub: &mapp->notifier);
931 mmap_read_lock(mm);
932 ret = hmm_range_fault(range: &mapp->range);
933 mmap_read_unlock(mm);
934 if (ret) {
935 if (time_after(jiffies, timeout)) {
936 ret = -ETIME;
937 goto put_mm;
938 }
939
940 if (ret == -EBUSY) {
941 amdxdna_umap_put(mapp);
942 goto again;
943 }
944
945 goto put_mm;
946 }
947
948 down_write(sem: &xdna->notifier_lock);
949 if (mmu_interval_read_retry(interval_sub: &mapp->notifier, seq: mapp->range.notifier_seq)) {
950 up_write(sem: &xdna->notifier_lock);
951 amdxdna_umap_put(mapp);
952 goto again;
953 }
954 mapp->invalid = false;
955 up_write(sem: &xdna->notifier_lock);
956 amdxdna_umap_put(mapp);
957 goto again;
958
959put_mm:
960 amdxdna_umap_put(mapp);
961 mmput(mm);
962 return ret;
963}
964
965int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq)
966{
967 struct amdxdna_dev *xdna = hwctx->client->xdna;
968 struct ww_acquire_ctx acquire_ctx;
969 struct dma_fence_chain *chain;
970 struct amdxdna_gem_obj *abo;
971 unsigned long timeout = 0;
972 int ret, i;
973
974 ret = down_interruptible(sem: &hwctx->priv->job_sem);
975 if (ret) {
976 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret);
977 return ret;
978 }
979
980 chain = dma_fence_chain_alloc();
981 if (!chain) {
982 XDNA_ERR(xdna, "Alloc fence chain failed");
983 ret = -ENOMEM;
984 goto up_sem;
985 }
986
987 ret = drm_sched_job_init(job: &job->base, entity: &hwctx->priv->entity, credits: 1, owner: hwctx,
988 drm_client_id: hwctx->client->filp->client_id);
989 if (ret) {
990 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret);
991 goto free_chain;
992 }
993
994 ret = amdxdna_pm_resume_get(xdna);
995 if (ret)
996 goto cleanup_job;
997
998retry:
999 ret = drm_gem_lock_reservations(objs: job->bos, count: job->bo_cnt, acquire_ctx: &acquire_ctx);
1000 if (ret) {
1001 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret);
1002 goto suspend_put;
1003 }
1004
1005 for (i = 0; i < job->bo_cnt; i++) {
1006 ret = dma_resv_reserve_fences(obj: job->bos[i]->resv, num_fences: 1);
1007 if (ret) {
1008 XDNA_WARN(xdna, "Failed to reserve fences %d", ret);
1009 drm_gem_unlock_reservations(objs: job->bos, count: job->bo_cnt, acquire_ctx: &acquire_ctx);
1010 goto suspend_put;
1011 }
1012 }
1013
1014 down_read(sem: &xdna->notifier_lock);
1015 for (i = 0; i < job->bo_cnt; i++) {
1016 abo = to_xdna_obj(gobj: job->bos[i]);
1017 if (abo->mem.map_invalid) {
1018 up_read(sem: &xdna->notifier_lock);
1019 drm_gem_unlock_reservations(objs: job->bos, count: job->bo_cnt, acquire_ctx: &acquire_ctx);
1020 if (!timeout) {
1021 timeout = jiffies +
1022 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1023 } else if (time_after(jiffies, timeout)) {
1024 ret = -ETIME;
1025 goto suspend_put;
1026 }
1027
1028 ret = aie2_populate_range(abo);
1029 if (ret)
1030 goto suspend_put;
1031 goto retry;
1032 }
1033 }
1034
1035 mutex_lock(&hwctx->priv->io_lock);
1036 drm_sched_job_arm(job: &job->base);
1037 job->out_fence = dma_fence_get(fence: &job->base.s_fence->finished);
1038 for (i = 0; i < job->bo_cnt; i++)
1039 dma_resv_add_fence(obj: job->bos[i]->resv, fence: job->out_fence, usage: DMA_RESV_USAGE_WRITE);
1040 job->seq = hwctx->priv->seq++;
1041 kref_get(kref: &job->refcnt);
1042 drm_sched_entity_push_job(sched_job: &job->base);
1043
1044 *seq = job->seq;
1045 drm_syncobj_add_point(syncobj: hwctx->priv->syncobj, chain, fence: job->out_fence, point: *seq);
1046 mutex_unlock(lock: &hwctx->priv->io_lock);
1047
1048 up_read(sem: &xdna->notifier_lock);
1049 drm_gem_unlock_reservations(objs: job->bos, count: job->bo_cnt, acquire_ctx: &acquire_ctx);
1050
1051 aie2_job_put(job);
1052 atomic64_inc(v: &hwctx->job_submit_cnt);
1053
1054 return 0;
1055
1056suspend_put:
1057 amdxdna_pm_suspend_put(xdna);
1058cleanup_job:
1059 drm_sched_job_cleanup(job: &job->base);
1060free_chain:
1061 dma_fence_chain_free(chain);
1062up_sem:
1063 up(sem: &hwctx->priv->job_sem);
1064 job->job_done = true;
1065 return ret;
1066}
1067
1068void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo,
1069 unsigned long cur_seq)
1070{
1071 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
1072 struct drm_gem_object *gobj = to_gobj(abo);
1073 long ret;
1074
1075 ret = dma_resv_wait_timeout(obj: gobj->resv, usage: DMA_RESV_USAGE_BOOKKEEP,
1076 intr: true, MAX_SCHEDULE_TIMEOUT);
1077 if (!ret || ret == -ERESTARTSYS)
1078 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret);
1079}
1080

source code of linux/drivers/accel/amdxdna/aie2_ctx.c