1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_device.h>
8#include <drm/drm_drv.h>
9#include <drm/drm_gem_shmem_helper.h>
10#include <drm/drm_managed.h>
11#include <drm/drm_print.h>
12#include <drm/gpu_scheduler.h>
13#include <linux/cleanup.h>
14#include <linux/errno.h>
15#include <linux/firmware.h>
16#include <linux/iommu.h>
17#include <linux/iopoll.h>
18#include <linux/pci.h>
19#include <linux/xarray.h>
20#include <asm/hypervisor.h>
21
22#include "aie2_msg_priv.h"
23#include "aie2_pci.h"
24#include "aie2_solver.h"
25#include "amdxdna_ctx.h"
26#include "amdxdna_gem.h"
27#include "amdxdna_mailbox.h"
28#include "amdxdna_pci_drv.h"
29#include "amdxdna_pm.h"
30
31static int aie2_max_col = XRS_MAX_COL;
32module_param(aie2_max_col, uint, 0600);
33MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used");
34
35/*
36 * The management mailbox channel is allocated by firmware.
37 * The related register and ring buffer information is on SRAM BAR.
38 * This struct is the register layout.
39 */
40#define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */
41struct mgmt_mbox_chann_info {
42 __u32 x2i_tail;
43 __u32 x2i_head;
44 __u32 x2i_buf;
45 __u32 x2i_buf_sz;
46 __u32 i2x_tail;
47 __u32 i2x_head;
48 __u32 i2x_buf;
49 __u32 i2x_buf_sz;
50 __u32 magic;
51 __u32 msi_id;
52 __u32 prot_major;
53 __u32 prot_minor;
54 __u32 rsvd[4];
55};
56
57static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor)
58{
59 const struct aie2_fw_feature_tbl *feature;
60 struct amdxdna_dev *xdna = ndev->xdna;
61
62 /*
63 * The driver supported mailbox behavior is defined by
64 * ndev->priv->protocol_major and protocol_minor.
65 *
66 * When protocol_major and fw_major are different, it means driver
67 * and firmware are incompatible.
68 */
69 if (ndev->priv->protocol_major != fw_major) {
70 XDNA_ERR(xdna, "Incompatible firmware protocol major %d minor %d",
71 fw_major, fw_minor);
72 return -EINVAL;
73 }
74
75 /*
76 * When protocol_minor is greater then fw_minor, that means driver
77 * relies on operation the installed firmware does not support.
78 */
79 if (ndev->priv->protocol_minor > fw_minor) {
80 XDNA_ERR(xdna, "Firmware minor version smaller than supported");
81 return -EINVAL;
82 }
83
84 for (feature = ndev->priv->fw_feature_tbl; feature && feature->min_minor;
85 feature++) {
86 if (fw_minor < feature->min_minor)
87 continue;
88 if (feature->max_minor > 0 && fw_minor > feature->max_minor)
89 continue;
90
91 set_bit(nr: feature->feature, addr: &ndev->feature_mask);
92 }
93
94 return 0;
95}
96
97static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
98{
99 struct amdxdna_dev *xdna = ndev->xdna;
100
101 XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
102 XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
103 XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
104 XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size);
105 XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
106 XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
107 XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
108 XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size);
109 XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
110 XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major);
111 XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor);
112}
113
114static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
115{
116 struct mgmt_mbox_chann_info info_regs;
117 struct xdna_mailbox_chann_res *i2x;
118 struct xdna_mailbox_chann_res *x2i;
119 u32 addr, off;
120 u32 *reg;
121 int ret;
122 int i;
123
124 /*
125 * Once firmware is alive, it will write management channel
126 * information in SRAM BAR and write the address of that information
127 * at FW_ALIVE_OFF offset in SRMA BAR.
128 *
129 * Read a non-zero value from FW_ALIVE_OFF implies that firmware
130 * is alive.
131 */
132 ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
133 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
134 if (ret || !addr)
135 return -ETIME;
136
137 off = AIE2_SRAM_OFF(ndev, addr);
138 reg = (u32 *)&info_regs;
139 for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++)
140 reg[i] = readl(addr: ndev->sram_base + off + i * sizeof(u32));
141
142 if (info_regs.magic != MGMT_MBOX_MAGIC) {
143 XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic);
144 ret = -EINVAL;
145 goto done;
146 }
147
148 i2x = &ndev->mgmt_i2x;
149 x2i = &ndev->mgmt_x2i;
150
151 i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
152 i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
153 i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf);
154 i2x->rb_size = info_regs.i2x_buf_sz;
155
156 x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head);
157 x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail);
158 x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
159 x2i->rb_size = info_regs.x2i_buf_sz;
160
161 ndev->mgmt_chan_idx = info_regs.msi_id;
162 ndev->mgmt_prot_major = info_regs.prot_major;
163 ndev->mgmt_prot_minor = info_regs.prot_minor;
164
165 ret = aie2_check_protocol(ndev, fw_major: ndev->mgmt_prot_major, fw_minor: ndev->mgmt_prot_minor);
166
167done:
168 aie2_dump_chann_info_debug(ndev);
169
170 /* Must clear address at FW_ALIVE_OFF */
171 writel(val: 0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
172
173 return ret;
174}
175
176int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
177 enum rt_config_category category, u32 *val)
178{
179 const struct rt_config *cfg;
180 u32 value;
181 int ret;
182
183 for (cfg = ndev->priv->rt_config; cfg->type; cfg++) {
184 if (cfg->category != category)
185 continue;
186
187 if (cfg->feature_mask &&
188 bitmap_subset(src1: &cfg->feature_mask, src2: &ndev->feature_mask, nbits: AIE2_FEATURE_MAX))
189 continue;
190
191 value = val ? *val : cfg->value;
192 ret = aie2_set_runtime_cfg(ndev, type: cfg->type, value);
193 if (ret) {
194 XDNA_ERR(ndev->xdna, "Set type %d value %d failed",
195 cfg->type, value);
196 return ret;
197 }
198 }
199
200 return 0;
201}
202
203static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
204{
205 int ret;
206
207 ret = aie2_suspend_fw(ndev);
208 if (ret) {
209 XDNA_ERR(ndev->xdna, "Suspend firmware failed");
210 return ret;
211 }
212
213 ret = aie2_resume_fw(ndev);
214 if (ret) {
215 XDNA_ERR(ndev->xdna, "Resume firmware failed");
216 return ret;
217 }
218
219 return 0;
220}
221
222static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
223{
224 int ret;
225
226 ret = aie2_runtime_cfg(ndev, category: AIE2_RT_CFG_INIT, NULL);
227 if (ret) {
228 XDNA_ERR(ndev->xdna, "Runtime config failed");
229 return ret;
230 }
231
232 ret = aie2_assign_mgmt_pasid(ndev, pasid: 0);
233 if (ret) {
234 XDNA_ERR(ndev->xdna, "Can not assign PASID");
235 return ret;
236 }
237
238 ret = aie2_xdna_reset(ndev);
239 if (ret) {
240 XDNA_ERR(ndev->xdna, "Reset firmware failed");
241 return ret;
242 }
243
244 return 0;
245}
246
247static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
248{
249 int ret;
250
251 ret = aie2_query_firmware_version(ndev, fw_ver: &ndev->xdna->fw_ver);
252 if (ret) {
253 XDNA_ERR(ndev->xdna, "query firmware version failed");
254 return ret;
255 }
256
257 ret = aie2_query_aie_version(ndev, version: &ndev->version);
258 if (ret) {
259 XDNA_ERR(ndev->xdna, "Query AIE version failed");
260 return ret;
261 }
262
263 ret = aie2_query_aie_metadata(ndev, metadata: &ndev->metadata);
264 if (ret) {
265 XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
266 return ret;
267 }
268
269 ndev->total_col = min(aie2_max_col, ndev->metadata.cols);
270
271 return 0;
272}
273
274static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
275{
276 if (aie2_suspend_fw(ndev))
277 XDNA_ERR(ndev->xdna, "Suspend_fw failed");
278 XDNA_DBG(ndev->xdna, "Firmware suspended");
279}
280
281static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
282{
283 struct amdxdna_hwctx *hwctx = cb_arg;
284 struct amdxdna_dev *xdna;
285 int ret;
286
287 xdna = hwctx->client->xdna;
288
289 hwctx->start_col = action->part.start_col;
290 hwctx->num_col = action->part.ncols;
291 ret = aie2_create_context(ndev: xdna->dev_handle, hwctx);
292 if (ret)
293 XDNA_ERR(xdna, "create context failed, ret %d", ret);
294
295 return ret;
296}
297
298static int aie2_xrs_unload(void *cb_arg)
299{
300 struct amdxdna_hwctx *hwctx = cb_arg;
301 struct amdxdna_dev *xdna;
302 int ret;
303
304 xdna = hwctx->client->xdna;
305
306 ret = aie2_destroy_context(ndev: xdna->dev_handle, hwctx);
307 if (ret)
308 XDNA_ERR(xdna, "destroy context failed, ret %d", ret);
309
310 return ret;
311}
312
313static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level)
314{
315 struct amdxdna_dev *xdna = to_xdna_dev(ddev);
316 struct amdxdna_dev_hdl *ndev;
317
318 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
319
320 ndev = xdna->dev_handle;
321 ndev->dft_dpm_level = dpm_level;
322 if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level)
323 return 0;
324
325 return ndev->priv->hw_ops.set_dpm(ndev, dpm_level);
326}
327
328static struct xrs_action_ops aie2_xrs_actions = {
329 .load = aie2_xrs_load,
330 .unload = aie2_xrs_unload,
331 .set_dft_dpm_level = aie2_xrs_set_dft_dpm_level,
332};
333
334static void aie2_hw_stop(struct amdxdna_dev *xdna)
335{
336 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
337 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
338
339 if (ndev->dev_status <= AIE2_DEV_INIT) {
340 XDNA_ERR(xdna, "device is already stopped");
341 return;
342 }
343
344 aie2_mgmt_fw_fini(ndev);
345 xdna_mailbox_stop_channel(mailbox_chann: ndev->mgmt_chann);
346 xdna_mailbox_destroy_channel(mailbox_chann: ndev->mgmt_chann);
347 ndev->mgmt_chann = NULL;
348 drmm_kfree(dev: &xdna->ddev, data: ndev->mbox);
349 ndev->mbox = NULL;
350 aie2_psp_stop(psp: ndev->psp_hdl);
351 aie2_smu_fini(ndev);
352 aie2_error_async_events_free(ndev);
353 pci_disable_device(dev: pdev);
354
355 ndev->dev_status = AIE2_DEV_INIT;
356}
357
358static int aie2_hw_start(struct amdxdna_dev *xdna)
359{
360 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
361 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
362 struct xdna_mailbox_res mbox_res;
363 u32 xdna_mailbox_intr_reg;
364 int mgmt_mb_irq, ret;
365
366 if (ndev->dev_status >= AIE2_DEV_START) {
367 XDNA_INFO(xdna, "device is already started");
368 return 0;
369 }
370
371 ret = pci_enable_device(dev: pdev);
372 if (ret) {
373 XDNA_ERR(xdna, "failed to enable device, ret %d", ret);
374 return ret;
375 }
376 pci_set_master(dev: pdev);
377
378 ret = aie2_smu_init(ndev);
379 if (ret) {
380 XDNA_ERR(xdna, "failed to init smu, ret %d", ret);
381 goto disable_dev;
382 }
383
384 ret = aie2_psp_start(psp: ndev->psp_hdl);
385 if (ret) {
386 XDNA_ERR(xdna, "failed to start psp, ret %d", ret);
387 goto fini_smu;
388 }
389
390 ret = aie2_get_mgmt_chann_info(ndev);
391 if (ret) {
392 XDNA_ERR(xdna, "firmware is not alive");
393 goto stop_psp;
394 }
395
396 mbox_res.ringbuf_base = ndev->sram_base;
397 mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar);
398 mbox_res.mbox_base = ndev->mbox_base;
399 mbox_res.mbox_size = MBOX_SIZE(ndev);
400 mbox_res.name = "xdna_mailbox";
401 ndev->mbox = xdnam_mailbox_create(ddev: &xdna->ddev, res: &mbox_res);
402 if (!ndev->mbox) {
403 XDNA_ERR(xdna, "failed to create mailbox device");
404 ret = -ENODEV;
405 goto stop_psp;
406 }
407
408 mgmt_mb_irq = pci_irq_vector(dev: pdev, nr: ndev->mgmt_chan_idx);
409 if (mgmt_mb_irq < 0) {
410 ret = mgmt_mb_irq;
411 XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
412 goto stop_psp;
413 }
414
415 xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
416 ndev->mgmt_chann = xdna_mailbox_create_channel(mailbox: ndev->mbox,
417 x2i: &ndev->mgmt_x2i,
418 i2x: &ndev->mgmt_i2x,
419 xdna_mailbox_intr_reg,
420 mb_irq: mgmt_mb_irq);
421 if (!ndev->mgmt_chann) {
422 XDNA_ERR(xdna, "failed to create management mailbox channel");
423 ret = -EINVAL;
424 goto stop_psp;
425 }
426
427 ret = aie2_pm_init(ndev);
428 if (ret) {
429 XDNA_ERR(xdna, "failed to init pm, ret %d", ret);
430 goto destroy_mgmt_chann;
431 }
432
433 ret = aie2_mgmt_fw_init(ndev);
434 if (ret) {
435 XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret);
436 goto destroy_mgmt_chann;
437 }
438
439 ret = aie2_mgmt_fw_query(ndev);
440 if (ret) {
441 XDNA_ERR(xdna, "failed to query fw, ret %d", ret);
442 goto destroy_mgmt_chann;
443 }
444
445 ret = aie2_error_async_events_alloc(ndev);
446 if (ret) {
447 XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret);
448 goto destroy_mgmt_chann;
449 }
450
451 ndev->dev_status = AIE2_DEV_START;
452
453 return 0;
454
455destroy_mgmt_chann:
456 xdna_mailbox_stop_channel(mailbox_chann: ndev->mgmt_chann);
457 xdna_mailbox_destroy_channel(mailbox_chann: ndev->mgmt_chann);
458stop_psp:
459 aie2_psp_stop(psp: ndev->psp_hdl);
460fini_smu:
461 aie2_smu_fini(ndev);
462disable_dev:
463 pci_disable_device(dev: pdev);
464
465 return ret;
466}
467
468static int aie2_hw_suspend(struct amdxdna_dev *xdna)
469{
470 struct amdxdna_client *client;
471
472 guard(mutex)(T: &xdna->dev_lock);
473 list_for_each_entry(client, &xdna->client_list, node)
474 aie2_hwctx_suspend(client);
475
476 aie2_hw_stop(xdna);
477
478 return 0;
479}
480
481static int aie2_hw_resume(struct amdxdna_dev *xdna)
482{
483 struct amdxdna_client *client;
484 int ret;
485
486 ret = aie2_hw_start(xdna);
487 if (ret) {
488 XDNA_ERR(xdna, "Start hardware failed, %d", ret);
489 return ret;
490 }
491
492 list_for_each_entry(client, &xdna->client_list, node) {
493 ret = aie2_hwctx_resume(client);
494 if (ret)
495 break;
496 }
497
498 return ret;
499}
500
501static int aie2_init(struct amdxdna_dev *xdna)
502{
503 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
504 void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
505 struct init_config xrs_cfg = { 0 };
506 struct amdxdna_dev_hdl *ndev;
507 struct psp_config psp_conf;
508 const struct firmware *fw;
509 unsigned long bars = 0;
510 int i, nvec, ret;
511
512 if (!hypervisor_is_type(type: X86_HYPER_NATIVE)) {
513 XDNA_ERR(xdna, "Running under hypervisor not supported");
514 return -EINVAL;
515 }
516
517 ndev = drmm_kzalloc(dev: &xdna->ddev, size: sizeof(*ndev), GFP_KERNEL);
518 if (!ndev)
519 return -ENOMEM;
520
521 ndev->priv = xdna->dev_info->dev_priv;
522 ndev->xdna = xdna;
523
524 ret = request_firmware(fw: &fw, name: ndev->priv->fw_path, device: &pdev->dev);
525 if (ret) {
526 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
527 ndev->priv->fw_path, ret);
528 return ret;
529 }
530
531 ret = pcim_enable_device(pdev);
532 if (ret) {
533 XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
534 goto release_fw;
535 }
536
537 for (i = 0; i < PSP_MAX_REGS; i++)
538 set_bit(PSP_REG_BAR(ndev, i), addr: &bars);
539
540 set_bit(nr: xdna->dev_info->sram_bar, addr: &bars);
541 set_bit(nr: xdna->dev_info->smu_bar, addr: &bars);
542 set_bit(nr: xdna->dev_info->mbox_bar, addr: &bars);
543
544 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
545 if (!test_bit(i, &bars))
546 continue;
547 tbl[i] = pcim_iomap(pdev, bar: i, maxlen: 0);
548 if (!tbl[i]) {
549 XDNA_ERR(xdna, "map bar %d failed", i);
550 ret = -ENOMEM;
551 goto release_fw;
552 }
553 }
554
555 ndev->sram_base = tbl[xdna->dev_info->sram_bar];
556 ndev->smu_base = tbl[xdna->dev_info->smu_bar];
557 ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
558
559 ret = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
560 if (ret) {
561 XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret);
562 goto release_fw;
563 }
564
565 nvec = pci_msix_vec_count(dev: pdev);
566 if (nvec <= 0) {
567 XDNA_ERR(xdna, "does not get number of interrupt vector");
568 ret = -EINVAL;
569 goto release_fw;
570 }
571
572 ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: nvec, max_vecs: nvec, PCI_IRQ_MSIX);
573 if (ret < 0) {
574 XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret);
575 goto release_fw;
576 }
577
578 psp_conf.fw_size = fw->size;
579 psp_conf.fw_buf = fw->data;
580 for (i = 0; i < PSP_MAX_REGS; i++)
581 psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
582 ndev->psp_hdl = aie2m_psp_create(ddev: &xdna->ddev, conf: &psp_conf);
583 if (!ndev->psp_hdl) {
584 XDNA_ERR(xdna, "failed to create psp");
585 ret = -ENOMEM;
586 goto release_fw;
587 }
588 xdna->dev_handle = ndev;
589
590 ret = aie2_hw_start(xdna);
591 if (ret) {
592 XDNA_ERR(xdna, "start npu failed, ret %d", ret);
593 goto release_fw;
594 }
595
596 xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1;
597 for (i = 0; i < xrs_cfg.clk_list.num_levels; i++)
598 xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk;
599 xrs_cfg.sys_eff_factor = 1;
600 xrs_cfg.ddev = &xdna->ddev;
601 xrs_cfg.actions = &aie2_xrs_actions;
602 xrs_cfg.total_col = ndev->total_col;
603
604 xdna->xrs_hdl = xrsm_init(cfg: &xrs_cfg);
605 if (!xdna->xrs_hdl) {
606 XDNA_ERR(xdna, "Initialize resolver failed");
607 ret = -EINVAL;
608 goto stop_hw;
609 }
610
611 release_firmware(fw);
612 aie2_msg_init(ndev);
613 amdxdna_pm_init(xdna);
614 return 0;
615
616stop_hw:
617 aie2_hw_stop(xdna);
618release_fw:
619 release_firmware(fw);
620
621 return ret;
622}
623
624static void aie2_fini(struct amdxdna_dev *xdna)
625{
626 amdxdna_pm_fini(xdna);
627 aie2_hw_stop(xdna);
628}
629
630static int aie2_get_aie_status(struct amdxdna_client *client,
631 struct amdxdna_drm_get_info *args)
632{
633 struct amdxdna_drm_query_aie_status status;
634 struct amdxdna_dev *xdna = client->xdna;
635 struct amdxdna_dev_hdl *ndev;
636 int ret;
637
638 ndev = xdna->dev_handle;
639 if (copy_from_user(to: &status, u64_to_user_ptr(args->buffer), n: sizeof(status))) {
640 XDNA_ERR(xdna, "Failed to copy AIE request into kernel");
641 return -EFAULT;
642 }
643
644 if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) {
645 XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.",
646 status.buffer_size, ndev->metadata.cols * ndev->metadata.size);
647 return -EINVAL;
648 }
649
650 ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer),
651 size: status.buffer_size, cols_filled: &status.cols_filled);
652 if (ret) {
653 XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret);
654 return ret;
655 }
656
657 if (copy_to_user(u64_to_user_ptr(args->buffer), from: &status, n: sizeof(status))) {
658 XDNA_ERR(xdna, "Failed to copy AIE request info to user space");
659 return -EFAULT;
660 }
661
662 return 0;
663}
664
665static int aie2_get_aie_metadata(struct amdxdna_client *client,
666 struct amdxdna_drm_get_info *args)
667{
668 struct amdxdna_drm_query_aie_metadata *meta;
669 struct amdxdna_dev *xdna = client->xdna;
670 struct amdxdna_dev_hdl *ndev;
671 int ret = 0;
672
673 ndev = xdna->dev_handle;
674 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
675 if (!meta)
676 return -ENOMEM;
677
678 meta->col_size = ndev->metadata.size;
679 meta->cols = ndev->metadata.cols;
680 meta->rows = ndev->metadata.rows;
681
682 meta->version.major = ndev->metadata.version.major;
683 meta->version.minor = ndev->metadata.version.minor;
684
685 meta->core.row_count = ndev->metadata.core.row_count;
686 meta->core.row_start = ndev->metadata.core.row_start;
687 meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count;
688 meta->core.lock_count = ndev->metadata.core.lock_count;
689 meta->core.event_reg_count = ndev->metadata.core.event_reg_count;
690
691 meta->mem.row_count = ndev->metadata.mem.row_count;
692 meta->mem.row_start = ndev->metadata.mem.row_start;
693 meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count;
694 meta->mem.lock_count = ndev->metadata.mem.lock_count;
695 meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count;
696
697 meta->shim.row_count = ndev->metadata.shim.row_count;
698 meta->shim.row_start = ndev->metadata.shim.row_start;
699 meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count;
700 meta->shim.lock_count = ndev->metadata.shim.lock_count;
701 meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count;
702
703 if (copy_to_user(u64_to_user_ptr(args->buffer), from: meta, n: sizeof(*meta)))
704 ret = -EFAULT;
705
706 kfree(objp: meta);
707 return ret;
708}
709
710static int aie2_get_aie_version(struct amdxdna_client *client,
711 struct amdxdna_drm_get_info *args)
712{
713 struct amdxdna_drm_query_aie_version version;
714 struct amdxdna_dev *xdna = client->xdna;
715 struct amdxdna_dev_hdl *ndev;
716
717 ndev = xdna->dev_handle;
718 version.major = ndev->version.major;
719 version.minor = ndev->version.minor;
720
721 if (copy_to_user(u64_to_user_ptr(args->buffer), from: &version, n: sizeof(version)))
722 return -EFAULT;
723
724 return 0;
725}
726
727static int aie2_get_firmware_version(struct amdxdna_client *client,
728 struct amdxdna_drm_get_info *args)
729{
730 struct amdxdna_drm_query_firmware_version version;
731 struct amdxdna_dev *xdna = client->xdna;
732
733 version.major = xdna->fw_ver.major;
734 version.minor = xdna->fw_ver.minor;
735 version.patch = xdna->fw_ver.sub;
736 version.build = xdna->fw_ver.build;
737
738 if (copy_to_user(u64_to_user_ptr(args->buffer), from: &version, n: sizeof(version)))
739 return -EFAULT;
740
741 return 0;
742}
743
744static int aie2_get_power_mode(struct amdxdna_client *client,
745 struct amdxdna_drm_get_info *args)
746{
747 struct amdxdna_drm_get_power_mode mode = {};
748 struct amdxdna_dev *xdna = client->xdna;
749 struct amdxdna_dev_hdl *ndev;
750
751 ndev = xdna->dev_handle;
752 mode.power_mode = ndev->pw_mode;
753
754 if (copy_to_user(u64_to_user_ptr(args->buffer), from: &mode, n: sizeof(mode)))
755 return -EFAULT;
756
757 return 0;
758}
759
760static int aie2_get_clock_metadata(struct amdxdna_client *client,
761 struct amdxdna_drm_get_info *args)
762{
763 struct amdxdna_drm_query_clock_metadata *clock;
764 struct amdxdna_dev *xdna = client->xdna;
765 struct amdxdna_dev_hdl *ndev;
766 int ret = 0;
767
768 ndev = xdna->dev_handle;
769 clock = kzalloc(sizeof(*clock), GFP_KERNEL);
770 if (!clock)
771 return -ENOMEM;
772
773 snprintf(buf: clock->mp_npu_clock.name, size: sizeof(clock->mp_npu_clock.name),
774 fmt: "MP-NPU Clock");
775 clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq;
776 snprintf(buf: clock->h_clock.name, size: sizeof(clock->h_clock.name), fmt: "H Clock");
777 clock->h_clock.freq_mhz = ndev->hclk_freq;
778
779 if (copy_to_user(u64_to_user_ptr(args->buffer), from: clock, n: sizeof(*clock)))
780 ret = -EFAULT;
781
782 kfree(objp: clock);
783 return ret;
784}
785
786static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg)
787{
788 struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL;
789 struct amdxdna_drm_get_array *array_args = arg;
790 struct amdxdna_drm_hwctx_entry __user *buf;
791 u32 size;
792
793 if (!array_args->num_element)
794 return -EINVAL;
795
796 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
797 if (!tmp)
798 return -ENOMEM;
799
800 tmp->pid = hwctx->client->pid;
801 tmp->context_id = hwctx->id;
802 tmp->start_col = hwctx->start_col;
803 tmp->num_col = hwctx->num_col;
804 tmp->command_submissions = hwctx->priv->seq;
805 tmp->command_completions = hwctx->priv->completed;
806 tmp->pasid = hwctx->client->pasid;
807 tmp->priority = hwctx->qos.priority;
808 tmp->gops = hwctx->qos.gops;
809 tmp->fps = hwctx->qos.fps;
810 tmp->dma_bandwidth = hwctx->qos.dma_bandwidth;
811 tmp->latency = hwctx->qos.latency;
812 tmp->frame_exec_time = hwctx->qos.frame_exec_time;
813 tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE;
814
815 buf = u64_to_user_ptr(array_args->buffer);
816 size = min(sizeof(*tmp), array_args->element_size);
817
818 if (copy_to_user(to: buf, from: tmp, n: size))
819 return -EFAULT;
820
821 array_args->buffer += size;
822 array_args->num_element--;
823
824 return 0;
825}
826
827static int aie2_get_hwctx_status(struct amdxdna_client *client,
828 struct amdxdna_drm_get_info *args)
829{
830 struct amdxdna_drm_get_array array_args;
831 struct amdxdna_dev *xdna = client->xdna;
832 struct amdxdna_client *tmp_client;
833 int ret;
834
835 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
836
837 array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx);
838 array_args.buffer = args->buffer;
839 array_args.num_element = args->buffer_size / array_args.element_size;
840 list_for_each_entry(tmp_client, &xdna->client_list, node) {
841 ret = amdxdna_hwctx_walk(client: tmp_client, arg: &array_args,
842 walk: aie2_hwctx_status_cb);
843 if (ret)
844 break;
845 }
846
847 args->buffer_size -= (u32)(array_args.buffer - args->buffer);
848 return 0;
849}
850
851static int aie2_query_resource_info(struct amdxdna_client *client,
852 struct amdxdna_drm_get_info *args)
853{
854 struct amdxdna_drm_get_resource_info res_info;
855 const struct amdxdna_dev_priv *priv;
856 struct amdxdna_dev_hdl *ndev;
857 struct amdxdna_dev *xdna;
858
859 xdna = client->xdna;
860 ndev = xdna->dev_handle;
861 priv = ndev->priv;
862
863 res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk;
864 res_info.npu_tops_max = ndev->max_tops;
865 res_info.npu_task_max = priv->hwctx_limit;
866 res_info.npu_tops_curr = ndev->curr_tops;
867 res_info.npu_task_curr = ndev->hwctx_num;
868
869 if (copy_to_user(u64_to_user_ptr(args->buffer), from: &res_info, n: sizeof(res_info)))
870 return -EFAULT;
871
872 return 0;
873}
874
875static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg)
876{
877 struct amdxdna_dev *xdna = hwctx->client->xdna;
878 u32 *map = arg;
879
880 if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) {
881 XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id,
882 xdna->dev_handle->priv->hwctx_limit);
883 return -EINVAL;
884 }
885
886 map[hwctx->fw_ctx_id] = hwctx->id;
887 return 0;
888}
889
890static int aie2_get_telemetry(struct amdxdna_client *client,
891 struct amdxdna_drm_get_info *args)
892{
893 struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL;
894 u32 telemetry_data_sz, header_sz, elem_num;
895 struct amdxdna_dev *xdna = client->xdna;
896 struct amdxdna_client *tmp_client;
897 int ret;
898
899 elem_num = xdna->dev_handle->priv->hwctx_limit;
900 header_sz = struct_size(header, map, elem_num);
901 if (args->buffer_size <= header_sz) {
902 XDNA_ERR(xdna, "Invalid buffer size");
903 return -EINVAL;
904 }
905
906 telemetry_data_sz = args->buffer_size - header_sz;
907 if (telemetry_data_sz > SZ_4M) {
908 XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz);
909 return -EINVAL;
910 }
911
912 header = kzalloc(header_sz, GFP_KERNEL);
913 if (!header)
914 return -ENOMEM;
915
916 if (copy_from_user(to: header, u64_to_user_ptr(args->buffer), n: sizeof(*header))) {
917 XDNA_ERR(xdna, "Failed to copy telemetry header from user");
918 return -EFAULT;
919 }
920
921 header->map_num_elements = elem_num;
922 list_for_each_entry(tmp_client, &xdna->client_list, node) {
923 ret = amdxdna_hwctx_walk(client: tmp_client, arg: &header->map,
924 walk: aie2_fill_hwctx_map);
925 if (ret)
926 return ret;
927 }
928
929 ret = aie2_query_telemetry(ndev: xdna->dev_handle,
930 u64_to_user_ptr(args->buffer + header_sz),
931 size: telemetry_data_sz, header);
932 if (ret) {
933 XDNA_ERR(xdna, "Query telemetry failed ret %d", ret);
934 return ret;
935 }
936
937 if (copy_to_user(u64_to_user_ptr(args->buffer), from: header, n: header_sz)) {
938 XDNA_ERR(xdna, "Copy header failed");
939 return -EFAULT;
940 }
941
942 return 0;
943}
944
945static int aie2_get_preempt_state(struct amdxdna_client *client,
946 struct amdxdna_drm_get_info *args)
947{
948 struct amdxdna_drm_attribute_state state = {};
949 struct amdxdna_dev *xdna = client->xdna;
950 struct amdxdna_dev_hdl *ndev;
951
952 ndev = xdna->dev_handle;
953 if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE)
954 state.state = ndev->force_preempt_enabled;
955 else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE)
956 state.state = ndev->frame_boundary_preempt;
957
958 if (copy_to_user(u64_to_user_ptr(args->buffer), from: &state, n: sizeof(state)))
959 return -EFAULT;
960
961 return 0;
962}
963
964static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
965{
966 struct amdxdna_dev *xdna = client->xdna;
967 int ret, idx;
968
969 if (!drm_dev_enter(dev: &xdna->ddev, idx: &idx))
970 return -ENODEV;
971
972 ret = amdxdna_pm_resume_get(xdna);
973 if (ret)
974 goto dev_exit;
975
976 switch (args->param) {
977 case DRM_AMDXDNA_QUERY_AIE_STATUS:
978 ret = aie2_get_aie_status(client, args);
979 break;
980 case DRM_AMDXDNA_QUERY_AIE_METADATA:
981 ret = aie2_get_aie_metadata(client, args);
982 break;
983 case DRM_AMDXDNA_QUERY_AIE_VERSION:
984 ret = aie2_get_aie_version(client, args);
985 break;
986 case DRM_AMDXDNA_QUERY_CLOCK_METADATA:
987 ret = aie2_get_clock_metadata(client, args);
988 break;
989 case DRM_AMDXDNA_QUERY_HW_CONTEXTS:
990 ret = aie2_get_hwctx_status(client, args);
991 break;
992 case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION:
993 ret = aie2_get_firmware_version(client, args);
994 break;
995 case DRM_AMDXDNA_GET_POWER_MODE:
996 ret = aie2_get_power_mode(client, args);
997 break;
998 case DRM_AMDXDNA_QUERY_TELEMETRY:
999 ret = aie2_get_telemetry(client, args);
1000 break;
1001 case DRM_AMDXDNA_QUERY_RESOURCE_INFO:
1002 ret = aie2_query_resource_info(client, args);
1003 break;
1004 case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE:
1005 case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE:
1006 ret = aie2_get_preempt_state(client, args);
1007 break;
1008 default:
1009 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1010 ret = -EOPNOTSUPP;
1011 }
1012
1013 amdxdna_pm_suspend_put(xdna);
1014 XDNA_DBG(xdna, "Got param %d", args->param);
1015
1016dev_exit:
1017 drm_dev_exit(idx);
1018 return ret;
1019}
1020
1021static int aie2_query_ctx_status_array(struct amdxdna_client *client,
1022 struct amdxdna_drm_get_array *args)
1023{
1024 struct amdxdna_drm_get_array array_args;
1025 struct amdxdna_dev *xdna = client->xdna;
1026 struct amdxdna_client *tmp_client;
1027 int ret;
1028
1029 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
1030
1031 if (args->element_size > SZ_4K || args->num_element > SZ_1K) {
1032 XDNA_DBG(xdna, "Invalid element size %d or number of element %d",
1033 args->element_size, args->num_element);
1034 return -EINVAL;
1035 }
1036
1037 array_args.element_size = min(args->element_size,
1038 sizeof(struct amdxdna_drm_hwctx_entry));
1039 array_args.buffer = args->buffer;
1040 array_args.num_element = args->num_element * args->element_size /
1041 array_args.element_size;
1042 list_for_each_entry(tmp_client, &xdna->client_list, node) {
1043 ret = amdxdna_hwctx_walk(client: tmp_client, arg: &array_args,
1044 walk: aie2_hwctx_status_cb);
1045 if (ret)
1046 break;
1047 }
1048
1049 args->element_size = array_args.element_size;
1050 args->num_element = (u32)((array_args.buffer - args->buffer) /
1051 args->element_size);
1052
1053 return 0;
1054}
1055
1056static int aie2_get_array(struct amdxdna_client *client,
1057 struct amdxdna_drm_get_array *args)
1058{
1059 struct amdxdna_dev *xdna = client->xdna;
1060 int ret, idx;
1061
1062 if (!drm_dev_enter(dev: &xdna->ddev, idx: &idx))
1063 return -ENODEV;
1064
1065 ret = amdxdna_pm_resume_get(xdna);
1066 if (ret)
1067 goto dev_exit;
1068
1069 switch (args->param) {
1070 case DRM_AMDXDNA_HW_CONTEXT_ALL:
1071 ret = aie2_query_ctx_status_array(client, args);
1072 break;
1073 case DRM_AMDXDNA_HW_LAST_ASYNC_ERR:
1074 ret = aie2_get_array_async_error(ndev: xdna->dev_handle, args);
1075 break;
1076 default:
1077 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1078 ret = -EOPNOTSUPP;
1079 }
1080
1081 amdxdna_pm_suspend_put(xdna);
1082 XDNA_DBG(xdna, "Got param %d", args->param);
1083
1084dev_exit:
1085 drm_dev_exit(idx);
1086 return ret;
1087}
1088
1089static int aie2_set_power_mode(struct amdxdna_client *client,
1090 struct amdxdna_drm_set_state *args)
1091{
1092 struct amdxdna_drm_set_power_mode power_state;
1093 enum amdxdna_power_mode_type power_mode;
1094 struct amdxdna_dev *xdna = client->xdna;
1095
1096 if (copy_from_user(to: &power_state, u64_to_user_ptr(args->buffer),
1097 n: sizeof(power_state))) {
1098 XDNA_ERR(xdna, "Failed to copy power mode request into kernel");
1099 return -EFAULT;
1100 }
1101
1102 if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad)))
1103 return -EINVAL;
1104
1105 power_mode = power_state.power_mode;
1106 if (power_mode > POWER_MODE_TURBO) {
1107 XDNA_ERR(xdna, "Invalid power mode %d", power_mode);
1108 return -EINVAL;
1109 }
1110
1111 return aie2_pm_set_mode(ndev: xdna->dev_handle, target: power_mode);
1112}
1113
1114static int aie2_set_preempt_state(struct amdxdna_client *client,
1115 struct amdxdna_drm_set_state *args)
1116{
1117 struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle;
1118 struct amdxdna_drm_attribute_state state;
1119 u32 val;
1120 int ret;
1121
1122 if (copy_from_user(to: &state, u64_to_user_ptr(args->buffer), n: sizeof(state)))
1123 return -EFAULT;
1124
1125 if (state.state > 1)
1126 return -EINVAL;
1127
1128 if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad)))
1129 return -EINVAL;
1130
1131 if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) {
1132 ndev->force_preempt_enabled = state.state;
1133 } else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) {
1134 val = state.state;
1135 ret = aie2_runtime_cfg(ndev, category: AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT,
1136 val: &val);
1137 if (ret)
1138 return ret;
1139
1140 ndev->frame_boundary_preempt = state.state;
1141 }
1142
1143 return 0;
1144}
1145
1146static int aie2_set_state(struct amdxdna_client *client,
1147 struct amdxdna_drm_set_state *args)
1148{
1149 struct amdxdna_dev *xdna = client->xdna;
1150 int ret, idx;
1151
1152 if (!drm_dev_enter(dev: &xdna->ddev, idx: &idx))
1153 return -ENODEV;
1154
1155 ret = amdxdna_pm_resume_get(xdna);
1156 if (ret)
1157 goto dev_exit;
1158
1159 switch (args->param) {
1160 case DRM_AMDXDNA_SET_POWER_MODE:
1161 ret = aie2_set_power_mode(client, args);
1162 break;
1163 case DRM_AMDXDNA_SET_FORCE_PREEMPT:
1164 case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT:
1165 ret = aie2_set_preempt_state(client, args);
1166 break;
1167 default:
1168 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
1169 ret = -EOPNOTSUPP;
1170 break;
1171 }
1172
1173 amdxdna_pm_suspend_put(xdna);
1174dev_exit:
1175 drm_dev_exit(idx);
1176 return ret;
1177}
1178
1179const struct amdxdna_dev_ops aie2_ops = {
1180 .init = aie2_init,
1181 .fini = aie2_fini,
1182 .resume = aie2_hw_resume,
1183 .suspend = aie2_hw_suspend,
1184 .get_aie_info = aie2_get_info,
1185 .set_aie_state = aie2_set_state,
1186 .hwctx_init = aie2_hwctx_init,
1187 .hwctx_fini = aie2_hwctx_fini,
1188 .hwctx_config = aie2_hwctx_config,
1189 .hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo,
1190 .cmd_submit = aie2_cmd_submit,
1191 .hmm_invalidate = aie2_hmm_invalidate,
1192 .get_array = aie2_get_array,
1193};
1194

source code of linux/drivers/accel/amdxdna/aie2_pci.c