1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright 2016-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/printk.h>
26#include <linux/slab.h>
27#include <linux/uaccess.h>
28#include "kfd_priv.h"
29#include "kfd_mqd_manager.h"
30#include "v9_structs.h"
31#include "gc/gc_9_0_offset.h"
32#include "gc/gc_9_0_sh_mask.h"
33#include "sdma0/sdma0_4_0_sh_mask.h"
34#include "amdgpu_amdkfd.h"
35#include "kfd_device_queue_manager.h"
36
37static void update_mqd(struct mqd_manager *mm, void *mqd,
38 struct queue_properties *q,
39 struct mqd_update_info *minfo);
40
41static uint64_t mqd_stride_v9(struct mqd_manager *mm,
42 struct queue_properties *q)
43{
44 if (mm->dev->kfd->cwsr_enabled &&
45 q->type == KFD_QUEUE_TYPE_COMPUTE)
46 return ALIGN(q->ctl_stack_size, PAGE_SIZE) +
47 ALIGN(sizeof(struct v9_mqd), PAGE_SIZE);
48
49 return mm->mqd_size;
50}
51
52static inline struct v9_mqd *get_mqd(void *mqd)
53{
54 return (struct v9_mqd *)mqd;
55}
56
57static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
58{
59 return (struct v9_sdma_mqd *)mqd;
60}
61
62static void update_cu_mask(struct mqd_manager *mm, void *mqd,
63 struct mqd_update_info *minfo, uint32_t inst)
64{
65 struct v9_mqd *m;
66 uint32_t se_mask[KFD_MAX_NUM_SE] = {0};
67
68 if (!minfo || !minfo->cu_mask.ptr)
69 return;
70
71 mqd_symmetrically_map_cu_mask(mm,
72 cu_mask: minfo->cu_mask.ptr, cu_mask_count: minfo->cu_mask.count, se_mask, inst);
73
74 m = get_mqd(mqd);
75
76 m->compute_static_thread_mgmt_se0 = se_mask[0];
77 m->compute_static_thread_mgmt_se1 = se_mask[1];
78 m->compute_static_thread_mgmt_se2 = se_mask[2];
79 m->compute_static_thread_mgmt_se3 = se_mask[3];
80 if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3)) {
81 m->compute_static_thread_mgmt_se4 = se_mask[4];
82 m->compute_static_thread_mgmt_se5 = se_mask[5];
83 m->compute_static_thread_mgmt_se6 = se_mask[6];
84 m->compute_static_thread_mgmt_se7 = se_mask[7];
85
86 pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n",
87 m->compute_static_thread_mgmt_se0,
88 m->compute_static_thread_mgmt_se1,
89 m->compute_static_thread_mgmt_se2,
90 m->compute_static_thread_mgmt_se3,
91 m->compute_static_thread_mgmt_se4,
92 m->compute_static_thread_mgmt_se5,
93 m->compute_static_thread_mgmt_se6,
94 m->compute_static_thread_mgmt_se7);
95 } else {
96 pr_debug("inst: %u, update cu mask to %#x %#x %#x %#x\n",
97 inst, m->compute_static_thread_mgmt_se0,
98 m->compute_static_thread_mgmt_se1,
99 m->compute_static_thread_mgmt_se2,
100 m->compute_static_thread_mgmt_se3);
101 }
102}
103
104static void set_priority(struct v9_mqd *m, struct queue_properties *q)
105{
106 m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
107 m->cp_hqd_queue_priority = q->priority;
108}
109
110static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,
111 struct queue_properties *q)
112{
113 int retval;
114 struct kfd_mem_obj *mqd_mem_obj = NULL;
115
116 /* For V9 only, due to a HW bug, the control stack of a user mode
117 * compute queue needs to be allocated just behind the page boundary
118 * of its regular MQD buffer. So we allocate an enlarged MQD buffer:
119 * the first page of the buffer serves as the regular MQD buffer
120 * purpose and the remaining is for control stack. Although the two
121 * parts are in the same buffer object, they need different memory
122 * types: MQD part needs UC (uncached) as usual, while control stack
123 * needs NC (non coherent), which is different from the UC type which
124 * is used when control stack is allocated in user space.
125 *
126 * Because of all those, we use the gtt allocation function instead
127 * of sub-allocation function for this enlarged MQD buffer. Moreover,
128 * in order to achieve two memory types in a single buffer object, we
129 * pass a special bo flag AMDGPU_GEM_CREATE_CP_MQD_GFX9 to instruct
130 * amdgpu memory functions to do so.
131 */
132 if (node->kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
133 mqd_mem_obj = kzalloc(size: sizeof(struct kfd_mem_obj), GFP_KERNEL);
134 if (!mqd_mem_obj)
135 return NULL;
136 retval = amdgpu_amdkfd_alloc_gtt_mem(adev: node->adev,
137 size: (ALIGN(q->ctl_stack_size, PAGE_SIZE) +
138 ALIGN(sizeof(struct v9_mqd), PAGE_SIZE)) *
139 NUM_XCC(node->xcc_mask),
140 mem_obj: &(mqd_mem_obj->gtt_mem),
141 gpu_addr: &(mqd_mem_obj->gpu_addr),
142 cpu_ptr: (void *)&(mqd_mem_obj->cpu_ptr), mqd_gfx9: true);
143
144 if (retval) {
145 kfree(objp: mqd_mem_obj);
146 return NULL;
147 }
148 } else {
149 retval = kfd_gtt_sa_allocate(node, size: sizeof(struct v9_mqd),
150 mem_obj: &mqd_mem_obj);
151 if (retval)
152 return NULL;
153 }
154
155 return mqd_mem_obj;
156}
157
158static void init_mqd(struct mqd_manager *mm, void **mqd,
159 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
160 struct queue_properties *q)
161{
162 uint64_t addr;
163 struct v9_mqd *m;
164
165 m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
166 addr = mqd_mem_obj->gpu_addr;
167
168 memset(m, 0, sizeof(struct v9_mqd));
169
170 m->header = 0xC0310800;
171 m->compute_pipelinestat_enable = 1;
172 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
173 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
174 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
175 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
176 m->compute_static_thread_mgmt_se4 = 0xFFFFFFFF;
177 m->compute_static_thread_mgmt_se5 = 0xFFFFFFFF;
178 m->compute_static_thread_mgmt_se6 = 0xFFFFFFFF;
179 m->compute_static_thread_mgmt_se7 = 0xFFFFFFFF;
180
181 m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
182 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
183
184 m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;
185
186 m->cp_mqd_base_addr_lo = lower_32_bits(addr);
187 m->cp_mqd_base_addr_hi = upper_32_bits(addr);
188
189 m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
190 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
191 1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
192
193 /* Set cp_hqd_hq_scheduler0 bit 14 to 1 to have the CP set up the
194 * DISPATCH_PTR. This is required for the kfd debugger
195 */
196 m->cp_hqd_hq_status0 = 1 << 14;
197
198 if (q->format == KFD_QUEUE_FORMAT_AQL)
199 m->cp_hqd_aql_control =
200 1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
201
202 if (q->tba_addr) {
203 m->compute_pgm_rsrc2 |=
204 (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
205 }
206
207 if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
208 m->cp_hqd_persistent_state |=
209 (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
210 m->cp_hqd_ctx_save_base_addr_lo =
211 lower_32_bits(q->ctx_save_restore_area_address);
212 m->cp_hqd_ctx_save_base_addr_hi =
213 upper_32_bits(q->ctx_save_restore_area_address);
214 m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
215 m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
216 m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
217 m->cp_hqd_wg_state_offset = q->ctl_stack_size;
218 }
219
220 *mqd = m;
221 if (gart_addr)
222 *gart_addr = addr;
223 update_mqd(mm, mqd: m, q, NULL);
224}
225
226static int load_mqd(struct mqd_manager *mm, void *mqd,
227 uint32_t pipe_id, uint32_t queue_id,
228 struct queue_properties *p, struct mm_struct *mms)
229{
230 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
231 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
232
233 return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
234 (uint32_t __user *)p->write_ptr,
235 wptr_shift, 0, mms, 0);
236}
237
238static void update_mqd(struct mqd_manager *mm, void *mqd,
239 struct queue_properties *q,
240 struct mqd_update_info *minfo)
241{
242 struct v9_mqd *m;
243
244 m = get_mqd(mqd);
245
246 m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;
247 m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
248 pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
249
250 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
251 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
252
253 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
254 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
255 m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
256 m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
257
258 m->cp_hqd_pq_doorbell_control =
259 q->doorbell_off <<
260 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
261 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
262 m->cp_hqd_pq_doorbell_control);
263
264 m->cp_hqd_ib_control =
265 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
266 1 << CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT;
267
268 /*
269 * HW does not clamp this field correctly. Maximum EOP queue size
270 * is constrained by per-SE EOP done signal count, which is 8-bit.
271 * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
272 * more than (EOP entry count - 1) so a queue size of 0x800 dwords
273 * is safe, giving a maximum field value of 0xA.
274 *
275 * Also, do calculation only if EOP is used (size > 0), otherwise
276 * the order_base_2 calculation provides incorrect result.
277 *
278 */
279 m->cp_hqd_eop_control = q->eop_ring_buffer_size ?
280 min(0xA, order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0;
281
282 m->cp_hqd_eop_base_addr_lo =
283 lower_32_bits(q->eop_ring_buffer_address >> 8);
284 m->cp_hqd_eop_base_addr_hi =
285 upper_32_bits(q->eop_ring_buffer_address >> 8);
286
287 m->cp_hqd_iq_timer = 0;
288
289 m->cp_hqd_vmid = q->vmid;
290
291 if (q->format == KFD_QUEUE_FORMAT_AQL) {
292 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
293 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
294 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT |
295 1 << CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT;
296 m->cp_hqd_pq_doorbell_control |= 1 <<
297 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
298 }
299 if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
300 m->cp_hqd_ctx_save_control = 0;
301
302 if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3))
303 update_cu_mask(mm, mqd, minfo, inst: 0);
304 set_priority(m, q);
305
306 if (minfo && KFD_GC_VERSION(mm->dev) >= IP_VERSION(9, 4, 2)) {
307 if (minfo->update_flag & UPDATE_FLAG_IS_GWS)
308 m->compute_resource_limits |=
309 COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK;
310 else
311 m->compute_resource_limits &=
312 ~COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK;
313 }
314
315 q->is_active = QUEUE_IS_ACTIVE(*q);
316}
317
318
319static uint32_t read_doorbell_id(void *mqd)
320{
321 struct v9_mqd *m = (struct v9_mqd *)mqd;
322
323 return m->queue_doorbell_id0;
324}
325
326static int get_wave_state(struct mqd_manager *mm, void *mqd,
327 struct queue_properties *q,
328 void __user *ctl_stack,
329 u32 *ctl_stack_used_size,
330 u32 *save_area_used_size)
331{
332 struct v9_mqd *m;
333 struct kfd_context_save_area_header header;
334
335 /* Control stack is located one page after MQD. */
336 void *mqd_ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
337
338 m = get_mqd(mqd);
339
340 *ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
341 m->cp_hqd_cntl_stack_offset;
342 *save_area_used_size = m->cp_hqd_wg_state_offset -
343 m->cp_hqd_cntl_stack_size;
344
345 header.wave_state.control_stack_size = *ctl_stack_used_size;
346 header.wave_state.wave_state_size = *save_area_used_size;
347
348 header.wave_state.wave_state_offset = m->cp_hqd_wg_state_offset;
349 header.wave_state.control_stack_offset = m->cp_hqd_cntl_stack_offset;
350
351 if (copy_to_user(to: ctl_stack, from: &header, n: sizeof(header.wave_state)))
352 return -EFAULT;
353
354 if (copy_to_user(to: ctl_stack + m->cp_hqd_cntl_stack_offset,
355 from: mqd_ctl_stack + m->cp_hqd_cntl_stack_offset,
356 n: *ctl_stack_used_size))
357 return -EFAULT;
358
359 return 0;
360}
361
362static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
363{
364 struct v9_mqd *m = get_mqd(mqd);
365
366 *ctl_stack_size = m->cp_hqd_cntl_stack_size;
367}
368
369static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
370{
371 struct v9_mqd *m;
372 /* Control stack is located one page after MQD. */
373 void *ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
374
375 m = get_mqd(mqd);
376
377 memcpy(mqd_dst, m, sizeof(struct v9_mqd));
378 memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size);
379}
380
381static void restore_mqd(struct mqd_manager *mm, void **mqd,
382 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
383 struct queue_properties *qp,
384 const void *mqd_src,
385 const void *ctl_stack_src, u32 ctl_stack_size)
386{
387 uint64_t addr;
388 struct v9_mqd *m;
389 void *ctl_stack;
390
391 m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
392 addr = mqd_mem_obj->gpu_addr;
393
394 memcpy(m, mqd_src, sizeof(*m));
395
396 *mqd = m;
397 if (gart_addr)
398 *gart_addr = addr;
399
400 /* Control stack is located one page after MQD. */
401 ctl_stack = (void *)((uintptr_t)*mqd + PAGE_SIZE);
402 memcpy(ctl_stack, ctl_stack_src, ctl_stack_size);
403
404 m->cp_hqd_pq_doorbell_control =
405 qp->doorbell_off <<
406 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
407 pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
408 m->cp_hqd_pq_doorbell_control);
409
410 qp->is_active = 0;
411}
412
413static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
414 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
415 struct queue_properties *q)
416{
417 struct v9_mqd *m;
418
419 init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
420
421 m = get_mqd(mqd: *mqd);
422
423 m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
424 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
425}
426
427static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,
428 enum kfd_preempt_type type, unsigned int timeout,
429 uint32_t pipe_id, uint32_t queue_id)
430{
431 int err;
432 struct v9_mqd *m;
433 u32 doorbell_off;
434
435 m = get_mqd(mqd);
436
437 doorbell_off = m->cp_hqd_pq_doorbell_control >>
438 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
439 err = amdgpu_amdkfd_unmap_hiq(adev: mm->dev->adev, doorbell_off, inst: 0);
440 if (err)
441 pr_debug("Destroy HIQ MQD failed: %d\n", err);
442
443 return err;
444}
445
446static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
447 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
448 struct queue_properties *q)
449{
450 struct v9_sdma_mqd *m;
451
452 m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
453
454 memset(m, 0, sizeof(struct v9_sdma_mqd));
455
456 *mqd = m;
457 if (gart_addr)
458 *gart_addr = mqd_mem_obj->gpu_addr;
459
460 mm->update_mqd(mm, m, q, NULL);
461}
462
463#define SDMA_RLC_DUMMY_DEFAULT 0xf
464
465static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
466 struct queue_properties *q,
467 struct mqd_update_info *minfo)
468{
469 struct v9_sdma_mqd *m;
470
471 m = get_sdma_mqd(mqd);
472 m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4)
473 << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
474 q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
475 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
476 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
477
478 m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
479 m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
480 m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
481 m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
482 m->sdmax_rlcx_doorbell_offset =
483 q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
484
485 m->sdma_engine_id = q->sdma_engine_id;
486 m->sdma_queue_id = q->sdma_queue_id;
487 m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
488
489 q->is_active = QUEUE_IS_ACTIVE(*q);
490}
491
492static void checkpoint_mqd_sdma(struct mqd_manager *mm,
493 void *mqd,
494 void *mqd_dst,
495 void *ctl_stack_dst)
496{
497 struct v9_sdma_mqd *m;
498
499 m = get_sdma_mqd(mqd);
500
501 memcpy(mqd_dst, m, sizeof(struct v9_sdma_mqd));
502}
503
504static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
505 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
506 struct queue_properties *qp,
507 const void *mqd_src,
508 const void *ctl_stack_src, const u32 ctl_stack_size)
509{
510 uint64_t addr;
511 struct v9_sdma_mqd *m;
512
513 m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
514 addr = mqd_mem_obj->gpu_addr;
515
516 memcpy(m, mqd_src, sizeof(*m));
517
518 m->sdmax_rlcx_doorbell_offset =
519 qp->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
520
521 *mqd = m;
522 if (gart_addr)
523 *gart_addr = addr;
524
525 qp->is_active = 0;
526}
527
528static void init_mqd_hiq_v9_4_3(struct mqd_manager *mm, void **mqd,
529 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
530 struct queue_properties *q)
531{
532 struct v9_mqd *m;
533 int xcc = 0;
534 struct kfd_mem_obj xcc_mqd_mem_obj;
535 uint64_t xcc_gart_addr = 0;
536
537 memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj));
538
539 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
540 kfd_get_hiq_xcc_mqd(dev: mm->dev, mqd_mem_obj: &xcc_mqd_mem_obj, virtual_xcc_id: xcc);
541
542 init_mqd(mm, mqd: (void **)&m, mqd_mem_obj: &xcc_mqd_mem_obj, gart_addr: &xcc_gart_addr, q);
543
544 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
545 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
546 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
547 m->cp_mqd_stride_size = kfd_hiq_mqd_stride(dev: mm->dev);
548 if (xcc == 0) {
549 /* Set no_update_rptr = 0 in Master XCC */
550 m->cp_hqd_pq_control &= ~CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
551
552 /* Set the MQD pointer and gart address to XCC0 MQD */
553 *mqd = m;
554 *gart_addr = xcc_gart_addr;
555 }
556 }
557}
558
559static int hiq_load_mqd_kiq_v9_4_3(struct mqd_manager *mm, void *mqd,
560 uint32_t pipe_id, uint32_t queue_id,
561 struct queue_properties *p, struct mm_struct *mms)
562{
563 uint32_t xcc_mask = mm->dev->xcc_mask;
564 int xcc_id, err, inst = 0;
565 void *xcc_mqd;
566 uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(dev: mm->dev);
567
568 for_each_inst(xcc_id, xcc_mask) {
569 xcc_mqd = mqd + hiq_mqd_size * inst;
570 err = mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, xcc_mqd,
571 pipe_id, queue_id,
572 p->doorbell_off, xcc_id);
573 if (err) {
574 pr_debug("Failed to load HIQ MQD for XCC: %d\n", inst);
575 break;
576 }
577 ++inst;
578 }
579
580 return err;
581}
582
583static int destroy_hiq_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
584 enum kfd_preempt_type type, unsigned int timeout,
585 uint32_t pipe_id, uint32_t queue_id)
586{
587 uint32_t xcc_mask = mm->dev->xcc_mask;
588 int xcc_id, err, inst = 0;
589 uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(dev: mm->dev);
590 struct v9_mqd *m;
591 u32 doorbell_off;
592
593 for_each_inst(xcc_id, xcc_mask) {
594 m = get_mqd(mqd: mqd + hiq_mqd_size * inst);
595
596 doorbell_off = m->cp_hqd_pq_doorbell_control >>
597 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
598
599 err = amdgpu_amdkfd_unmap_hiq(adev: mm->dev->adev, doorbell_off, inst: xcc_id);
600 if (err) {
601 pr_debug("Destroy HIQ MQD failed for xcc: %d\n", inst);
602 break;
603 }
604 ++inst;
605 }
606
607 return err;
608}
609
610static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj,
611 struct kfd_mem_obj *xcc_mqd_mem_obj,
612 uint64_t offset)
613{
614 xcc_mqd_mem_obj->gtt_mem = (offset == 0) ?
615 mqd_mem_obj->gtt_mem : NULL;
616 xcc_mqd_mem_obj->gpu_addr = mqd_mem_obj->gpu_addr + offset;
617 xcc_mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t)mqd_mem_obj->cpu_ptr
618 + offset);
619}
620
621static void init_mqd_v9_4_3(struct mqd_manager *mm, void **mqd,
622 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
623 struct queue_properties *q)
624{
625 struct v9_mqd *m;
626 int xcc = 0;
627 struct kfd_mem_obj xcc_mqd_mem_obj;
628 uint64_t xcc_gart_addr = 0;
629 uint64_t xcc_ctx_save_restore_area_address;
630 uint64_t offset = mm->mqd_stride(mm, q);
631 uint32_t local_xcc_start = mm->dev->dqm->current_logical_xcc_start++;
632
633 memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj));
634 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
635 get_xcc_mqd(mqd_mem_obj, xcc_mqd_mem_obj: &xcc_mqd_mem_obj, offset: offset*xcc);
636
637 init_mqd(mm, mqd: (void **)&m, mqd_mem_obj: &xcc_mqd_mem_obj, gart_addr: &xcc_gart_addr, q);
638
639 m->cp_mqd_stride_size = offset;
640
641 /*
642 * Update the CWSR address for each XCC if CWSR is enabled
643 * and CWSR area is allocated in thunk
644 */
645 if (mm->dev->kfd->cwsr_enabled &&
646 q->ctx_save_restore_area_address) {
647 xcc_ctx_save_restore_area_address =
648 q->ctx_save_restore_area_address +
649 (xcc * q->ctx_save_restore_area_size);
650
651 m->cp_hqd_ctx_save_base_addr_lo =
652 lower_32_bits(xcc_ctx_save_restore_area_address);
653 m->cp_hqd_ctx_save_base_addr_hi =
654 upper_32_bits(xcc_ctx_save_restore_area_address);
655 }
656
657 if (q->format == KFD_QUEUE_FORMAT_AQL) {
658 m->compute_tg_chunk_size = 1;
659 m->compute_current_logic_xcc_id =
660 (local_xcc_start + xcc) %
661 NUM_XCC(mm->dev->xcc_mask);
662
663 switch (xcc) {
664 case 0:
665 /* Master XCC */
666 m->cp_hqd_pq_control &=
667 ~CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
668 break;
669 default:
670 break;
671 }
672 } else {
673 /* PM4 Queue */
674 m->compute_current_logic_xcc_id = 0;
675 m->compute_tg_chunk_size = 0;
676 m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
677 }
678
679 if (xcc == 0) {
680 /* Set the MQD pointer and gart address to XCC0 MQD */
681 *mqd = m;
682 *gart_addr = xcc_gart_addr;
683 }
684 }
685}
686
687static void update_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
688 struct queue_properties *q, struct mqd_update_info *minfo)
689{
690 struct v9_mqd *m;
691 int xcc = 0;
692 uint64_t size = mm->mqd_stride(mm, q);
693
694 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
695 m = get_mqd(mqd: mqd + size * xcc);
696 update_mqd(mm, mqd: m, q, minfo);
697
698 update_cu_mask(mm, mqd, minfo, inst: xcc);
699
700 if (q->format == KFD_QUEUE_FORMAT_AQL) {
701 switch (xcc) {
702 case 0:
703 /* Master XCC */
704 m->cp_hqd_pq_control &=
705 ~CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
706 break;
707 default:
708 break;
709 }
710 m->compute_tg_chunk_size = 1;
711 } else {
712 /* PM4 Queue */
713 m->compute_current_logic_xcc_id = 0;
714 m->compute_tg_chunk_size = 0;
715 m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
716 }
717 }
718}
719
720static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
721 enum kfd_preempt_type type, unsigned int timeout,
722 uint32_t pipe_id, uint32_t queue_id)
723{
724 uint32_t xcc_mask = mm->dev->xcc_mask;
725 int xcc_id, err, inst = 0;
726 void *xcc_mqd;
727 struct v9_mqd *m;
728 uint64_t mqd_offset;
729
730 m = get_mqd(mqd);
731 mqd_offset = m->cp_mqd_stride_size;
732
733 for_each_inst(xcc_id, xcc_mask) {
734 xcc_mqd = mqd + mqd_offset * inst;
735 err = mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, xcc_mqd,
736 type, timeout, pipe_id,
737 queue_id, xcc_id);
738 if (err) {
739 pr_debug("Destroy MQD failed for xcc: %d\n", inst);
740 break;
741 }
742 ++inst;
743 }
744
745 return err;
746}
747
748static int load_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
749 uint32_t pipe_id, uint32_t queue_id,
750 struct queue_properties *p, struct mm_struct *mms)
751{
752 /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
753 uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
754 uint32_t xcc_mask = mm->dev->xcc_mask;
755 int xcc_id, err, inst = 0;
756 void *xcc_mqd;
757 uint64_t mqd_stride_size = mm->mqd_stride(mm, p);
758
759 for_each_inst(xcc_id, xcc_mask) {
760 xcc_mqd = mqd + mqd_stride_size * inst;
761 err = mm->dev->kfd2kgd->hqd_load(
762 mm->dev->adev, xcc_mqd, pipe_id, queue_id,
763 (uint32_t __user *)p->write_ptr, wptr_shift, 0, mms,
764 xcc_id);
765 if (err) {
766 pr_debug("Load MQD failed for xcc: %d\n", inst);
767 break;
768 }
769 ++inst;
770 }
771
772 return err;
773}
774
775static int get_wave_state_v9_4_3(struct mqd_manager *mm, void *mqd,
776 struct queue_properties *q,
777 void __user *ctl_stack,
778 u32 *ctl_stack_used_size,
779 u32 *save_area_used_size)
780{
781 int xcc, err = 0;
782 void *xcc_mqd;
783 void __user *xcc_ctl_stack;
784 uint64_t mqd_stride_size = mm->mqd_stride(mm, q);
785 u32 tmp_ctl_stack_used_size = 0, tmp_save_area_used_size = 0;
786
787 for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
788 xcc_mqd = mqd + mqd_stride_size * xcc;
789 xcc_ctl_stack = (void __user *)((uintptr_t)ctl_stack +
790 q->ctx_save_restore_area_size * xcc);
791
792 err = get_wave_state(mm, mqd: xcc_mqd, q, ctl_stack: xcc_ctl_stack,
793 ctl_stack_used_size: &tmp_ctl_stack_used_size,
794 save_area_used_size: &tmp_save_area_used_size);
795 if (err)
796 break;
797
798 /*
799 * Set the ctl_stack_used_size and save_area_used_size to
800 * ctl_stack_used_size and save_area_used_size of XCC 0 when
801 * passing the info the user-space.
802 * For multi XCC, user-space would have to look at the header
803 * info of each Control stack area to determine the control
804 * stack size and save area used.
805 */
806 if (xcc == 0) {
807 *ctl_stack_used_size = tmp_ctl_stack_used_size;
808 *save_area_used_size = tmp_save_area_used_size;
809 }
810 }
811
812 return err;
813}
814
815#if defined(CONFIG_DEBUG_FS)
816
817static int debugfs_show_mqd(struct seq_file *m, void *data)
818{
819 seq_hex_dump(m, prefix_str: " ", prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4,
820 buf: data, len: sizeof(struct v9_mqd), ascii: false);
821 return 0;
822}
823
824static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
825{
826 seq_hex_dump(m, prefix_str: " ", prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4,
827 buf: data, len: sizeof(struct v9_sdma_mqd), ascii: false);
828 return 0;
829}
830
831#endif
832
833struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
834 struct kfd_node *dev)
835{
836 struct mqd_manager *mqd;
837
838 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
839 return NULL;
840
841 mqd = kzalloc(size: sizeof(*mqd), GFP_KERNEL);
842 if (!mqd)
843 return NULL;
844
845 mqd->dev = dev;
846
847 switch (type) {
848 case KFD_MQD_TYPE_CP:
849 mqd->allocate_mqd = allocate_mqd;
850 mqd->free_mqd = kfd_free_mqd_cp;
851 mqd->is_occupied = kfd_is_occupied_cp;
852 mqd->get_checkpoint_info = get_checkpoint_info;
853 mqd->checkpoint_mqd = checkpoint_mqd;
854 mqd->restore_mqd = restore_mqd;
855 mqd->mqd_size = sizeof(struct v9_mqd);
856 mqd->mqd_stride = mqd_stride_v9;
857#if defined(CONFIG_DEBUG_FS)
858 mqd->debugfs_show_mqd = debugfs_show_mqd;
859#endif
860 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) {
861 mqd->init_mqd = init_mqd_v9_4_3;
862 mqd->load_mqd = load_mqd_v9_4_3;
863 mqd->update_mqd = update_mqd_v9_4_3;
864 mqd->destroy_mqd = destroy_mqd_v9_4_3;
865 mqd->get_wave_state = get_wave_state_v9_4_3;
866 } else {
867 mqd->init_mqd = init_mqd;
868 mqd->load_mqd = load_mqd;
869 mqd->update_mqd = update_mqd;
870 mqd->destroy_mqd = kfd_destroy_mqd_cp;
871 mqd->get_wave_state = get_wave_state;
872 }
873 break;
874 case KFD_MQD_TYPE_HIQ:
875 mqd->allocate_mqd = allocate_hiq_mqd;
876 mqd->free_mqd = free_mqd_hiq_sdma;
877 mqd->update_mqd = update_mqd;
878 mqd->is_occupied = kfd_is_occupied_cp;
879 mqd->mqd_size = sizeof(struct v9_mqd);
880 mqd->mqd_stride = kfd_mqd_stride;
881#if defined(CONFIG_DEBUG_FS)
882 mqd->debugfs_show_mqd = debugfs_show_mqd;
883#endif
884 mqd->read_doorbell_id = read_doorbell_id;
885 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) {
886 mqd->init_mqd = init_mqd_hiq_v9_4_3;
887 mqd->load_mqd = hiq_load_mqd_kiq_v9_4_3;
888 mqd->destroy_mqd = destroy_hiq_mqd_v9_4_3;
889 } else {
890 mqd->init_mqd = init_mqd_hiq;
891 mqd->load_mqd = kfd_hiq_load_mqd_kiq;
892 mqd->destroy_mqd = destroy_hiq_mqd;
893 }
894 break;
895 case KFD_MQD_TYPE_DIQ:
896 mqd->allocate_mqd = allocate_mqd;
897 mqd->init_mqd = init_mqd_hiq;
898 mqd->free_mqd = kfd_free_mqd_cp;
899 mqd->load_mqd = load_mqd;
900 mqd->update_mqd = update_mqd;
901 mqd->destroy_mqd = kfd_destroy_mqd_cp;
902 mqd->is_occupied = kfd_is_occupied_cp;
903 mqd->mqd_size = sizeof(struct v9_mqd);
904#if defined(CONFIG_DEBUG_FS)
905 mqd->debugfs_show_mqd = debugfs_show_mqd;
906#endif
907 break;
908 case KFD_MQD_TYPE_SDMA:
909 mqd->allocate_mqd = allocate_sdma_mqd;
910 mqd->init_mqd = init_mqd_sdma;
911 mqd->free_mqd = free_mqd_hiq_sdma;
912 mqd->load_mqd = kfd_load_mqd_sdma;
913 mqd->update_mqd = update_mqd_sdma;
914 mqd->destroy_mqd = kfd_destroy_mqd_sdma;
915 mqd->is_occupied = kfd_is_occupied_sdma;
916 mqd->checkpoint_mqd = checkpoint_mqd_sdma;
917 mqd->restore_mqd = restore_mqd_sdma;
918 mqd->mqd_size = sizeof(struct v9_sdma_mqd);
919 mqd->mqd_stride = kfd_mqd_stride;
920#if defined(CONFIG_DEBUG_FS)
921 mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
922#endif
923 break;
924 default:
925 kfree(objp: mqd);
926 return NULL;
927 }
928
929 return mqd;
930}
931

source code of linux/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c