1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/ratelimit.h>
26#include <linux/printk.h>
27#include <linux/slab.h>
28#include <linux/list.h>
29#include <linux/types.h>
30#include <linux/bitops.h>
31#include <linux/sched.h>
32#include "kfd_priv.h"
33#include "kfd_device_queue_manager.h"
34#include "kfd_mqd_manager.h"
35#include "cik_regs.h"
36#include "kfd_kernel_queue.h"
37#include "amdgpu_amdkfd.h"
38#include "amdgpu_reset.h"
39#include "amdgpu_sdma.h"
40#include "mes_v11_api_def.h"
41#include "kfd_debug.h"
42
43/* Size of the per-pipe EOP queue */
44#define CIK_HPD_EOP_BYTES_LOG2 11
45#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
46/* See unmap_queues_cpsch() */
47#define USE_DEFAULT_GRACE_PERIOD 0xffffffff
48
49static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
50 u32 pasid, unsigned int vmid);
51
52static int execute_queues_cpsch(struct device_queue_manager *dqm,
53 enum kfd_unmap_queues_filter filter,
54 uint32_t filter_param,
55 uint32_t grace_period);
56static int unmap_queues_cpsch(struct device_queue_manager *dqm,
57 enum kfd_unmap_queues_filter filter,
58 uint32_t filter_param,
59 uint32_t grace_period,
60 bool reset);
61
62static int map_queues_cpsch(struct device_queue_manager *dqm);
63
64static void deallocate_sdma_queue(struct device_queue_manager *dqm,
65 struct queue *q);
66
67static inline void deallocate_hqd(struct device_queue_manager *dqm,
68 struct queue *q);
69static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
70static int allocate_sdma_queue(struct device_queue_manager *dqm,
71 struct queue *q, const uint32_t *restore_sdma_id);
72
73static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
74
75static inline
76enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
77{
78 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
79 return KFD_MQD_TYPE_SDMA;
80 return KFD_MQD_TYPE_CP;
81}
82
83static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
84{
85 int i;
86 int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
87 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
88
89 /* queue is available for KFD usage if bit is 1 */
90 for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
91 if (test_bit(pipe_offset + i,
92 dqm->dev->kfd->shared_resources.cp_queue_bitmap))
93 return true;
94 return false;
95}
96
97unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
98{
99 return bitmap_weight(src: dqm->dev->kfd->shared_resources.cp_queue_bitmap,
100 AMDGPU_MAX_QUEUES);
101}
102
103unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
104{
105 return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
106}
107
108unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
109{
110 return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
111}
112
113static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
114{
115 return kfd_get_num_sdma_engines(kdev: dqm->dev) +
116 kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
117}
118
119unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
120{
121 return kfd_get_num_sdma_engines(kdev: dqm->dev) *
122 dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
123}
124
125unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
126{
127 return kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev) *
128 dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
129}
130
131static void init_sdma_bitmaps(struct device_queue_manager *dqm)
132{
133 bitmap_zero(dst: dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
134 bitmap_set(map: dqm->sdma_bitmap, start: 0, nbits: get_num_sdma_queues(dqm));
135
136 bitmap_zero(dst: dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
137 bitmap_set(map: dqm->xgmi_sdma_bitmap, start: 0, nbits: get_num_xgmi_sdma_queues(dqm));
138
139 /* Mask out the reserved queues */
140 bitmap_andnot(dst: dqm->sdma_bitmap, src1: dqm->sdma_bitmap,
141 src2: dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
142 KFD_MAX_SDMA_QUEUES);
143}
144
145void program_sh_mem_settings(struct device_queue_manager *dqm,
146 struct qcm_process_device *qpd)
147{
148 uint32_t xcc_mask = dqm->dev->xcc_mask;
149 int xcc_id;
150
151 for_each_inst(xcc_id, xcc_mask)
152 dqm->dev->kfd2kgd->program_sh_mem_settings(
153 dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,
154 qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,
155 qpd->sh_mem_bases, xcc_id);
156}
157
158static void kfd_hws_hang(struct device_queue_manager *dqm)
159{
160 struct device_process_node *cur;
161 struct qcm_process_device *qpd;
162 struct queue *q;
163
164 /* Mark all device queues as reset. */
165 list_for_each_entry(cur, &dqm->queues, list) {
166 qpd = cur->qpd;
167 list_for_each_entry(q, &qpd->queues_list, list) {
168 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
169
170 pdd->has_reset_queue = true;
171 }
172 }
173
174 /*
175 * Issue a GPU reset if HWS is unresponsive
176 */
177 amdgpu_amdkfd_gpu_reset(adev: dqm->dev->adev);
178}
179
180static int convert_to_mes_queue_type(int queue_type)
181{
182 int mes_queue_type;
183
184 switch (queue_type) {
185 case KFD_QUEUE_TYPE_COMPUTE:
186 mes_queue_type = MES_QUEUE_TYPE_COMPUTE;
187 break;
188 case KFD_QUEUE_TYPE_SDMA:
189 mes_queue_type = MES_QUEUE_TYPE_SDMA;
190 break;
191 default:
192 WARN(1, "Invalid queue type %d", queue_type);
193 mes_queue_type = -EINVAL;
194 break;
195 }
196
197 return mes_queue_type;
198}
199
200static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
201 struct qcm_process_device *qpd)
202{
203 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
204 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
205 struct mes_add_queue_input queue_input;
206 int r, queue_type;
207 uint64_t wptr_addr_off;
208
209 if (!dqm->sched_running || dqm->sched_halt)
210 return 0;
211 if (!down_read_trylock(sem: &adev->reset_domain->sem))
212 return -EIO;
213
214 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
215 queue_input.process_id = pdd->pasid;
216 queue_input.page_table_base_addr = qpd->page_table_base;
217 queue_input.process_va_start = 0;
218 queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
219 /* MES unit for quantum is 100ns */
220 queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */
221 queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;
222 queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */
223 queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
224 queue_input.inprocess_gang_priority = q->properties.priority;
225 queue_input.gang_global_priority_level =
226 AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
227 queue_input.doorbell_offset = q->properties.doorbell_off;
228 queue_input.mqd_addr = q->gart_mqd_addr;
229 queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;
230
231 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);
232 queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(bo: q->properties.wptr_bo) + wptr_addr_off;
233
234 queue_input.is_kfd_process = 1;
235 queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
236 queue_input.queue_size = q->properties.queue_size >> 2;
237
238 queue_input.paging = false;
239 queue_input.tba_addr = qpd->tba_addr;
240 queue_input.tma_addr = qpd->tma_addr;
241 queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(dev: q->device);
242 queue_input.skip_process_ctx_clear =
243 qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&
244 (qpd->pqm->process->debug_trap_enabled ||
245 kfd_dbg_has_ttmps_always_setup(dev: q->device));
246
247 queue_type = convert_to_mes_queue_type(queue_type: q->properties.type);
248 if (queue_type < 0) {
249 dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",
250 q->properties.type);
251 up_read(sem: &adev->reset_domain->sem);
252 return -EINVAL;
253 }
254 queue_input.queue_type = (uint32_t)queue_type;
255
256 queue_input.exclusively_scheduled = q->properties.is_gws;
257
258 amdgpu_mes_lock(mes: &adev->mes);
259 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
260 amdgpu_mes_unlock(mes: &adev->mes);
261 up_read(sem: &adev->reset_domain->sem);
262 if (r) {
263 dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",
264 q->properties.doorbell_off);
265 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
266 kfd_hws_hang(dqm);
267 }
268
269 return r;
270}
271
272static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
273 struct qcm_process_device *qpd)
274{
275 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
276 int r;
277 struct mes_remove_queue_input queue_input;
278
279 if (!dqm->sched_running || dqm->sched_halt)
280 return 0;
281 if (!down_read_trylock(sem: &adev->reset_domain->sem))
282 return -EIO;
283
284 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
285 queue_input.doorbell_offset = q->properties.doorbell_off;
286 queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
287
288 amdgpu_mes_lock(mes: &adev->mes);
289 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
290 amdgpu_mes_unlock(mes: &adev->mes);
291 up_read(sem: &adev->reset_domain->sem);
292
293 if (r) {
294 dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
295 q->properties.doorbell_off);
296 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
297 kfd_hws_hang(dqm);
298 }
299
300 return r;
301}
302
303static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
304{
305 struct device_process_node *cur;
306 struct device *dev = dqm->dev->adev->dev;
307 struct qcm_process_device *qpd;
308 struct queue *q;
309 int retval = 0;
310
311 list_for_each_entry(cur, &dqm->queues, list) {
312 qpd = cur->qpd;
313 list_for_each_entry(q, &qpd->queues_list, list) {
314 if (q->properties.is_active) {
315 retval = remove_queue_mes(dqm, q, qpd);
316 if (retval) {
317 dev_err(dev, "%s: Failed to remove queue %d for dev %d",
318 __func__,
319 q->properties.queue_id,
320 dqm->dev->id);
321 return retval;
322 }
323 }
324 }
325 }
326
327 return retval;
328}
329
330static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
331{
332 struct device_process_node *cur;
333 struct device *dev = dqm->dev->adev->dev;
334 struct qcm_process_device *qpd;
335 struct queue *q;
336 int retval = 0;
337
338 list_for_each_entry(cur, &dqm->queues, list) {
339 qpd = cur->qpd;
340 list_for_each_entry(q, &qpd->queues_list, list) {
341 if (!q->properties.is_active)
342 continue;
343 retval = add_queue_mes(dqm, q, qpd);
344 if (retval) {
345 dev_err(dev, "%s: Failed to add queue %d for dev %d",
346 __func__,
347 q->properties.queue_id,
348 dqm->dev->id);
349 return retval;
350 }
351 }
352 }
353
354 return retval;
355}
356
357static int suspend_all_queues_mes(struct device_queue_manager *dqm)
358{
359 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
360 int r = 0;
361
362 if (!down_read_trylock(sem: &adev->reset_domain->sem))
363 return -EIO;
364
365 r = amdgpu_mes_suspend(adev);
366 up_read(sem: &adev->reset_domain->sem);
367
368 if (r) {
369 dev_err(adev->dev, "failed to suspend gangs from MES\n");
370 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
371 kfd_hws_hang(dqm);
372 }
373
374 return r;
375}
376
377static int resume_all_queues_mes(struct device_queue_manager *dqm)
378{
379 struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
380 int r = 0;
381
382 if (!down_read_trylock(sem: &adev->reset_domain->sem))
383 return -EIO;
384
385 r = amdgpu_mes_resume(adev);
386 up_read(sem: &adev->reset_domain->sem);
387
388 if (r) {
389 dev_err(adev->dev, "failed to resume gangs from MES\n");
390 dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
391 kfd_hws_hang(dqm);
392 }
393
394 return r;
395}
396
397static void increment_queue_count(struct device_queue_manager *dqm,
398 struct qcm_process_device *qpd,
399 struct queue *q)
400{
401 dqm->active_queue_count++;
402 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
403 q->properties.type == KFD_QUEUE_TYPE_DIQ)
404 dqm->active_cp_queue_count++;
405
406 if (q->properties.is_gws) {
407 dqm->gws_queue_count++;
408 qpd->mapped_gws_queue = true;
409 }
410}
411
412static void decrement_queue_count(struct device_queue_manager *dqm,
413 struct qcm_process_device *qpd,
414 struct queue *q)
415{
416 dqm->active_queue_count--;
417 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
418 q->properties.type == KFD_QUEUE_TYPE_DIQ)
419 dqm->active_cp_queue_count--;
420
421 if (q->properties.is_gws) {
422 dqm->gws_queue_count--;
423 qpd->mapped_gws_queue = false;
424 }
425}
426
427/*
428 * Allocate a doorbell ID to this queue.
429 * If doorbell_id is passed in, make sure requested ID is valid then allocate it.
430 */
431static int allocate_doorbell(struct qcm_process_device *qpd,
432 struct queue *q,
433 uint32_t const *restore_id)
434{
435 struct kfd_node *dev = qpd->dqm->dev;
436
437 if (!KFD_IS_SOC15(dev)) {
438 /* On pre-SOC15 chips we need to use the queue ID to
439 * preserve the user mode ABI.
440 */
441
442 if (restore_id && *restore_id != q->properties.queue_id)
443 return -EINVAL;
444
445 q->doorbell_id = q->properties.queue_id;
446 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
447 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
448 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
449 * doorbell assignments based on the engine and queue id.
450 * The doobell index distance between RLC (2*i) and (2*i+1)
451 * for a SDMA engine is 512.
452 */
453
454 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
455
456 /*
457 * q->properties.sdma_engine_id corresponds to the virtual
458 * sdma engine number. However, for doorbell allocation,
459 * we need the physical sdma engine id in order to get the
460 * correct doorbell offset.
461 */
462 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id *
463 get_num_all_sdma_engines(dqm: qpd->dqm) +
464 q->properties.sdma_engine_id]
465 + (q->properties.sdma_queue_id & 1)
466 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
467 + (q->properties.sdma_queue_id >> 1);
468
469 if (restore_id && *restore_id != valid_id)
470 return -EINVAL;
471 q->doorbell_id = valid_id;
472 } else {
473 /* For CP queues on SOC15 */
474 if (restore_id) {
475 /* make sure that ID is free */
476 if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
477 return -EINVAL;
478
479 q->doorbell_id = *restore_id;
480 } else {
481 /* or reserve a free doorbell ID */
482 unsigned int found;
483
484 found = find_first_zero_bit(addr: qpd->doorbell_bitmap,
485 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
486 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
487 pr_debug("No doorbells available");
488 return -EBUSY;
489 }
490 set_bit(nr: found, addr: qpd->doorbell_bitmap);
491 q->doorbell_id = found;
492 }
493 }
494
495 q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(adev: dev->adev,
496 db_bo: qpd->proc_doorbells,
497 doorbell_index: q->doorbell_id,
498 db_size: dev->kfd->device_info.doorbell_size);
499 return 0;
500}
501
502static void deallocate_doorbell(struct qcm_process_device *qpd,
503 struct queue *q)
504{
505 unsigned int old;
506 struct kfd_node *dev = qpd->dqm->dev;
507
508 if (!KFD_IS_SOC15(dev) ||
509 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
510 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
511 return;
512
513 old = test_and_clear_bit(nr: q->doorbell_id, addr: qpd->doorbell_bitmap);
514 WARN_ON(!old);
515}
516
517static void program_trap_handler_settings(struct device_queue_manager *dqm,
518 struct qcm_process_device *qpd)
519{
520 uint32_t xcc_mask = dqm->dev->xcc_mask;
521 int xcc_id;
522
523 if (dqm->dev->kfd2kgd->program_trap_handler_settings)
524 for_each_inst(xcc_id, xcc_mask)
525 dqm->dev->kfd2kgd->program_trap_handler_settings(
526 dqm->dev->adev, qpd->vmid, qpd->tba_addr,
527 qpd->tma_addr, xcc_id);
528}
529
530static int allocate_vmid(struct device_queue_manager *dqm,
531 struct qcm_process_device *qpd,
532 struct queue *q)
533{
534 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
535 struct device *dev = dqm->dev->adev->dev;
536 int allocated_vmid = -1, i;
537
538 for (i = dqm->dev->vm_info.first_vmid_kfd;
539 i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
540 if (!dqm->vmid_pasid[i]) {
541 allocated_vmid = i;
542 break;
543 }
544 }
545
546 if (allocated_vmid < 0) {
547 dev_err(dev, "no more vmid to allocate\n");
548 return -ENOSPC;
549 }
550
551 pr_debug("vmid allocated: %d\n", allocated_vmid);
552
553 dqm->vmid_pasid[allocated_vmid] = pdd->pasid;
554
555 set_pasid_vmid_mapping(dqm, pasid: pdd->pasid, vmid: allocated_vmid);
556
557 qpd->vmid = allocated_vmid;
558 q->properties.vmid = allocated_vmid;
559
560 program_sh_mem_settings(dqm, qpd);
561
562 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
563 program_trap_handler_settings(dqm, qpd);
564
565 /* qpd->page_table_base is set earlier when register_process()
566 * is called, i.e. when the first queue is created.
567 */
568 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
569 qpd->vmid,
570 qpd->page_table_base);
571 /* invalidate the VM context after pasid and vmid mapping is set up */
572 kfd_flush_tlb(qpd_to_pdd(qpd), type: TLB_FLUSH_LEGACY);
573
574 if (dqm->dev->kfd2kgd->set_scratch_backing_va)
575 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
576 qpd->sh_hidden_private_base, qpd->vmid);
577
578 return 0;
579}
580
581static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
582 struct qcm_process_device *qpd)
583{
584 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
585 int ret;
586
587 if (!qpd->ib_kaddr)
588 return -ENOMEM;
589
590 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
591 if (ret)
592 return ret;
593
594 return amdgpu_amdkfd_submit_ib(adev: kdev->adev, engine: KGD_ENGINE_MEC1, vmid: qpd->vmid,
595 gpu_addr: qpd->ib_base, ib_cmd: (uint32_t *)qpd->ib_kaddr,
596 ib_len: pmf->release_mem_size / sizeof(uint32_t));
597}
598
599static void deallocate_vmid(struct device_queue_manager *dqm,
600 struct qcm_process_device *qpd,
601 struct queue *q)
602{
603 struct device *dev = dqm->dev->adev->dev;
604
605 /* On GFX v7, CP doesn't flush TC at dequeue */
606 if (q->device->adev->asic_type == CHIP_HAWAII)
607 if (flush_texture_cache_nocpsch(kdev: q->device, qpd))
608 dev_err(dev, "Failed to flush TC\n");
609
610 kfd_flush_tlb(qpd_to_pdd(qpd), type: TLB_FLUSH_LEGACY);
611
612 /* Release the vmid mapping */
613 set_pasid_vmid_mapping(dqm, pasid: 0, vmid: qpd->vmid);
614 dqm->vmid_pasid[qpd->vmid] = 0;
615
616 qpd->vmid = 0;
617 q->properties.vmid = 0;
618}
619
620static int create_queue_nocpsch(struct device_queue_manager *dqm,
621 struct queue *q,
622 struct qcm_process_device *qpd,
623 const struct kfd_criu_queue_priv_data *qd,
624 const void *restore_mqd, const void *restore_ctl_stack)
625{
626 struct mqd_manager *mqd_mgr;
627 int retval;
628
629 dqm_lock(dqm);
630
631 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
632 pr_warn("Can't create new usermode queue because %d queues were already created\n",
633 dqm->total_queue_count);
634 retval = -EPERM;
635 goto out_unlock;
636 }
637
638 if (list_empty(head: &qpd->queues_list)) {
639 retval = allocate_vmid(dqm, qpd, q);
640 if (retval)
641 goto out_unlock;
642 }
643 q->properties.vmid = qpd->vmid;
644 /*
645 * Eviction state logic: mark all queues as evicted, even ones
646 * not currently active. Restoring inactive queues later only
647 * updates the is_evicted flag but is a no-op otherwise.
648 */
649 q->properties.is_evicted = !!qpd->evicted;
650
651 q->properties.tba_addr = qpd->tba_addr;
652 q->properties.tma_addr = qpd->tma_addr;
653
654 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
655 type: q->properties.type)];
656 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
657 retval = allocate_hqd(dqm, q);
658 if (retval)
659 goto deallocate_vmid;
660 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
661 q->pipe, q->queue);
662 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
663 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
664 retval = allocate_sdma_queue(dqm, q, restore_sdma_id: qd ? &qd->sdma_id : NULL);
665 if (retval)
666 goto deallocate_vmid;
667 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
668 }
669
670 retval = allocate_doorbell(qpd, q, restore_id: qd ? &qd->doorbell_id : NULL);
671 if (retval)
672 goto out_deallocate_hqd;
673
674 /* Temporarily release dqm lock to avoid a circular lock dependency */
675 dqm_unlock(dqm);
676 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
677 dqm_lock(dqm);
678
679 if (!q->mqd_mem_obj) {
680 retval = -ENOMEM;
681 goto out_deallocate_doorbell;
682 }
683
684 if (qd)
685 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
686 &q->properties, restore_mqd, restore_ctl_stack,
687 qd->ctl_stack_size);
688 else
689 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
690 &q->gart_mqd_addr, &q->properties);
691
692 if (q->properties.is_active) {
693 if (!dqm->sched_running) {
694 WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
695 goto add_queue_to_list;
696 }
697
698 if (WARN(q->process->mm != current->mm,
699 "should only run in user thread"))
700 retval = -EFAULT;
701 else
702 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
703 q->queue, &q->properties, current->mm);
704 if (retval)
705 goto out_free_mqd;
706 }
707
708add_queue_to_list:
709 list_add(new: &q->list, head: &qpd->queues_list);
710 qpd->queue_count++;
711 if (q->properties.is_active)
712 increment_queue_count(dqm, qpd, q);
713
714 /*
715 * Unconditionally increment this counter, regardless of the queue's
716 * type or whether the queue is active.
717 */
718 dqm->total_queue_count++;
719 pr_debug("Total of %d queues are accountable so far\n",
720 dqm->total_queue_count);
721 goto out_unlock;
722
723out_free_mqd:
724 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
725out_deallocate_doorbell:
726 deallocate_doorbell(qpd, q);
727out_deallocate_hqd:
728 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
729 deallocate_hqd(dqm, q);
730 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
731 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
732 deallocate_sdma_queue(dqm, q);
733deallocate_vmid:
734 if (list_empty(head: &qpd->queues_list))
735 deallocate_vmid(dqm, qpd, q);
736out_unlock:
737 dqm_unlock(dqm);
738 return retval;
739}
740
741static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
742{
743 bool set;
744 int pipe, bit, i;
745
746 set = false;
747
748 for (pipe = dqm->next_pipe_to_allocate, i = 0;
749 i < get_pipes_per_mec(dqm);
750 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
751
752 if (!is_pipe_enabled(dqm, mec: 0, pipe))
753 continue;
754
755 if (dqm->allocated_queues[pipe] != 0) {
756 bit = ffs(dqm->allocated_queues[pipe]) - 1;
757 dqm->allocated_queues[pipe] &= ~(1 << bit);
758 q->pipe = pipe;
759 q->queue = bit;
760 set = true;
761 break;
762 }
763 }
764
765 if (!set)
766 return -EBUSY;
767
768 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
769 /* horizontal hqd allocation */
770 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
771
772 return 0;
773}
774
775static inline void deallocate_hqd(struct device_queue_manager *dqm,
776 struct queue *q)
777{
778 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
779}
780
781#define SQ_IND_CMD_CMD_KILL 0x00000003
782#define SQ_IND_CMD_MODE_BROADCAST 0x00000001
783
784static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
785{
786 int status = 0;
787 unsigned int vmid;
788 uint16_t queried_pasid;
789 union SQ_CMD_BITS reg_sq_cmd;
790 union GRBM_GFX_INDEX_BITS reg_gfx_index;
791 struct kfd_process_device *pdd;
792 int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
793 int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
794 uint32_t xcc_mask = dev->xcc_mask;
795 int xcc_id;
796
797 reg_sq_cmd.u32All = 0;
798 reg_gfx_index.u32All = 0;
799
800 pr_debug("Killing all process wavefronts\n");
801
802 if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {
803 dev_err(dev->adev->dev, "no vmid pasid mapping supported\n");
804 return -EOPNOTSUPP;
805 }
806
807 /* taking the VMID for that process on the safe way using PDD */
808 pdd = kfd_get_process_device_data(dev, p);
809 if (!pdd)
810 return -EFAULT;
811
812 /* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
813 * ATC_VMID15_PASID_MAPPING
814 * to check which VMID the current process is mapped to.
815 */
816
817 for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
818 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
819 (dev->adev, vmid, &queried_pasid);
820
821 if (status && queried_pasid == pdd->pasid) {
822 pr_debug("Killing wave fronts of vmid %d and process pid %d\n",
823 vmid, p->lead_thread->pid);
824 break;
825 }
826 }
827
828 if (vmid > last_vmid_to_scan) {
829 dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n",
830 p->lead_thread->pid);
831 return -EFAULT;
832 }
833
834 reg_gfx_index.bits.sh_broadcast_writes = 1;
835 reg_gfx_index.bits.se_broadcast_writes = 1;
836 reg_gfx_index.bits.instance_broadcast_writes = 1;
837 reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
838 reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
839 reg_sq_cmd.bits.vm_id = vmid;
840
841 for_each_inst(xcc_id, xcc_mask)
842 dev->kfd2kgd->wave_control_execute(
843 dev->adev, reg_gfx_index.u32All,
844 reg_sq_cmd.u32All, xcc_id);
845
846 return 0;
847}
848
849/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
850 * to avoid asynchronized access
851 */
852static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
853 struct qcm_process_device *qpd,
854 struct queue *q)
855{
856 int retval;
857 struct mqd_manager *mqd_mgr;
858
859 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
860 type: q->properties.type)];
861
862 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
863 deallocate_hqd(dqm, q);
864 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
865 deallocate_sdma_queue(dqm, q);
866 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
867 deallocate_sdma_queue(dqm, q);
868 else {
869 pr_debug("q->properties.type %d is invalid\n",
870 q->properties.type);
871 return -EINVAL;
872 }
873 dqm->total_queue_count--;
874
875 deallocate_doorbell(qpd, q);
876
877 if (!dqm->sched_running) {
878 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
879 return 0;
880 }
881
882 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
883 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
884 KFD_UNMAP_LATENCY_MS,
885 q->pipe, q->queue);
886 if (retval == -ETIME)
887 qpd->reset_wavefronts = true;
888
889 list_del(entry: &q->list);
890 if (list_empty(head: &qpd->queues_list)) {
891 if (qpd->reset_wavefronts) {
892 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
893 dqm->dev);
894 /* dbgdev_wave_reset_wavefronts has to be called before
895 * deallocate_vmid(), i.e. when vmid is still in use.
896 */
897 dbgdev_wave_reset_wavefronts(dev: dqm->dev,
898 p: qpd->pqm->process);
899 qpd->reset_wavefronts = false;
900 }
901
902 deallocate_vmid(dqm, qpd, q);
903 }
904 qpd->queue_count--;
905 if (q->properties.is_active)
906 decrement_queue_count(dqm, qpd, q);
907
908 return retval;
909}
910
911static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
912 struct qcm_process_device *qpd,
913 struct queue *q)
914{
915 int retval;
916 uint64_t sdma_val = 0;
917 struct device *dev = dqm->dev->adev->dev;
918 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
919 struct mqd_manager *mqd_mgr =
920 dqm->mqd_mgrs[get_mqd_type_from_queue_type(type: q->properties.type)];
921
922 /* Get the SDMA queue stats */
923 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
924 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
925 retval = read_sdma_queue_counter(q_rptr: (uint64_t __user *)q->properties.read_ptr,
926 val: &sdma_val);
927 if (retval)
928 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
929 q->properties.queue_id);
930 }
931
932 dqm_lock(dqm);
933 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
934 if (!retval)
935 pdd->sdma_past_activity_counter += sdma_val;
936 dqm_unlock(dqm);
937
938 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
939
940 return retval;
941}
942
943static int update_queue(struct device_queue_manager *dqm, struct queue *q,
944 struct mqd_update_info *minfo)
945{
946 int retval = 0;
947 struct device *dev = dqm->dev->adev->dev;
948 struct mqd_manager *mqd_mgr;
949 struct kfd_process_device *pdd;
950 bool prev_active = false;
951
952 dqm_lock(dqm);
953 pdd = kfd_get_process_device_data(dev: q->device, p: q->process);
954 if (!pdd) {
955 retval = -ENODEV;
956 goto out_unlock;
957 }
958 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
959 type: q->properties.type)];
960
961 /* Save previous activity state for counters */
962 prev_active = q->properties.is_active;
963
964 /* Make sure the queue is unmapped before updating the MQD */
965 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
966 if (!dqm->dev->kfd->shared_resources.enable_mes)
967 retval = unmap_queues_cpsch(dqm,
968 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD, reset: false);
969 else if (prev_active)
970 retval = remove_queue_mes(dqm, q, qpd: &pdd->qpd);
971
972 /* queue is reset so inaccessable */
973 if (pdd->has_reset_queue) {
974 retval = -EACCES;
975 goto out_unlock;
976 }
977
978 if (retval) {
979 dev_err(dev, "unmap queue failed\n");
980 goto out_unlock;
981 }
982 } else if (prev_active &&
983 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
984 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
985 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
986
987 if (!dqm->sched_running) {
988 WARN_ONCE(1, "Update non-HWS queue while stopped\n");
989 goto out_unlock;
990 }
991
992 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
993 (dqm->dev->kfd->cwsr_enabled ?
994 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
995 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
996 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
997 if (retval) {
998 dev_err(dev, "destroy mqd failed\n");
999 goto out_unlock;
1000 }
1001 }
1002
1003 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
1004
1005 /*
1006 * check active state vs. the previous state and modify
1007 * counter accordingly. map_queues_cpsch uses the
1008 * dqm->active_queue_count to determine whether a new runlist must be
1009 * uploaded.
1010 */
1011 if (q->properties.is_active && !prev_active) {
1012 increment_queue_count(dqm, qpd: &pdd->qpd, q);
1013 } else if (!q->properties.is_active && prev_active) {
1014 decrement_queue_count(dqm, qpd: &pdd->qpd, q);
1015 } else if (q->gws && !q->properties.is_gws) {
1016 if (q->properties.is_active) {
1017 dqm->gws_queue_count++;
1018 pdd->qpd.mapped_gws_queue = true;
1019 }
1020 q->properties.is_gws = true;
1021 } else if (!q->gws && q->properties.is_gws) {
1022 if (q->properties.is_active) {
1023 dqm->gws_queue_count--;
1024 pdd->qpd.mapped_gws_queue = false;
1025 }
1026 q->properties.is_gws = false;
1027 }
1028
1029 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
1030 if (!dqm->dev->kfd->shared_resources.enable_mes)
1031 retval = map_queues_cpsch(dqm);
1032 else if (q->properties.is_active)
1033 retval = add_queue_mes(dqm, q, qpd: &pdd->qpd);
1034 } else if (q->properties.is_active &&
1035 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1036 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1037 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1038 if (WARN(q->process->mm != current->mm,
1039 "should only run in user thread"))
1040 retval = -EFAULT;
1041 else
1042 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
1043 q->pipe, q->queue,
1044 &q->properties, current->mm);
1045 }
1046
1047out_unlock:
1048 dqm_unlock(dqm);
1049 return retval;
1050}
1051
1052/* suspend_single_queue does not lock the dqm like the
1053 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should
1054 * lock the dqm before calling, and unlock after calling.
1055 *
1056 * The reason we don't lock the dqm is because this function may be
1057 * called on multiple queues in a loop, so rather than locking/unlocking
1058 * multiple times, we will just keep the dqm locked for all of the calls.
1059 */
1060static int suspend_single_queue(struct device_queue_manager *dqm,
1061 struct kfd_process_device *pdd,
1062 struct queue *q)
1063{
1064 bool is_new;
1065
1066 if (q->properties.is_suspended)
1067 return 0;
1068
1069 pr_debug("Suspending process pid %d queue [%i]\n",
1070 pdd->process->lead_thread->pid,
1071 q->properties.queue_id);
1072
1073 is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);
1074
1075 if (is_new || q->properties.is_being_destroyed) {
1076 pr_debug("Suspend: skip %s queue id %i\n",
1077 is_new ? "new" : "destroyed",
1078 q->properties.queue_id);
1079 return -EBUSY;
1080 }
1081
1082 q->properties.is_suspended = true;
1083 if (q->properties.is_active) {
1084 if (dqm->dev->kfd->shared_resources.enable_mes) {
1085 int r = remove_queue_mes(dqm, q, qpd: &pdd->qpd);
1086
1087 if (r)
1088 return r;
1089 }
1090
1091 decrement_queue_count(dqm, qpd: &pdd->qpd, q);
1092 q->properties.is_active = false;
1093 }
1094
1095 return 0;
1096}
1097
1098/* resume_single_queue does not lock the dqm like the functions
1099 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should
1100 * lock the dqm before calling, and unlock after calling.
1101 *
1102 * The reason we don't lock the dqm is because this function may be
1103 * called on multiple queues in a loop, so rather than locking/unlocking
1104 * multiple times, we will just keep the dqm locked for all of the calls.
1105 */
1106static int resume_single_queue(struct device_queue_manager *dqm,
1107 struct qcm_process_device *qpd,
1108 struct queue *q)
1109{
1110 struct kfd_process_device *pdd;
1111
1112 if (!q->properties.is_suspended)
1113 return 0;
1114
1115 pdd = qpd_to_pdd(qpd);
1116
1117 pr_debug("Restoring from suspend process pid %d queue [%i]\n",
1118 pdd->process->lead_thread->pid,
1119 q->properties.queue_id);
1120
1121 q->properties.is_suspended = false;
1122
1123 if (QUEUE_IS_ACTIVE(q->properties)) {
1124 if (dqm->dev->kfd->shared_resources.enable_mes) {
1125 int r = add_queue_mes(dqm, q, qpd: &pdd->qpd);
1126
1127 if (r)
1128 return r;
1129 }
1130
1131 q->properties.is_active = true;
1132 increment_queue_count(dqm, qpd, q);
1133 }
1134
1135 return 0;
1136}
1137
1138static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
1139 struct qcm_process_device *qpd)
1140{
1141 struct queue *q;
1142 struct mqd_manager *mqd_mgr;
1143 struct kfd_process_device *pdd;
1144 int retval, ret = 0;
1145
1146 dqm_lock(dqm);
1147 if (qpd->evicted++ > 0) /* already evicted, do nothing */
1148 goto out;
1149
1150 pdd = qpd_to_pdd(qpd);
1151 pr_debug_ratelimited("Evicting process pid %d queues\n",
1152 pdd->process->lead_thread->pid);
1153
1154 pdd->last_evict_timestamp = get_jiffies_64();
1155 /* Mark all queues as evicted. Deactivate all active queues on
1156 * the qpd.
1157 */
1158 list_for_each_entry(q, &qpd->queues_list, list) {
1159 q->properties.is_evicted = true;
1160 if (!q->properties.is_active)
1161 continue;
1162
1163 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1164 type: q->properties.type)];
1165 q->properties.is_active = false;
1166 decrement_queue_count(dqm, qpd, q);
1167
1168 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
1169 continue;
1170
1171 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1172 (dqm->dev->kfd->cwsr_enabled ?
1173 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1174 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1175 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1176 if (retval && !ret)
1177 /* Return the first error, but keep going to
1178 * maintain a consistent eviction state
1179 */
1180 ret = retval;
1181 }
1182
1183out:
1184 dqm_unlock(dqm);
1185 return ret;
1186}
1187
1188static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
1189 struct qcm_process_device *qpd)
1190{
1191 struct queue *q;
1192 struct device *dev = dqm->dev->adev->dev;
1193 struct kfd_process_device *pdd;
1194 int retval = 0;
1195
1196 dqm_lock(dqm);
1197 if (qpd->evicted++ > 0) /* already evicted, do nothing */
1198 goto out;
1199
1200 pdd = qpd_to_pdd(qpd);
1201
1202 /* The debugger creates processes that temporarily have not acquired
1203 * all VMs for all devices and has no VMs itself.
1204 * Skip queue eviction on process eviction.
1205 */
1206 if (!pdd->drm_priv)
1207 goto out;
1208
1209 pr_debug_ratelimited("Evicting process pid %d queues\n",
1210 pdd->process->lead_thread->pid);
1211
1212 /* Mark all queues as evicted. Deactivate all active queues on
1213 * the qpd.
1214 */
1215 list_for_each_entry(q, &qpd->queues_list, list) {
1216 q->properties.is_evicted = true;
1217 if (!q->properties.is_active)
1218 continue;
1219
1220 q->properties.is_active = false;
1221 decrement_queue_count(dqm, qpd, q);
1222
1223 if (dqm->dev->kfd->shared_resources.enable_mes) {
1224 int err;
1225
1226 err = remove_queue_mes(dqm, q, qpd);
1227 if (err) {
1228 dev_err(dev, "Failed to evict queue %d\n",
1229 q->properties.queue_id);
1230 retval = err;
1231 }
1232 }
1233 }
1234 pdd->last_evict_timestamp = get_jiffies_64();
1235 if (!dqm->dev->kfd->shared_resources.enable_mes)
1236 retval = execute_queues_cpsch(dqm,
1237 filter: qpd->is_debug ?
1238 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
1239 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
1240 USE_DEFAULT_GRACE_PERIOD);
1241
1242out:
1243 dqm_unlock(dqm);
1244 return retval;
1245}
1246
1247static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
1248 struct qcm_process_device *qpd)
1249{
1250 struct mm_struct *mm = NULL;
1251 struct queue *q;
1252 struct mqd_manager *mqd_mgr;
1253 struct kfd_process_device *pdd;
1254 uint64_t pd_base;
1255 uint64_t eviction_duration;
1256 int retval, ret = 0;
1257
1258 pdd = qpd_to_pdd(qpd);
1259 /* Retrieve PD base */
1260 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(drm_priv: pdd->drm_priv);
1261
1262 dqm_lock(dqm);
1263 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1264 goto out;
1265 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1266 qpd->evicted--;
1267 goto out;
1268 }
1269
1270 pr_debug_ratelimited("Restoring process pid %d queues\n",
1271 pdd->process->lead_thread->pid);
1272
1273 /* Update PD Base in QPD */
1274 qpd->page_table_base = pd_base;
1275 pr_debug("Updated PD address to 0x%llx\n", pd_base);
1276
1277 if (!list_empty(head: &qpd->queues_list)) {
1278 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
1279 dqm->dev->adev,
1280 qpd->vmid,
1281 qpd->page_table_base);
1282 kfd_flush_tlb(pdd, type: TLB_FLUSH_LEGACY);
1283 }
1284
1285 /* Take a safe reference to the mm_struct, which may otherwise
1286 * disappear even while the kfd_process is still referenced.
1287 */
1288 mm = get_task_mm(task: pdd->process->lead_thread);
1289 if (!mm) {
1290 ret = -EFAULT;
1291 goto out;
1292 }
1293
1294 /* Remove the eviction flags. Activate queues that are not
1295 * inactive for other reasons.
1296 */
1297 list_for_each_entry(q, &qpd->queues_list, list) {
1298 q->properties.is_evicted = false;
1299 if (!QUEUE_IS_ACTIVE(q->properties))
1300 continue;
1301
1302 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1303 type: q->properties.type)];
1304 q->properties.is_active = true;
1305 increment_queue_count(dqm, qpd, q);
1306
1307 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
1308 continue;
1309
1310 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1311 q->queue, &q->properties, mm);
1312 if (retval && !ret)
1313 /* Return the first error, but keep going to
1314 * maintain a consistent eviction state
1315 */
1316 ret = retval;
1317 }
1318 qpd->evicted = 0;
1319 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1320 atomic64_add(i: eviction_duration, v: &pdd->evict_duration_counter);
1321out:
1322 if (mm)
1323 mmput(mm);
1324 dqm_unlock(dqm);
1325 return ret;
1326}
1327
1328static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
1329 struct qcm_process_device *qpd)
1330{
1331 struct queue *q;
1332 struct device *dev = dqm->dev->adev->dev;
1333 struct kfd_process_device *pdd;
1334 uint64_t eviction_duration;
1335 int retval = 0;
1336
1337 pdd = qpd_to_pdd(qpd);
1338
1339 dqm_lock(dqm);
1340 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1341 goto out;
1342 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1343 qpd->evicted--;
1344 goto out;
1345 }
1346
1347 /* The debugger creates processes that temporarily have not acquired
1348 * all VMs for all devices and has no VMs itself.
1349 * Skip queue restore on process restore.
1350 */
1351 if (!pdd->drm_priv)
1352 goto vm_not_acquired;
1353
1354 pr_debug_ratelimited("Restoring process pid %d queues\n",
1355 pdd->process->lead_thread->pid);
1356
1357 /* Update PD Base in QPD */
1358 qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(drm_priv: pdd->drm_priv);
1359 pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);
1360
1361 /* activate all active queues on the qpd */
1362 list_for_each_entry(q, &qpd->queues_list, list) {
1363 q->properties.is_evicted = false;
1364 if (!QUEUE_IS_ACTIVE(q->properties))
1365 continue;
1366
1367 q->properties.is_active = true;
1368 increment_queue_count(dqm, qpd: &pdd->qpd, q);
1369
1370 if (dqm->dev->kfd->shared_resources.enable_mes) {
1371 retval = add_queue_mes(dqm, q, qpd);
1372 if (retval) {
1373 dev_err(dev, "Failed to restore queue %d\n",
1374 q->properties.queue_id);
1375 goto out;
1376 }
1377 }
1378 }
1379 if (!dqm->dev->kfd->shared_resources.enable_mes)
1380 retval = execute_queues_cpsch(dqm,
1381 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
1382 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1383 atomic64_add(i: eviction_duration, v: &pdd->evict_duration_counter);
1384vm_not_acquired:
1385 qpd->evicted = 0;
1386out:
1387 dqm_unlock(dqm);
1388 return retval;
1389}
1390
1391static int register_process(struct device_queue_manager *dqm,
1392 struct qcm_process_device *qpd)
1393{
1394 struct device_process_node *n;
1395 struct kfd_process_device *pdd;
1396 uint64_t pd_base;
1397 int retval;
1398
1399 n = kzalloc(sizeof(*n), GFP_KERNEL);
1400 if (!n)
1401 return -ENOMEM;
1402
1403 n->qpd = qpd;
1404
1405 pdd = qpd_to_pdd(qpd);
1406 /* Retrieve PD base */
1407 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(drm_priv: pdd->drm_priv);
1408
1409 dqm_lock(dqm);
1410 list_add(new: &n->list, head: &dqm->queues);
1411
1412 /* Update PD Base in QPD */
1413 qpd->page_table_base = pd_base;
1414 pr_debug("Updated PD address to 0x%llx\n", pd_base);
1415
1416 retval = dqm->asic_ops.update_qpd(dqm, qpd);
1417
1418 dqm->processes_count++;
1419
1420 dqm_unlock(dqm);
1421
1422 /* Outside the DQM lock because under the DQM lock we can't do
1423 * reclaim or take other locks that others hold while reclaiming.
1424 */
1425 kfd_inc_compute_active(dev: dqm->dev);
1426
1427 return retval;
1428}
1429
1430static int unregister_process(struct device_queue_manager *dqm,
1431 struct qcm_process_device *qpd)
1432{
1433 int retval;
1434 struct device_process_node *cur, *next;
1435
1436 pr_debug("qpd->queues_list is %s\n",
1437 list_empty(&qpd->queues_list) ? "empty" : "not empty");
1438
1439 retval = 0;
1440 dqm_lock(dqm);
1441
1442 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
1443 if (qpd == cur->qpd) {
1444 list_del(entry: &cur->list);
1445 kfree(objp: cur);
1446 dqm->processes_count--;
1447 goto out;
1448 }
1449 }
1450 /* qpd not found in dqm list */
1451 retval = 1;
1452out:
1453 dqm_unlock(dqm);
1454
1455 /* Outside the DQM lock because under the DQM lock we can't do
1456 * reclaim or take other locks that others hold while reclaiming.
1457 */
1458 if (!retval)
1459 kfd_dec_compute_active(dev: dqm->dev);
1460
1461 return retval;
1462}
1463
1464static int
1465set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
1466 unsigned int vmid)
1467{
1468 uint32_t xcc_mask = dqm->dev->xcc_mask;
1469 int xcc_id, ret;
1470
1471 for_each_inst(xcc_id, xcc_mask) {
1472 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
1473 dqm->dev->adev, pasid, vmid, xcc_id);
1474 if (ret)
1475 break;
1476 }
1477
1478 return ret;
1479}
1480
1481static void init_interrupts(struct device_queue_manager *dqm)
1482{
1483 uint32_t xcc_mask = dqm->dev->xcc_mask;
1484 unsigned int i, xcc_id;
1485
1486 for_each_inst(xcc_id, xcc_mask) {
1487 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {
1488 if (is_pipe_enabled(dqm, mec: 0, pipe: i)) {
1489 dqm->dev->kfd2kgd->init_interrupts(
1490 dqm->dev->adev, i, xcc_id);
1491 }
1492 }
1493 }
1494}
1495
1496static int initialize_nocpsch(struct device_queue_manager *dqm)
1497{
1498 int pipe, queue;
1499
1500 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1501
1502 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
1503 sizeof(unsigned int), GFP_KERNEL);
1504 if (!dqm->allocated_queues)
1505 return -ENOMEM;
1506
1507 mutex_init(&dqm->lock_hidden);
1508 INIT_LIST_HEAD(list: &dqm->queues);
1509 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
1510 dqm->active_cp_queue_count = 0;
1511 dqm->gws_queue_count = 0;
1512
1513 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1514 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1515
1516 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
1517 if (test_bit(pipe_offset + queue,
1518 dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1519 dqm->allocated_queues[pipe] |= 1 << queue;
1520 }
1521
1522 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
1523
1524 init_sdma_bitmaps(dqm);
1525
1526 return 0;
1527}
1528
1529static void uninitialize(struct device_queue_manager *dqm)
1530{
1531 int i;
1532
1533 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
1534
1535 kfree(objp: dqm->allocated_queues);
1536 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
1537 kfree(objp: dqm->mqd_mgrs[i]);
1538 mutex_destroy(lock: &dqm->lock_hidden);
1539}
1540
1541static int start_nocpsch(struct device_queue_manager *dqm)
1542{
1543 int r = 0;
1544
1545 pr_info("SW scheduler is used");
1546 init_interrupts(dqm);
1547
1548 if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1549 r = pm_init(pm: &dqm->packet_mgr, dqm);
1550 if (!r)
1551 dqm->sched_running = true;
1552
1553 return r;
1554}
1555
1556static int stop_nocpsch(struct device_queue_manager *dqm)
1557{
1558 dqm_lock(dqm);
1559 if (!dqm->sched_running) {
1560 dqm_unlock(dqm);
1561 return 0;
1562 }
1563
1564 if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1565 pm_uninit(pm: &dqm->packet_mgr);
1566 dqm->sched_running = false;
1567 dqm_unlock(dqm);
1568
1569 return 0;
1570}
1571
1572static int allocate_sdma_queue(struct device_queue_manager *dqm,
1573 struct queue *q, const uint32_t *restore_sdma_id)
1574{
1575 struct device *dev = dqm->dev->adev->dev;
1576 int bit;
1577
1578 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1579 if (bitmap_empty(src: dqm->sdma_bitmap, nbits: get_num_sdma_queues(dqm))) {
1580 dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n",
1581 get_num_sdma_queues(dqm));
1582 return -ENOMEM;
1583 }
1584
1585 if (restore_sdma_id) {
1586 /* Re-use existing sdma_id */
1587 if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
1588 dev_err(dev, "SDMA queue already in use\n");
1589 return -EBUSY;
1590 }
1591 clear_bit(nr: *restore_sdma_id, addr: dqm->sdma_bitmap);
1592 q->sdma_id = *restore_sdma_id;
1593 } else {
1594 /* Find first available sdma_id */
1595 bit = find_first_bit(addr: dqm->sdma_bitmap,
1596 size: get_num_sdma_queues(dqm));
1597 clear_bit(nr: bit, addr: dqm->sdma_bitmap);
1598 q->sdma_id = bit;
1599 }
1600
1601 q->properties.sdma_engine_id =
1602 q->sdma_id % kfd_get_num_sdma_engines(kdev: dqm->dev);
1603 q->properties.sdma_queue_id = q->sdma_id /
1604 kfd_get_num_sdma_engines(kdev: dqm->dev);
1605 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1606 if (bitmap_empty(src: dqm->xgmi_sdma_bitmap, nbits: get_num_xgmi_sdma_queues(dqm))) {
1607 dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n",
1608 get_num_xgmi_sdma_queues(dqm));
1609 return -ENOMEM;
1610 }
1611 if (restore_sdma_id) {
1612 /* Re-use existing sdma_id */
1613 if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
1614 dev_err(dev, "SDMA queue already in use\n");
1615 return -EBUSY;
1616 }
1617 clear_bit(nr: *restore_sdma_id, addr: dqm->xgmi_sdma_bitmap);
1618 q->sdma_id = *restore_sdma_id;
1619 } else {
1620 bit = find_first_bit(addr: dqm->xgmi_sdma_bitmap,
1621 size: get_num_xgmi_sdma_queues(dqm));
1622 clear_bit(nr: bit, addr: dqm->xgmi_sdma_bitmap);
1623 q->sdma_id = bit;
1624 }
1625 /* sdma_engine_id is sdma id including
1626 * both PCIe-optimized SDMAs and XGMI-
1627 * optimized SDMAs. The calculation below
1628 * assumes the first N engines are always
1629 * PCIe-optimized ones
1630 */
1631 q->properties.sdma_engine_id =
1632 kfd_get_num_sdma_engines(kdev: dqm->dev) +
1633 q->sdma_id % kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
1634 q->properties.sdma_queue_id = q->sdma_id /
1635 kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
1636 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1637 int i, num_queues, num_engines, eng_offset = 0, start_engine;
1638 bool free_bit_found = false, is_xgmi = false;
1639
1640 if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(kdev: dqm->dev)) {
1641 num_queues = get_num_sdma_queues(dqm);
1642 num_engines = kfd_get_num_sdma_engines(kdev: dqm->dev);
1643 q->properties.type = KFD_QUEUE_TYPE_SDMA;
1644 } else {
1645 num_queues = get_num_xgmi_sdma_queues(dqm);
1646 num_engines = kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
1647 eng_offset = kfd_get_num_sdma_engines(kdev: dqm->dev);
1648 q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI;
1649 is_xgmi = true;
1650 }
1651
1652 /* Scan available bit based on target engine ID. */
1653 start_engine = q->properties.sdma_engine_id - eng_offset;
1654 for (i = start_engine; i < num_queues; i += num_engines) {
1655
1656 if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap))
1657 continue;
1658
1659 clear_bit(nr: i, addr: is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap);
1660 q->sdma_id = i;
1661 q->properties.sdma_queue_id = q->sdma_id / num_engines;
1662 free_bit_found = true;
1663 break;
1664 }
1665
1666 if (!free_bit_found) {
1667 dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n",
1668 q->properties.sdma_engine_id, num_queues);
1669 return -ENOMEM;
1670 }
1671 }
1672
1673 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1674 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
1675
1676 return 0;
1677}
1678
1679static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1680 struct queue *q)
1681{
1682 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1683 if (q->sdma_id >= get_num_sdma_queues(dqm))
1684 return;
1685 set_bit(nr: q->sdma_id, addr: dqm->sdma_bitmap);
1686 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1687 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1688 return;
1689 set_bit(nr: q->sdma_id, addr: dqm->xgmi_sdma_bitmap);
1690 }
1691}
1692
1693/*
1694 * Device Queue Manager implementation for cp scheduler
1695 */
1696
1697static int set_sched_resources(struct device_queue_manager *dqm)
1698{
1699 int i, mec;
1700 struct scheduling_resources res;
1701 struct device *dev = dqm->dev->adev->dev;
1702
1703 res.vmid_mask = dqm->dev->compute_vmid_bitmap;
1704
1705 res.queue_mask = 0;
1706 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
1707 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
1708 / dqm->dev->kfd->shared_resources.num_pipe_per_mec;
1709
1710 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1711 continue;
1712
1713 /* only acquire queues from the first MEC */
1714 if (mec > 0)
1715 continue;
1716
1717 /* This situation may be hit in the future if a new HW
1718 * generation exposes more than 64 queues. If so, the
1719 * definition of res.queue_mask needs updating
1720 */
1721 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
1722 dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i);
1723 break;
1724 }
1725
1726 res.queue_mask |= 1ull
1727 << amdgpu_queue_mask_bit_to_set_resource_bit(
1728 adev: dqm->dev->adev, queue_bit: i);
1729 }
1730 res.gws_mask = ~0ull;
1731 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
1732
1733 pr_debug("Scheduling resources:\n"
1734 "vmid mask: 0x%8X\n"
1735 "queue mask: 0x%8llX\n",
1736 res.vmid_mask, res.queue_mask);
1737
1738 return pm_send_set_resources(pm: &dqm->packet_mgr, res: &res);
1739}
1740
1741static int initialize_cpsch(struct device_queue_manager *dqm)
1742{
1743 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1744
1745 mutex_init(&dqm->lock_hidden);
1746 INIT_LIST_HEAD(list: &dqm->queues);
1747 dqm->active_queue_count = dqm->processes_count = 0;
1748 dqm->active_cp_queue_count = 0;
1749 dqm->gws_queue_count = 0;
1750 dqm->active_runlist = false;
1751 dqm->trap_debug_vmid = 0;
1752
1753 init_sdma_bitmaps(dqm);
1754
1755 update_dqm_wait_times(dqm);
1756 return 0;
1757}
1758
1759/* halt_cpsch:
1760 * Unmap queues so the schedule doesn't continue remaining jobs in the queue.
1761 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch
1762 * is called.
1763 */
1764static int halt_cpsch(struct device_queue_manager *dqm)
1765{
1766 int ret = 0;
1767
1768 dqm_lock(dqm);
1769 if (!dqm->sched_running) {
1770 dqm_unlock(dqm);
1771 return 0;
1772 }
1773
1774 WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n");
1775
1776 if (!dqm->is_hws_hang) {
1777 if (!dqm->dev->kfd->shared_resources.enable_mes)
1778 ret = unmap_queues_cpsch(dqm,
1779 filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
1780 USE_DEFAULT_GRACE_PERIOD, reset: false);
1781 else
1782 ret = remove_all_kfd_queues_mes(dqm);
1783 }
1784 dqm->sched_halt = true;
1785 dqm_unlock(dqm);
1786
1787 return ret;
1788}
1789
1790/* unhalt_cpsch
1791 * Unset dqm->sched_halt and map queues back to runlist
1792 */
1793static int unhalt_cpsch(struct device_queue_manager *dqm)
1794{
1795 int ret = 0;
1796
1797 dqm_lock(dqm);
1798 if (!dqm->sched_running || !dqm->sched_halt) {
1799 WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");
1800 dqm_unlock(dqm);
1801 return 0;
1802 }
1803 dqm->sched_halt = false;
1804 if (!dqm->dev->kfd->shared_resources.enable_mes)
1805 ret = execute_queues_cpsch(dqm,
1806 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
1807 filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
1808 else
1809 ret = add_all_kfd_queues_mes(dqm);
1810
1811 dqm_unlock(dqm);
1812
1813 return ret;
1814}
1815
1816static int start_cpsch(struct device_queue_manager *dqm)
1817{
1818 struct device *dev = dqm->dev->adev->dev;
1819 int retval, num_hw_queue_slots;
1820
1821 retval = 0;
1822
1823 dqm_lock(dqm);
1824
1825 if (!dqm->dev->kfd->shared_resources.enable_mes) {
1826 retval = pm_init(pm: &dqm->packet_mgr, dqm);
1827 if (retval)
1828 goto fail_packet_manager_init;
1829
1830 retval = set_sched_resources(dqm);
1831 if (retval)
1832 goto fail_set_sched_resources;
1833 }
1834 pr_debug("Allocating fence memory\n");
1835
1836 /* allocate fence memory on the gart */
1837 retval = kfd_gtt_sa_allocate(node: dqm->dev, size: sizeof(*dqm->fence_addr),
1838 mem_obj: &dqm->fence_mem);
1839
1840 if (retval)
1841 goto fail_allocate_vidmem;
1842
1843 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
1844 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
1845
1846 init_interrupts(dqm);
1847
1848 /* clear hang status when driver try to start the hw scheduler */
1849 dqm->sched_running = true;
1850
1851 if (!dqm->dev->kfd->shared_resources.enable_mes) {
1852 if (pm_config_dequeue_wait_counts(pm: &dqm->packet_mgr,
1853 cmd: KFD_DEQUEUE_WAIT_INIT, wait_counts_config: 0 /* unused */))
1854 dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
1855 execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
1856 }
1857
1858 /* setup per-queue reset detection buffer */
1859 num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe *
1860 dqm->dev->kfd->shared_resources.num_pipe_per_mec *
1861 NUM_XCC(dqm->dev->xcc_mask);
1862
1863 dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info);
1864 dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL);
1865
1866 if (!dqm->detect_hang_info) {
1867 retval = -ENOMEM;
1868 goto fail_detect_hang_buffer;
1869 }
1870
1871 dqm_unlock(dqm);
1872
1873 return 0;
1874fail_detect_hang_buffer:
1875 kfd_gtt_sa_free(node: dqm->dev, mem_obj: dqm->fence_mem);
1876fail_allocate_vidmem:
1877fail_set_sched_resources:
1878 if (!dqm->dev->kfd->shared_resources.enable_mes)
1879 pm_uninit(pm: &dqm->packet_mgr);
1880fail_packet_manager_init:
1881 dqm_unlock(dqm);
1882 return retval;
1883}
1884
1885static int stop_cpsch(struct device_queue_manager *dqm)
1886{
1887 dqm_lock(dqm);
1888 if (!dqm->sched_running) {
1889 dqm_unlock(dqm);
1890 return 0;
1891 }
1892
1893 if (!dqm->dev->kfd->shared_resources.enable_mes)
1894 unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD, reset: false);
1895 else
1896 remove_all_kfd_queues_mes(dqm);
1897
1898 dqm->sched_running = false;
1899
1900 if (!dqm->dev->kfd->shared_resources.enable_mes)
1901 pm_release_ib(pm: &dqm->packet_mgr);
1902
1903 kfd_gtt_sa_free(node: dqm->dev, mem_obj: dqm->fence_mem);
1904 if (!dqm->dev->kfd->shared_resources.enable_mes)
1905 pm_uninit(pm: &dqm->packet_mgr);
1906 kfree(objp: dqm->detect_hang_info);
1907 dqm->detect_hang_info = NULL;
1908 dqm_unlock(dqm);
1909
1910 return 0;
1911}
1912
1913static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1914 struct kernel_queue *kq,
1915 struct qcm_process_device *qpd)
1916{
1917 dqm_lock(dqm);
1918 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1919 pr_warn("Can't create new kernel queue because %d queues were already created\n",
1920 dqm->total_queue_count);
1921 dqm_unlock(dqm);
1922 return -EPERM;
1923 }
1924
1925 /*
1926 * Unconditionally increment this counter, regardless of the queue's
1927 * type or whether the queue is active.
1928 */
1929 dqm->total_queue_count++;
1930 pr_debug("Total of %d queues are accountable so far\n",
1931 dqm->total_queue_count);
1932
1933 list_add(new: &kq->list, head: &qpd->priv_queue_list);
1934 increment_queue_count(dqm, qpd, q: kq->queue);
1935 qpd->is_debug = true;
1936 execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
1937 USE_DEFAULT_GRACE_PERIOD);
1938 dqm_unlock(dqm);
1939
1940 return 0;
1941}
1942
1943static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1944 struct kernel_queue *kq,
1945 struct qcm_process_device *qpd)
1946{
1947 dqm_lock(dqm);
1948 list_del(entry: &kq->list);
1949 decrement_queue_count(dqm, qpd, q: kq->queue);
1950 qpd->is_debug = false;
1951 execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
1952 USE_DEFAULT_GRACE_PERIOD);
1953 /*
1954 * Unconditionally decrement this counter, regardless of the queue's
1955 * type.
1956 */
1957 dqm->total_queue_count--;
1958 pr_debug("Total of %d queues are accountable so far\n",
1959 dqm->total_queue_count);
1960 dqm_unlock(dqm);
1961}
1962
1963static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1964 struct qcm_process_device *qpd,
1965 const struct kfd_criu_queue_priv_data *qd,
1966 const void *restore_mqd, const void *restore_ctl_stack)
1967{
1968 int retval;
1969 struct mqd_manager *mqd_mgr;
1970
1971 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1972 pr_warn("Can't create new usermode queue because %d queues were already created\n",
1973 dqm->total_queue_count);
1974 retval = -EPERM;
1975 goto out;
1976 }
1977
1978 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1979 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI ||
1980 q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1981 dqm_lock(dqm);
1982 retval = allocate_sdma_queue(dqm, q, restore_sdma_id: qd ? &qd->sdma_id : NULL);
1983 dqm_unlock(dqm);
1984 if (retval)
1985 goto out;
1986 }
1987
1988 retval = allocate_doorbell(qpd, q, restore_id: qd ? &qd->doorbell_id : NULL);
1989 if (retval)
1990 goto out_deallocate_sdma_queue;
1991
1992 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1993 type: q->properties.type)];
1994
1995 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1996 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1997 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
1998 q->properties.tba_addr = qpd->tba_addr;
1999 q->properties.tma_addr = qpd->tma_addr;
2000 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
2001 if (!q->mqd_mem_obj) {
2002 retval = -ENOMEM;
2003 goto out_deallocate_doorbell;
2004 }
2005
2006 dqm_lock(dqm);
2007 /*
2008 * Eviction state logic: mark all queues as evicted, even ones
2009 * not currently active. Restoring inactive queues later only
2010 * updates the is_evicted flag but is a no-op otherwise.
2011 */
2012 q->properties.is_evicted = !!qpd->evicted;
2013 q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&
2014 kfd_dbg_has_cwsr_workaround(dev: q->device);
2015
2016 if (qd)
2017 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
2018 &q->properties, restore_mqd, restore_ctl_stack,
2019 qd->ctl_stack_size);
2020 else
2021 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
2022 &q->gart_mqd_addr, &q->properties);
2023
2024 list_add(new: &q->list, head: &qpd->queues_list);
2025 qpd->queue_count++;
2026
2027 if (q->properties.is_active) {
2028 increment_queue_count(dqm, qpd, q);
2029
2030 if (!dqm->dev->kfd->shared_resources.enable_mes)
2031 retval = execute_queues_cpsch(dqm,
2032 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
2033 else
2034 retval = add_queue_mes(dqm, q, qpd);
2035 if (retval)
2036 goto cleanup_queue;
2037 }
2038
2039 /*
2040 * Unconditionally increment this counter, regardless of the queue's
2041 * type or whether the queue is active.
2042 */
2043 dqm->total_queue_count++;
2044
2045 pr_debug("Total of %d queues are accountable so far\n",
2046 dqm->total_queue_count);
2047
2048 dqm_unlock(dqm);
2049 return retval;
2050
2051cleanup_queue:
2052 qpd->queue_count--;
2053 list_del(entry: &q->list);
2054 if (q->properties.is_active)
2055 decrement_queue_count(dqm, qpd, q);
2056 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2057 dqm_unlock(dqm);
2058out_deallocate_doorbell:
2059 deallocate_doorbell(qpd, q);
2060out_deallocate_sdma_queue:
2061 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2062 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
2063 dqm_lock(dqm);
2064 deallocate_sdma_queue(dqm, q);
2065 dqm_unlock(dqm);
2066 }
2067out:
2068 return retval;
2069}
2070
2071int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,
2072 uint64_t fence_value,
2073 unsigned int timeout_ms)
2074{
2075 unsigned long end_jiffies = msecs_to_jiffies(m: timeout_ms) + jiffies;
2076 struct device *dev = dqm->dev->adev->dev;
2077 uint64_t *fence_addr = dqm->fence_addr;
2078
2079 while (*fence_addr != fence_value) {
2080 /* Fatal err detected, this response won't come */
2081 if (amdgpu_amdkfd_is_fed(adev: dqm->dev->adev))
2082 return -EIO;
2083
2084 if (time_after(jiffies, end_jiffies)) {
2085 dev_err(dev, "qcm fence wait loop timeout expired\n");
2086 /* In HWS case, this is used to halt the driver thread
2087 * in order not to mess up CP states before doing
2088 * scandumps for FW debugging.
2089 */
2090 while (halt_if_hws_hang)
2091 schedule();
2092
2093 return -ETIME;
2094 }
2095 schedule();
2096 }
2097
2098 return 0;
2099}
2100
2101/* dqm->lock mutex has to be locked before calling this function */
2102static int map_queues_cpsch(struct device_queue_manager *dqm)
2103{
2104 struct device *dev = dqm->dev->adev->dev;
2105 int retval;
2106
2107 if (!dqm->sched_running || dqm->sched_halt)
2108 return 0;
2109 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
2110 return 0;
2111 if (dqm->active_runlist)
2112 return 0;
2113
2114 retval = pm_send_runlist(pm: &dqm->packet_mgr, dqm_queues: &dqm->queues);
2115 pr_debug("%s sent runlist\n", __func__);
2116 if (retval) {
2117 dev_err(dev, "failed to execute runlist\n");
2118 return retval;
2119 }
2120 dqm->active_runlist = true;
2121
2122 return retval;
2123}
2124
2125static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
2126 struct qcm_process_device *qpd)
2127{
2128 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2129
2130 dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n",
2131 q->properties.queue_id, pdd->process->lead_thread->pid);
2132
2133 pdd->has_reset_queue = true;
2134 if (q->properties.is_active) {
2135 q->properties.is_active = false;
2136 decrement_queue_count(dqm, qpd, q);
2137 }
2138}
2139
2140static int detect_queue_hang(struct device_queue_manager *dqm)
2141{
2142 int i;
2143
2144 /* detect should be used only in dqm locked queue reset */
2145 if (WARN_ON(dqm->detect_hang_count > 0))
2146 return 0;
2147
2148 memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size);
2149
2150 for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
2151 uint32_t mec, pipe, queue;
2152 int xcc_id;
2153
2154 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
2155 / dqm->dev->kfd->shared_resources.num_pipe_per_mec;
2156
2157 if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
2158 continue;
2159
2160 amdgpu_queue_mask_bit_to_mec_queue(adev: dqm->dev->adev, bit: i, mec: &mec, pipe: &pipe, queue: &queue);
2161
2162 for_each_inst(xcc_id, dqm->dev->xcc_mask) {
2163 uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr(
2164 dqm->dev->adev, pipe, queue, xcc_id);
2165 struct dqm_detect_hang_info hang_info;
2166
2167 if (!queue_addr)
2168 continue;
2169
2170 hang_info.pipe_id = pipe;
2171 hang_info.queue_id = queue;
2172 hang_info.xcc_id = xcc_id;
2173 hang_info.queue_address = queue_addr;
2174
2175 dqm->detect_hang_info[dqm->detect_hang_count] = hang_info;
2176 dqm->detect_hang_count++;
2177 }
2178 }
2179
2180 return dqm->detect_hang_count;
2181}
2182
2183static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address)
2184{
2185 struct device_process_node *cur;
2186 struct qcm_process_device *qpd;
2187 struct queue *q;
2188
2189 list_for_each_entry(cur, &dqm->queues, list) {
2190 qpd = cur->qpd;
2191 list_for_each_entry(q, &qpd->queues_list, list) {
2192 if (queue_address == q->properties.queue_address)
2193 return q;
2194 }
2195 }
2196
2197 return NULL;
2198}
2199
2200static int reset_hung_queues(struct device_queue_manager *dqm)
2201{
2202 int r = 0, reset_count = 0, i;
2203
2204 if (!dqm->detect_hang_info || dqm->is_hws_hang)
2205 return -EIO;
2206
2207 /* assume dqm locked. */
2208 if (!detect_queue_hang(dqm))
2209 return -ENOTRECOVERABLE;
2210
2211 for (i = 0; i < dqm->detect_hang_count; i++) {
2212 struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i];
2213 struct queue *q = find_queue_by_address(dqm, queue_address: hang_info.queue_address);
2214 struct kfd_process_device *pdd;
2215 uint64_t queue_addr = 0;
2216
2217 if (!q) {
2218 r = -ENOTRECOVERABLE;
2219 goto reset_fail;
2220 }
2221
2222 pdd = kfd_get_process_device_data(dev: dqm->dev, p: q->process);
2223 if (!pdd) {
2224 r = -ENOTRECOVERABLE;
2225 goto reset_fail;
2226 }
2227
2228 queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev,
2229 hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id,
2230 KFD_UNMAP_LATENCY_MS);
2231
2232 /* either reset failed or we reset an unexpected queue. */
2233 if (queue_addr != q->properties.queue_address) {
2234 r = -ENOTRECOVERABLE;
2235 goto reset_fail;
2236 }
2237
2238 set_queue_as_reset(dqm, q, qpd: &pdd->qpd);
2239 reset_count++;
2240 }
2241
2242 if (reset_count == dqm->detect_hang_count)
2243 kfd_signal_reset_event(dev: dqm->dev);
2244 else
2245 r = -ENOTRECOVERABLE;
2246
2247reset_fail:
2248 dqm->detect_hang_count = 0;
2249
2250 return r;
2251}
2252
2253static bool sdma_has_hang(struct device_queue_manager *dqm)
2254{
2255 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2256 int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2257 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2258 int i, j;
2259
2260 for (i = engine_start; i < engine_end; i++) {
2261 for (j = 0; j < num_queues_per_eng; j++) {
2262 if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j))
2263 continue;
2264
2265 return true;
2266 }
2267 }
2268
2269 return false;
2270}
2271
2272static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm,
2273 uint32_t doorbell_off)
2274{
2275 struct device_process_node *cur;
2276 struct qcm_process_device *qpd;
2277 struct queue *q;
2278
2279 list_for_each_entry(cur, &dqm->queues, list) {
2280 qpd = cur->qpd;
2281 list_for_each_entry(q, &qpd->queues_list, list) {
2282 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2283 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
2284 q->properties.doorbell_off == doorbell_off) {
2285 set_queue_as_reset(dqm, q, qpd);
2286 return true;
2287 }
2288 }
2289 }
2290
2291 return false;
2292}
2293
2294static int reset_hung_queues_sdma(struct device_queue_manager *dqm)
2295{
2296 int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2297 int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2298 int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2299 int r = 0, i, j;
2300
2301 if (dqm->is_hws_hang)
2302 return -EIO;
2303
2304 /* Scan for hung HW queues and reset engine. */
2305 dqm->detect_hang_count = 0;
2306 for (i = engine_start; i < engine_end; i++) {
2307 for (j = 0; j < num_queues_per_eng; j++) {
2308 uint32_t doorbell_off =
2309 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j);
2310
2311 if (!doorbell_off)
2312 continue;
2313
2314 /* Reset engine and check. */
2315 if (amdgpu_sdma_reset_engine(adev: dqm->dev->adev, instance_id: i) ||
2316 dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) ||
2317 !set_sdma_queue_as_reset(dqm, doorbell_off)) {
2318 r = -ENOTRECOVERABLE;
2319 goto reset_fail;
2320 }
2321
2322 /* Should only expect one queue active per engine */
2323 dqm->detect_hang_count++;
2324 break;
2325 }
2326 }
2327
2328 /* Signal process reset */
2329 if (dqm->detect_hang_count)
2330 kfd_signal_reset_event(dev: dqm->dev);
2331 else
2332 r = -ENOTRECOVERABLE;
2333
2334reset_fail:
2335 dqm->detect_hang_count = 0;
2336
2337 return r;
2338}
2339
2340static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma)
2341{
2342 while (halt_if_hws_hang)
2343 schedule();
2344
2345 if (!amdgpu_gpu_recovery)
2346 return -ENOTRECOVERABLE;
2347
2348 return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm);
2349}
2350
2351/* dqm->lock mutex has to be locked before calling this function
2352 *
2353 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time
2354 * for context switch latency. Lower values are used by debugger
2355 * since context switching are triggered at high frequency.
2356 * This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE
2357 *
2358 */
2359static int unmap_queues_cpsch(struct device_queue_manager *dqm,
2360 enum kfd_unmap_queues_filter filter,
2361 uint32_t filter_param,
2362 uint32_t grace_period,
2363 bool reset)
2364{
2365 struct device *dev = dqm->dev->adev->dev;
2366 struct mqd_manager *mqd_mgr;
2367 int retval;
2368
2369 if (!dqm->sched_running)
2370 return 0;
2371 if (!dqm->active_runlist)
2372 return 0;
2373 if (!down_read_trylock(sem: &dqm->dev->adev->reset_domain->sem))
2374 return -EIO;
2375
2376 if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2377 retval = pm_config_dequeue_wait_counts(pm: &dqm->packet_mgr,
2378 cmd: KFD_DEQUEUE_WAIT_SET_SCH_WAVE, wait_counts_config: grace_period);
2379 if (retval)
2380 goto out;
2381 }
2382
2383 retval = pm_send_unmap_queue(pm: &dqm->packet_mgr, mode: filter, filter_param, reset);
2384 if (retval)
2385 goto out;
2386
2387 *dqm->fence_addr = KFD_FENCE_INIT;
2388 mb();
2389 pm_send_query_status(pm: &dqm->packet_mgr, fence_address: dqm->fence_gpu_addr,
2390 KFD_FENCE_COMPLETED);
2391 /* should be timed out */
2392 retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED,
2393 timeout_ms: queue_preemption_timeout_ms);
2394 if (retval) {
2395 dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
2396 kfd_hws_hang(dqm);
2397 goto out;
2398 }
2399
2400 /* In the current MEC firmware implementation, if compute queue
2401 * doesn't response to the preemption request in time, HIQ will
2402 * abandon the unmap request without returning any timeout error
2403 * to driver. Instead, MEC firmware will log the doorbell of the
2404 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
2405 * To make sure the queue unmap was successful, driver need to
2406 * check those fields
2407 */
2408 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
2409 if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) &&
2410 reset_queues_on_hws_hang(dqm, is_sdma: false))
2411 goto reset_fail;
2412
2413 /* Check for SDMA hang and attempt SDMA reset */
2414 if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, is_sdma: true))
2415 goto reset_fail;
2416
2417 /* We need to reset the grace period value for this device */
2418 if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2419 if (pm_config_dequeue_wait_counts(pm: &dqm->packet_mgr,
2420 cmd: KFD_DEQUEUE_WAIT_RESET, wait_counts_config: 0 /* unused */))
2421 dev_err(dev, "Failed to reset grace period\n");
2422 }
2423
2424 pm_release_ib(pm: &dqm->packet_mgr);
2425 dqm->active_runlist = false;
2426out:
2427 up_read(sem: &dqm->dev->adev->reset_domain->sem);
2428 return retval;
2429
2430reset_fail:
2431 dqm->is_hws_hang = true;
2432 kfd_hws_hang(dqm);
2433 up_read(sem: &dqm->dev->adev->reset_domain->sem);
2434 return -ETIME;
2435}
2436
2437/* only for compute queue */
2438static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid)
2439{
2440 int retval;
2441
2442 dqm_lock(dqm);
2443
2444 retval = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_BY_PASID,
2445 filter_param: pasid, USE_DEFAULT_GRACE_PERIOD, reset: true);
2446
2447 dqm_unlock(dqm);
2448 return retval;
2449}
2450
2451/* dqm->lock mutex has to be locked before calling this function */
2452static int execute_queues_cpsch(struct device_queue_manager *dqm,
2453 enum kfd_unmap_queues_filter filter,
2454 uint32_t filter_param,
2455 uint32_t grace_period)
2456{
2457 int retval;
2458
2459 if (!down_read_trylock(sem: &dqm->dev->adev->reset_domain->sem))
2460 return -EIO;
2461 retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, reset: false);
2462 if (!retval)
2463 retval = map_queues_cpsch(dqm);
2464 up_read(sem: &dqm->dev->adev->reset_domain->sem);
2465 return retval;
2466}
2467
2468static int wait_on_destroy_queue(struct device_queue_manager *dqm,
2469 struct queue *q)
2470{
2471 struct kfd_process_device *pdd = kfd_get_process_device_data(dev: q->device,
2472 p: q->process);
2473 int ret = 0;
2474
2475 if (WARN_ON(!pdd))
2476 return ret;
2477
2478 if (pdd->qpd.is_debug)
2479 return ret;
2480
2481 q->properties.is_being_destroyed = true;
2482
2483 if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {
2484 dqm_unlock(dqm);
2485 mutex_unlock(lock: &q->process->mutex);
2486 ret = wait_event_interruptible(dqm->destroy_wait,
2487 !q->properties.is_suspended);
2488
2489 mutex_lock(&q->process->mutex);
2490 dqm_lock(dqm);
2491 }
2492
2493 return ret;
2494}
2495
2496static int destroy_queue_cpsch(struct device_queue_manager *dqm,
2497 struct qcm_process_device *qpd,
2498 struct queue *q)
2499{
2500 int retval;
2501 struct mqd_manager *mqd_mgr;
2502 uint64_t sdma_val = 0;
2503 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2504 struct device *dev = dqm->dev->adev->dev;
2505
2506 /* Get the SDMA queue stats */
2507 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2508 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2509 retval = read_sdma_queue_counter(q_rptr: (uint64_t __user *)q->properties.read_ptr,
2510 val: &sdma_val);
2511 if (retval)
2512 dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
2513 q->properties.queue_id);
2514 }
2515
2516 /* remove queue from list to prevent rescheduling after preemption */
2517 dqm_lock(dqm);
2518
2519 retval = wait_on_destroy_queue(dqm, q);
2520
2521 if (retval) {
2522 dqm_unlock(dqm);
2523 return retval;
2524 }
2525
2526 if (qpd->is_debug) {
2527 /*
2528 * error, currently we do not allow to destroy a queue
2529 * of a currently debugged process
2530 */
2531 retval = -EBUSY;
2532 goto failed_try_destroy_debugged_queue;
2533
2534 }
2535
2536 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2537 type: q->properties.type)];
2538
2539 deallocate_doorbell(qpd, q);
2540
2541 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2542 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2543 deallocate_sdma_queue(dqm, q);
2544 pdd->sdma_past_activity_counter += sdma_val;
2545 }
2546
2547 if (q->properties.is_active) {
2548 decrement_queue_count(dqm, qpd, q);
2549 q->properties.is_active = false;
2550 if (!dqm->dev->kfd->shared_resources.enable_mes) {
2551 retval = execute_queues_cpsch(dqm,
2552 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
2553 USE_DEFAULT_GRACE_PERIOD);
2554 if (retval == -ETIME)
2555 qpd->reset_wavefronts = true;
2556 } else {
2557 retval = remove_queue_mes(dqm, q, qpd);
2558 }
2559 }
2560 list_del(entry: &q->list);
2561 qpd->queue_count--;
2562
2563 /*
2564 * Unconditionally decrement this counter, regardless of the queue's
2565 * type
2566 */
2567 dqm->total_queue_count--;
2568 pr_debug("Total of %d queues are accountable so far\n",
2569 dqm->total_queue_count);
2570
2571 dqm_unlock(dqm);
2572
2573 /*
2574 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid
2575 * circular locking
2576 */
2577 kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),
2578 process: qpd->pqm->process, dev: q->device,
2579 source_id: -1, use_worker: false, NULL, exception_data_size: 0);
2580
2581 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2582
2583 return retval;
2584
2585failed_try_destroy_debugged_queue:
2586
2587 dqm_unlock(dqm);
2588 return retval;
2589}
2590
2591static bool set_cache_memory_policy(struct device_queue_manager *dqm,
2592 struct qcm_process_device *qpd,
2593 enum cache_policy default_policy,
2594 enum cache_policy alternate_policy,
2595 void __user *alternate_aperture_base,
2596 uint64_t alternate_aperture_size,
2597 u32 misc_process_properties)
2598{
2599 bool retval = true;
2600
2601 if (!dqm->asic_ops.set_cache_memory_policy)
2602 return retval;
2603
2604 dqm_lock(dqm);
2605
2606 retval = dqm->asic_ops.set_cache_memory_policy(
2607 dqm,
2608 qpd,
2609 default_policy,
2610 alternate_policy,
2611 alternate_aperture_base,
2612 alternate_aperture_size,
2613 misc_process_properties);
2614
2615 if (retval)
2616 goto out;
2617
2618 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
2619 program_sh_mem_settings(dqm, qpd);
2620
2621 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
2622 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
2623 qpd->sh_mem_ape1_limit);
2624
2625out:
2626 dqm_unlock(dqm);
2627 return retval;
2628}
2629
2630static int process_termination_nocpsch(struct device_queue_manager *dqm,
2631 struct qcm_process_device *qpd)
2632{
2633 struct queue *q;
2634 struct device_process_node *cur, *next_dpn;
2635 int retval = 0;
2636 bool found = false;
2637
2638 dqm_lock(dqm);
2639
2640 /* Clear all user mode queues */
2641 while (!list_empty(head: &qpd->queues_list)) {
2642 struct mqd_manager *mqd_mgr;
2643 int ret;
2644
2645 q = list_first_entry(&qpd->queues_list, struct queue, list);
2646 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2647 type: q->properties.type)];
2648 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
2649 if (ret)
2650 retval = ret;
2651 dqm_unlock(dqm);
2652 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2653 dqm_lock(dqm);
2654 }
2655
2656 /* Unregister process */
2657 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2658 if (qpd == cur->qpd) {
2659 list_del(entry: &cur->list);
2660 kfree(objp: cur);
2661 dqm->processes_count--;
2662 found = true;
2663 break;
2664 }
2665 }
2666
2667 dqm_unlock(dqm);
2668
2669 /* Outside the DQM lock because under the DQM lock we can't do
2670 * reclaim or take other locks that others hold while reclaiming.
2671 */
2672 if (found)
2673 kfd_dec_compute_active(dev: dqm->dev);
2674
2675 return retval;
2676}
2677
2678static int get_wave_state(struct device_queue_manager *dqm,
2679 struct queue *q,
2680 void __user *ctl_stack,
2681 u32 *ctl_stack_used_size,
2682 u32 *save_area_used_size)
2683{
2684 struct mqd_manager *mqd_mgr;
2685
2686 dqm_lock(dqm);
2687
2688 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
2689
2690 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
2691 q->properties.is_active || !q->device->kfd->cwsr_enabled ||
2692 !mqd_mgr->get_wave_state) {
2693 dqm_unlock(dqm);
2694 return -EINVAL;
2695 }
2696
2697 dqm_unlock(dqm);
2698
2699 /*
2700 * get_wave_state is outside the dqm lock to prevent circular locking
2701 * and the queue should be protected against destruction by the process
2702 * lock.
2703 */
2704 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,
2705 ctl_stack, ctl_stack_used_size, save_area_used_size);
2706}
2707
2708static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
2709 const struct queue *q,
2710 u32 *mqd_size,
2711 u32 *ctl_stack_size)
2712{
2713 struct mqd_manager *mqd_mgr;
2714 enum KFD_MQD_TYPE mqd_type =
2715 get_mqd_type_from_queue_type(type: q->properties.type);
2716
2717 dqm_lock(dqm);
2718 mqd_mgr = dqm->mqd_mgrs[mqd_type];
2719 *mqd_size = mqd_mgr->mqd_size;
2720 *ctl_stack_size = 0;
2721
2722 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
2723 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
2724
2725 dqm_unlock(dqm);
2726}
2727
2728static int checkpoint_mqd(struct device_queue_manager *dqm,
2729 const struct queue *q,
2730 void *mqd,
2731 void *ctl_stack)
2732{
2733 struct mqd_manager *mqd_mgr;
2734 int r = 0;
2735 enum KFD_MQD_TYPE mqd_type =
2736 get_mqd_type_from_queue_type(type: q->properties.type);
2737
2738 dqm_lock(dqm);
2739
2740 if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
2741 r = -EINVAL;
2742 goto dqm_unlock;
2743 }
2744
2745 mqd_mgr = dqm->mqd_mgrs[mqd_type];
2746 if (!mqd_mgr->checkpoint_mqd) {
2747 r = -EOPNOTSUPP;
2748 goto dqm_unlock;
2749 }
2750
2751 mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
2752
2753dqm_unlock:
2754 dqm_unlock(dqm);
2755 return r;
2756}
2757
2758static int process_termination_cpsch(struct device_queue_manager *dqm,
2759 struct qcm_process_device *qpd)
2760{
2761 int retval;
2762 struct queue *q;
2763 struct device *dev = dqm->dev->adev->dev;
2764 struct kernel_queue *kq, *kq_next;
2765 struct mqd_manager *mqd_mgr;
2766 struct device_process_node *cur, *next_dpn;
2767 enum kfd_unmap_queues_filter filter =
2768 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
2769 bool found = false;
2770
2771 retval = 0;
2772
2773 dqm_lock(dqm);
2774
2775 /* Clean all kernel queues */
2776 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
2777 list_del(entry: &kq->list);
2778 decrement_queue_count(dqm, qpd, q: kq->queue);
2779 qpd->is_debug = false;
2780 dqm->total_queue_count--;
2781 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
2782 }
2783
2784 /* Clear all user mode queues */
2785 list_for_each_entry(q, &qpd->queues_list, list) {
2786 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
2787 deallocate_sdma_queue(dqm, q);
2788 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2789 deallocate_sdma_queue(dqm, q);
2790
2791 if (q->properties.is_active) {
2792 decrement_queue_count(dqm, qpd, q);
2793
2794 if (dqm->dev->kfd->shared_resources.enable_mes) {
2795 retval = remove_queue_mes(dqm, q, qpd);
2796 if (retval)
2797 dev_err(dev, "Failed to remove queue %d\n",
2798 q->properties.queue_id);
2799 }
2800 }
2801
2802 dqm->total_queue_count--;
2803 }
2804
2805 /* Unregister process */
2806 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2807 if (qpd == cur->qpd) {
2808 list_del(entry: &cur->list);
2809 kfree(objp: cur);
2810 dqm->processes_count--;
2811 found = true;
2812 break;
2813 }
2814 }
2815
2816 if (!dqm->dev->kfd->shared_resources.enable_mes)
2817 retval = execute_queues_cpsch(dqm, filter, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
2818
2819 if ((retval || qpd->reset_wavefronts) &&
2820 down_read_trylock(sem: &dqm->dev->adev->reset_domain->sem)) {
2821 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
2822 dbgdev_wave_reset_wavefronts(dev: dqm->dev, p: qpd->pqm->process);
2823 qpd->reset_wavefronts = false;
2824 up_read(sem: &dqm->dev->adev->reset_domain->sem);
2825 }
2826
2827 /* Lastly, free mqd resources.
2828 * Do free_mqd() after dqm_unlock to avoid circular locking.
2829 */
2830 while (!list_empty(head: &qpd->queues_list)) {
2831 q = list_first_entry(&qpd->queues_list, struct queue, list);
2832 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2833 type: q->properties.type)];
2834 list_del(entry: &q->list);
2835 qpd->queue_count--;
2836 dqm_unlock(dqm);
2837 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2838 dqm_lock(dqm);
2839 }
2840 dqm_unlock(dqm);
2841
2842 /* Outside the DQM lock because under the DQM lock we can't do
2843 * reclaim or take other locks that others hold while reclaiming.
2844 */
2845 if (found)
2846 kfd_dec_compute_active(dev: dqm->dev);
2847
2848 return retval;
2849}
2850
2851static int init_mqd_managers(struct device_queue_manager *dqm)
2852{
2853 int i, j;
2854 struct device *dev = dqm->dev->adev->dev;
2855 struct mqd_manager *mqd_mgr;
2856
2857 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
2858 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
2859 if (!mqd_mgr) {
2860 dev_err(dev, "mqd manager [%d] initialization failed\n", i);
2861 goto out_free;
2862 }
2863 dqm->mqd_mgrs[i] = mqd_mgr;
2864 }
2865
2866 return 0;
2867
2868out_free:
2869 for (j = 0; j < i; j++) {
2870 kfree(objp: dqm->mqd_mgrs[j]);
2871 dqm->mqd_mgrs[j] = NULL;
2872 }
2873
2874 return -ENOMEM;
2875}
2876
2877/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
2878static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
2879{
2880 int retval;
2881 struct kfd_node *dev = dqm->dev;
2882 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
2883 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
2884 get_num_all_sdma_engines(dqm) *
2885 dev->kfd->device_info.num_sdma_queues_per_engine +
2886 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *
2887 NUM_XCC(dqm->dev->xcc_mask));
2888
2889 retval = amdgpu_amdkfd_alloc_gtt_mem(adev: dev->adev, size,
2890 mem_obj: &(mem_obj->gtt_mem), gpu_addr: &(mem_obj->gpu_addr),
2891 cpu_ptr: (void *)&(mem_obj->cpu_ptr), mqd_gfx9: false);
2892
2893 return retval;
2894}
2895
2896struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
2897{
2898 struct device_queue_manager *dqm;
2899
2900 pr_debug("Loading device queue manager\n");
2901
2902 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
2903 if (!dqm)
2904 return NULL;
2905
2906 switch (dev->adev->asic_type) {
2907 /* HWS is not available on Hawaii. */
2908 case CHIP_HAWAII:
2909 /* HWS depends on CWSR for timely dequeue. CWSR is not
2910 * available on Tonga.
2911 *
2912 * FIXME: This argument also applies to Kaveri.
2913 */
2914 case CHIP_TONGA:
2915 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
2916 break;
2917 default:
2918 dqm->sched_policy = sched_policy;
2919 break;
2920 }
2921
2922 dqm->dev = dev;
2923 switch (dqm->sched_policy) {
2924 case KFD_SCHED_POLICY_HWS:
2925 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
2926 /* initialize dqm for cp scheduling */
2927 dqm->ops.create_queue = create_queue_cpsch;
2928 dqm->ops.initialize = initialize_cpsch;
2929 dqm->ops.start = start_cpsch;
2930 dqm->ops.stop = stop_cpsch;
2931 dqm->ops.halt = halt_cpsch;
2932 dqm->ops.unhalt = unhalt_cpsch;
2933 dqm->ops.destroy_queue = destroy_queue_cpsch;
2934 dqm->ops.update_queue = update_queue;
2935 dqm->ops.register_process = register_process;
2936 dqm->ops.unregister_process = unregister_process;
2937 dqm->ops.uninitialize = uninitialize;
2938 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
2939 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
2940 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2941 dqm->ops.process_termination = process_termination_cpsch;
2942 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
2943 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
2944 dqm->ops.get_wave_state = get_wave_state;
2945 dqm->ops.reset_queues = reset_queues_cpsch;
2946 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2947 dqm->ops.checkpoint_mqd = checkpoint_mqd;
2948 break;
2949 case KFD_SCHED_POLICY_NO_HWS:
2950 /* initialize dqm for no cp scheduling */
2951 dqm->ops.start = start_nocpsch;
2952 dqm->ops.stop = stop_nocpsch;
2953 dqm->ops.create_queue = create_queue_nocpsch;
2954 dqm->ops.destroy_queue = destroy_queue_nocpsch;
2955 dqm->ops.update_queue = update_queue;
2956 dqm->ops.register_process = register_process;
2957 dqm->ops.unregister_process = unregister_process;
2958 dqm->ops.initialize = initialize_nocpsch;
2959 dqm->ops.uninitialize = uninitialize;
2960 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2961 dqm->ops.process_termination = process_termination_nocpsch;
2962 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
2963 dqm->ops.restore_process_queues =
2964 restore_process_queues_nocpsch;
2965 dqm->ops.get_wave_state = get_wave_state;
2966 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2967 dqm->ops.checkpoint_mqd = checkpoint_mqd;
2968 break;
2969 default:
2970 dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
2971 goto out_free;
2972 }
2973
2974 switch (dev->adev->asic_type) {
2975 case CHIP_KAVERI:
2976 case CHIP_HAWAII:
2977 device_queue_manager_init_cik(asic_ops: &dqm->asic_ops);
2978 break;
2979
2980 case CHIP_CARRIZO:
2981 case CHIP_TONGA:
2982 case CHIP_FIJI:
2983 case CHIP_POLARIS10:
2984 case CHIP_POLARIS11:
2985 case CHIP_POLARIS12:
2986 case CHIP_VEGAM:
2987 device_queue_manager_init_vi(asic_ops: &dqm->asic_ops);
2988 break;
2989
2990 default:
2991 if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
2992 device_queue_manager_init_v12(asic_ops: &dqm->asic_ops);
2993 else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
2994 device_queue_manager_init_v11(asic_ops: &dqm->asic_ops);
2995 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
2996 device_queue_manager_init_v10(asic_ops: &dqm->asic_ops);
2997 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
2998 device_queue_manager_init_v9(asic_ops: &dqm->asic_ops);
2999 else {
3000 WARN(1, "Unexpected ASIC family %u",
3001 dev->adev->asic_type);
3002 goto out_free;
3003 }
3004 }
3005
3006 if (init_mqd_managers(dqm))
3007 goto out_free;
3008
3009 if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
3010 dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n");
3011 goto out_free;
3012 }
3013
3014 if (!dqm->ops.initialize(dqm)) {
3015 init_waitqueue_head(&dqm->destroy_wait);
3016 return dqm;
3017 }
3018
3019out_free:
3020 kfree(objp: dqm);
3021 return NULL;
3022}
3023
3024static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
3025 struct kfd_mem_obj *mqd)
3026{
3027 WARN(!mqd, "No hiq sdma mqd trunk to free");
3028
3029 amdgpu_amdkfd_free_gtt_mem(adev: dev->adev, mem_obj: &mqd->gtt_mem);
3030}
3031
3032void device_queue_manager_uninit(struct device_queue_manager *dqm)
3033{
3034 dqm->ops.stop(dqm);
3035 dqm->ops.uninitialize(dqm);
3036 if (!dqm->dev->kfd->shared_resources.enable_mes)
3037 deallocate_hiq_sdma_mqd(dev: dqm->dev, mqd: &dqm->hiq_sdma_mqd);
3038 kfree(objp: dqm);
3039}
3040
3041int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)
3042{
3043 struct kfd_process_device *pdd = NULL;
3044 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, pdd: &pdd);
3045 struct device_queue_manager *dqm = knode->dqm;
3046 struct device *dev = dqm->dev->adev->dev;
3047 struct qcm_process_device *qpd;
3048 struct queue *q = NULL;
3049 int ret = 0;
3050
3051 if (!pdd)
3052 return -EINVAL;
3053
3054 dqm_lock(dqm);
3055
3056 if (pdd) {
3057 qpd = &pdd->qpd;
3058
3059 list_for_each_entry(q, &qpd->queues_list, list) {
3060 if (q->doorbell_id == doorbell_id && q->properties.is_active) {
3061 ret = suspend_all_queues_mes(dqm);
3062 if (ret) {
3063 dev_err(dev, "Suspending all queues failed");
3064 goto out;
3065 }
3066
3067 q->properties.is_evicted = true;
3068 q->properties.is_active = false;
3069 decrement_queue_count(dqm, qpd, q);
3070
3071 ret = remove_queue_mes(dqm, q, qpd);
3072 if (ret) {
3073 dev_err(dev, "Removing bad queue failed");
3074 goto out;
3075 }
3076
3077 ret = resume_all_queues_mes(dqm);
3078 if (ret)
3079 dev_err(dev, "Resuming all queues failed");
3080
3081 break;
3082 }
3083 }
3084 }
3085
3086out:
3087 dqm_unlock(dqm);
3088 kfd_unref_process(p);
3089 return ret;
3090}
3091
3092static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm,
3093 struct qcm_process_device *qpd)
3094{
3095 struct device *dev = dqm->dev->adev->dev;
3096 int ret = 0;
3097
3098 /* Check if process is already evicted */
3099 dqm_lock(dqm);
3100 if (qpd->evicted) {
3101 /* Increment the evicted count to make sure the
3102 * process stays evicted before its terminated.
3103 */
3104 qpd->evicted++;
3105 dqm_unlock(dqm);
3106 goto out;
3107 }
3108 dqm_unlock(dqm);
3109
3110 ret = suspend_all_queues_mes(dqm);
3111 if (ret) {
3112 dev_err(dev, "Suspending all queues failed");
3113 goto out;
3114 }
3115
3116 ret = dqm->ops.evict_process_queues(dqm, qpd);
3117 if (ret) {
3118 dev_err(dev, "Evicting process queues failed");
3119 goto out;
3120 }
3121
3122 ret = resume_all_queues_mes(dqm);
3123 if (ret)
3124 dev_err(dev, "Resuming all queues failed");
3125
3126out:
3127 return ret;
3128}
3129
3130int kfd_evict_process_device(struct kfd_process_device *pdd)
3131{
3132 struct device_queue_manager *dqm;
3133 struct kfd_process *p;
3134 int ret = 0;
3135
3136 p = pdd->process;
3137 dqm = pdd->dev->dqm;
3138
3139 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
3140
3141 if (dqm->dev->kfd->shared_resources.enable_mes)
3142 ret = kfd_dqm_evict_pasid_mes(dqm, qpd: &pdd->qpd);
3143 else
3144 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
3145
3146 return ret;
3147}
3148
3149int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
3150 struct qcm_process_device *qpd)
3151{
3152 int r;
3153 struct device *dev = dqm->dev->adev->dev;
3154 int updated_vmid_mask;
3155
3156 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3157 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3158 return -EINVAL;
3159 }
3160
3161 dqm_lock(dqm);
3162
3163 if (dqm->trap_debug_vmid != 0) {
3164 dev_err(dev, "Trap debug id already reserved\n");
3165 r = -EBUSY;
3166 goto out_unlock;
3167 }
3168
3169 r = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
3170 USE_DEFAULT_GRACE_PERIOD, reset: false);
3171 if (r)
3172 goto out_unlock;
3173
3174 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3175 updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);
3176
3177 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3178 dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;
3179 r = set_sched_resources(dqm);
3180 if (r)
3181 goto out_unlock;
3182
3183 r = map_queues_cpsch(dqm);
3184 if (r)
3185 goto out_unlock;
3186
3187 pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);
3188
3189out_unlock:
3190 dqm_unlock(dqm);
3191 return r;
3192}
3193
3194/*
3195 * Releases vmid for the trap debugger
3196 */
3197int release_debug_trap_vmid(struct device_queue_manager *dqm,
3198 struct qcm_process_device *qpd)
3199{
3200 struct device *dev = dqm->dev->adev->dev;
3201 int r;
3202 int updated_vmid_mask;
3203 uint32_t trap_debug_vmid;
3204
3205 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3206 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3207 return -EINVAL;
3208 }
3209
3210 dqm_lock(dqm);
3211 trap_debug_vmid = dqm->trap_debug_vmid;
3212 if (dqm->trap_debug_vmid == 0) {
3213 dev_err(dev, "Trap debug id is not reserved\n");
3214 r = -EINVAL;
3215 goto out_unlock;
3216 }
3217
3218 r = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
3219 USE_DEFAULT_GRACE_PERIOD, reset: false);
3220 if (r)
3221 goto out_unlock;
3222
3223 updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3224 updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);
3225
3226 dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3227 dqm->trap_debug_vmid = 0;
3228 r = set_sched_resources(dqm);
3229 if (r)
3230 goto out_unlock;
3231
3232 r = map_queues_cpsch(dqm);
3233 if (r)
3234 goto out_unlock;
3235
3236 pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);
3237
3238out_unlock:
3239 dqm_unlock(dqm);
3240 return r;
3241}
3242
3243#define QUEUE_NOT_FOUND -1
3244/* invalidate queue operation in array */
3245static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)
3246{
3247 int i;
3248
3249 for (i = 0; i < num_queues; i++)
3250 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;
3251}
3252
3253/* find queue index in array */
3254static int q_array_get_index(unsigned int queue_id,
3255 uint32_t num_queues,
3256 uint32_t *queue_ids)
3257{
3258 int i;
3259
3260 for (i = 0; i < num_queues; i++)
3261 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))
3262 return i;
3263
3264 return QUEUE_NOT_FOUND;
3265}
3266
3267struct copy_context_work_handler_workarea {
3268 struct work_struct copy_context_work;
3269 struct kfd_process *p;
3270};
3271
3272static void copy_context_work_handler(struct work_struct *work)
3273{
3274 struct copy_context_work_handler_workarea *workarea;
3275 struct mqd_manager *mqd_mgr;
3276 struct queue *q;
3277 struct mm_struct *mm;
3278 struct kfd_process *p;
3279 uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;
3280 int i;
3281
3282 workarea = container_of(work,
3283 struct copy_context_work_handler_workarea,
3284 copy_context_work);
3285
3286 p = workarea->p;
3287 mm = get_task_mm(task: p->lead_thread);
3288
3289 if (!mm)
3290 return;
3291
3292 kthread_use_mm(mm);
3293 for (i = 0; i < p->n_pdds; i++) {
3294 struct kfd_process_device *pdd = p->pdds[i];
3295 struct device_queue_manager *dqm = pdd->dev->dqm;
3296 struct qcm_process_device *qpd = &pdd->qpd;
3297
3298 list_for_each_entry(q, &qpd->queues_list, list) {
3299 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
3300 continue;
3301
3302 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
3303
3304 /* We ignore the return value from get_wave_state
3305 * because
3306 * i) right now, it always returns 0, and
3307 * ii) if we hit an error, we would continue to the
3308 * next queue anyway.
3309 */
3310 mqd_mgr->get_wave_state(mqd_mgr,
3311 q->mqd,
3312 &q->properties,
3313 (void __user *) q->properties.ctx_save_restore_area_address,
3314 &tmp_ctl_stack_used_size,
3315 &tmp_save_area_used_size);
3316 }
3317 }
3318 kthread_unuse_mm(mm);
3319 mmput(mm);
3320}
3321
3322static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
3323{
3324 size_t array_size = num_queues * sizeof(uint32_t);
3325
3326 if (!usr_queue_id_array)
3327 return NULL;
3328
3329 return memdup_user(usr_queue_id_array, array_size);
3330}
3331
3332int resume_queues(struct kfd_process *p,
3333 uint32_t num_queues,
3334 uint32_t *usr_queue_id_array)
3335{
3336 uint32_t *queue_ids = NULL;
3337 int total_resumed = 0;
3338 int i;
3339
3340 if (usr_queue_id_array) {
3341 queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3342
3343 if (IS_ERR(ptr: queue_ids))
3344 return PTR_ERR(ptr: queue_ids);
3345
3346 /* mask all queues as invalid. unmask per successful request */
3347 q_array_invalidate(num_queues, queue_ids);
3348 }
3349
3350 for (i = 0; i < p->n_pdds; i++) {
3351 struct kfd_process_device *pdd = p->pdds[i];
3352 struct device_queue_manager *dqm = pdd->dev->dqm;
3353 struct device *dev = dqm->dev->adev->dev;
3354 struct qcm_process_device *qpd = &pdd->qpd;
3355 struct queue *q;
3356 int r, per_device_resumed = 0;
3357
3358 dqm_lock(dqm);
3359
3360 /* unmask queues that resume or already resumed as valid */
3361 list_for_each_entry(q, &qpd->queues_list, list) {
3362 int q_idx = QUEUE_NOT_FOUND;
3363
3364 if (queue_ids)
3365 q_idx = q_array_get_index(
3366 queue_id: q->properties.queue_id,
3367 num_queues,
3368 queue_ids);
3369
3370 if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {
3371 int err = resume_single_queue(dqm, qpd: &pdd->qpd, q);
3372
3373 if (queue_ids) {
3374 if (!err) {
3375 queue_ids[q_idx] &=
3376 ~KFD_DBG_QUEUE_INVALID_MASK;
3377 } else {
3378 queue_ids[q_idx] |=
3379 KFD_DBG_QUEUE_ERROR_MASK;
3380 break;
3381 }
3382 }
3383
3384 if (dqm->dev->kfd->shared_resources.enable_mes) {
3385 wake_up_all(&dqm->destroy_wait);
3386 if (!err)
3387 total_resumed++;
3388 } else {
3389 per_device_resumed++;
3390 }
3391 }
3392 }
3393
3394 if (!per_device_resumed) {
3395 dqm_unlock(dqm);
3396 continue;
3397 }
3398
3399 r = execute_queues_cpsch(dqm,
3400 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
3401 filter_param: 0,
3402 USE_DEFAULT_GRACE_PERIOD);
3403 if (r) {
3404 dev_err(dev, "Failed to resume process queues\n");
3405 if (queue_ids) {
3406 list_for_each_entry(q, &qpd->queues_list, list) {
3407 int q_idx = q_array_get_index(
3408 queue_id: q->properties.queue_id,
3409 num_queues,
3410 queue_ids);
3411
3412 /* mask queue as error on resume fail */
3413 if (q_idx != QUEUE_NOT_FOUND)
3414 queue_ids[q_idx] |=
3415 KFD_DBG_QUEUE_ERROR_MASK;
3416 }
3417 }
3418 } else {
3419 wake_up_all(&dqm->destroy_wait);
3420 total_resumed += per_device_resumed;
3421 }
3422
3423 dqm_unlock(dqm);
3424 }
3425
3426 if (queue_ids) {
3427 if (copy_to_user(to: (void __user *)usr_queue_id_array, from: queue_ids,
3428 n: num_queues * sizeof(uint32_t)))
3429 pr_err("copy_to_user failed on queue resume\n");
3430
3431 kfree(objp: queue_ids);
3432 }
3433
3434 return total_resumed;
3435}
3436
3437int suspend_queues(struct kfd_process *p,
3438 uint32_t num_queues,
3439 uint32_t grace_period,
3440 uint64_t exception_clear_mask,
3441 uint32_t *usr_queue_id_array)
3442{
3443 uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3444 int total_suspended = 0;
3445 int i;
3446
3447 if (IS_ERR(ptr: queue_ids))
3448 return PTR_ERR(ptr: queue_ids);
3449
3450 /* mask all queues as invalid. umask on successful request */
3451 q_array_invalidate(num_queues, queue_ids);
3452
3453 for (i = 0; i < p->n_pdds; i++) {
3454 struct kfd_process_device *pdd = p->pdds[i];
3455 struct device_queue_manager *dqm = pdd->dev->dqm;
3456 struct device *dev = dqm->dev->adev->dev;
3457 struct qcm_process_device *qpd = &pdd->qpd;
3458 struct queue *q;
3459 int r, per_device_suspended = 0;
3460
3461 mutex_lock(&p->event_mutex);
3462 dqm_lock(dqm);
3463
3464 /* unmask queues that suspend or already suspended */
3465 list_for_each_entry(q, &qpd->queues_list, list) {
3466 int q_idx = q_array_get_index(queue_id: q->properties.queue_id,
3467 num_queues,
3468 queue_ids);
3469
3470 if (q_idx != QUEUE_NOT_FOUND) {
3471 int err = suspend_single_queue(dqm, pdd, q);
3472 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;
3473
3474 if (!err) {
3475 queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;
3476 if (exception_clear_mask && is_mes)
3477 q->properties.exception_status &=
3478 ~exception_clear_mask;
3479
3480 if (is_mes)
3481 total_suspended++;
3482 else
3483 per_device_suspended++;
3484 } else if (err != -EBUSY) {
3485 r = err;
3486 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3487 break;
3488 }
3489 }
3490 }
3491
3492 if (!per_device_suspended) {
3493 dqm_unlock(dqm);
3494 mutex_unlock(lock: &p->event_mutex);
3495 if (total_suspended)
3496 amdgpu_amdkfd_debug_mem_fence(adev: dqm->dev->adev);
3497 continue;
3498 }
3499
3500 r = execute_queues_cpsch(dqm,
3501 filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
3502 grace_period);
3503
3504 if (r)
3505 dev_err(dev, "Failed to suspend process queues.\n");
3506 else
3507 total_suspended += per_device_suspended;
3508
3509 list_for_each_entry(q, &qpd->queues_list, list) {
3510 int q_idx = q_array_get_index(queue_id: q->properties.queue_id,
3511 num_queues, queue_ids);
3512
3513 if (q_idx == QUEUE_NOT_FOUND)
3514 continue;
3515
3516 /* mask queue as error on suspend fail */
3517 if (r)
3518 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3519 else if (exception_clear_mask)
3520 q->properties.exception_status &=
3521 ~exception_clear_mask;
3522 }
3523
3524 dqm_unlock(dqm);
3525 mutex_unlock(lock: &p->event_mutex);
3526 amdgpu_device_flush_hdp(adev: dqm->dev->adev, NULL);
3527 }
3528
3529 if (total_suspended) {
3530 struct copy_context_work_handler_workarea copy_context_worker;
3531
3532 INIT_WORK_ONSTACK(
3533 &copy_context_worker.copy_context_work,
3534 copy_context_work_handler);
3535
3536 copy_context_worker.p = p;
3537
3538 schedule_work(work: &copy_context_worker.copy_context_work);
3539
3540
3541 flush_work(work: &copy_context_worker.copy_context_work);
3542 destroy_work_on_stack(work: &copy_context_worker.copy_context_work);
3543 }
3544
3545 if (copy_to_user(to: (void __user *)usr_queue_id_array, from: queue_ids,
3546 n: num_queues * sizeof(uint32_t)))
3547 pr_err("copy_to_user failed on queue suspend\n");
3548
3549 kfree(objp: queue_ids);
3550
3551 return total_suspended;
3552}
3553
3554static uint32_t set_queue_type_for_user(struct queue_properties *q_props)
3555{
3556 switch (q_props->type) {
3557 case KFD_QUEUE_TYPE_COMPUTE:
3558 return q_props->format == KFD_QUEUE_FORMAT_PM4
3559 ? KFD_IOC_QUEUE_TYPE_COMPUTE
3560 : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;
3561 case KFD_QUEUE_TYPE_SDMA:
3562 return KFD_IOC_QUEUE_TYPE_SDMA;
3563 case KFD_QUEUE_TYPE_SDMA_XGMI:
3564 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;
3565 default:
3566 WARN_ONCE(true, "queue type not recognized!");
3567 return 0xffffffff;
3568 };
3569}
3570
3571void set_queue_snapshot_entry(struct queue *q,
3572 uint64_t exception_clear_mask,
3573 struct kfd_queue_snapshot_entry *qss_entry)
3574{
3575 qss_entry->ring_base_address = q->properties.queue_address;
3576 qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;
3577 qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;
3578 qss_entry->ctx_save_restore_address =
3579 q->properties.ctx_save_restore_area_address;
3580 qss_entry->ctx_save_restore_area_size =
3581 q->properties.ctx_save_restore_area_size;
3582 qss_entry->exception_status = q->properties.exception_status;
3583 qss_entry->queue_id = q->properties.queue_id;
3584 qss_entry->gpu_id = q->device->id;
3585 qss_entry->ring_size = (uint32_t)q->properties.queue_size;
3586 qss_entry->queue_type = set_queue_type_for_user(&q->properties);
3587 q->properties.exception_status &= ~exception_clear_mask;
3588}
3589
3590int debug_lock_and_unmap(struct device_queue_manager *dqm)
3591{
3592 struct device *dev = dqm->dev->adev->dev;
3593 int r;
3594
3595 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3596 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3597 return -EINVAL;
3598 }
3599
3600 if (!kfd_dbg_is_per_vmid_supported(dev: dqm->dev))
3601 return 0;
3602
3603 dqm_lock(dqm);
3604
3605 r = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0, grace_period: 0, reset: false);
3606 if (r)
3607 dqm_unlock(dqm);
3608
3609 return r;
3610}
3611
3612int debug_map_and_unlock(struct device_queue_manager *dqm)
3613{
3614 struct device *dev = dqm->dev->adev->dev;
3615 int r;
3616
3617 if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3618 dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3619 return -EINVAL;
3620 }
3621
3622 if (!kfd_dbg_is_per_vmid_supported(dev: dqm->dev))
3623 return 0;
3624
3625 r = map_queues_cpsch(dqm);
3626
3627 dqm_unlock(dqm);
3628
3629 return r;
3630}
3631
3632int debug_refresh_runlist(struct device_queue_manager *dqm)
3633{
3634 int r = debug_lock_and_unmap(dqm);
3635
3636 if (r)
3637 return r;
3638
3639 return debug_map_and_unlock(dqm);
3640}
3641
3642bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
3643 struct qcm_process_device *qpd,
3644 int doorbell_off, u32 *queue_format)
3645{
3646 struct queue *q;
3647 bool r = false;
3648
3649 if (!queue_format)
3650 return r;
3651
3652 dqm_lock(dqm);
3653
3654 list_for_each_entry(q, &qpd->queues_list, list) {
3655 if (q->properties.doorbell_off == doorbell_off) {
3656 *queue_format = q->properties.format;
3657 r = true;
3658 goto out;
3659 }
3660 }
3661
3662out:
3663 dqm_unlock(dqm);
3664 return r;
3665}
3666#if defined(CONFIG_DEBUG_FS)
3667
3668static void seq_reg_dump(struct seq_file *m,
3669 uint32_t (*dump)[2], uint32_t n_regs)
3670{
3671 uint32_t i, count;
3672
3673 for (i = 0, count = 0; i < n_regs; i++) {
3674 if (count == 0 ||
3675 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
3676 seq_printf(m, fmt: "%s %08x: %08x",
3677 i ? "\n" : "",
3678 dump[i][0], dump[i][1]);
3679 count = 7;
3680 } else {
3681 seq_printf(m, fmt: " %08x", dump[i][1]);
3682 count--;
3683 }
3684 }
3685
3686 seq_puts(m, s: "\n");
3687}
3688
3689int dqm_debugfs_hqds(struct seq_file *m, void *data)
3690{
3691 struct device_queue_manager *dqm = data;
3692 uint32_t xcc_mask = dqm->dev->xcc_mask;
3693 uint32_t (*dump)[2], n_regs;
3694 int pipe, queue;
3695 int r = 0, xcc_id;
3696 uint32_t sdma_engine_start;
3697
3698 if (!dqm->sched_running) {
3699 seq_puts(m, s: " Device is stopped\n");
3700 return 0;
3701 }
3702
3703 for_each_inst(xcc_id, xcc_mask) {
3704 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3705 KFD_CIK_HIQ_PIPE,
3706 KFD_CIK_HIQ_QUEUE, &dump,
3707 &n_regs, xcc_id);
3708 if (!r) {
3709 seq_printf(
3710 m,
3711 fmt: " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",
3712 xcc_id,
3713 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,
3714 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),
3715 KFD_CIK_HIQ_QUEUE);
3716 seq_reg_dump(m, dump, n_regs);
3717
3718 kfree(objp: dump);
3719 }
3720
3721 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
3722 int pipe_offset = pipe * get_queues_per_pipe(dqm);
3723
3724 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
3725 if (!test_bit(pipe_offset + queue,
3726 dqm->dev->kfd->shared_resources.cp_queue_bitmap))
3727 continue;
3728
3729 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3730 pipe, queue,
3731 &dump, &n_regs,
3732 xcc_id);
3733 if (r)
3734 break;
3735
3736 seq_printf(m,
3737 fmt: " Inst %d, CP Pipe %d, Queue %d\n",
3738 xcc_id, pipe, queue);
3739 seq_reg_dump(m, dump, n_regs);
3740
3741 kfree(objp: dump);
3742 }
3743 }
3744 }
3745
3746 sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
3747 for (pipe = sdma_engine_start;
3748 pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));
3749 pipe++) {
3750 for (queue = 0;
3751 queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
3752 queue++) {
3753 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
3754 dqm->dev->adev, pipe, queue, &dump, &n_regs);
3755 if (r)
3756 break;
3757
3758 seq_printf(m, fmt: " SDMA Engine %d, RLC %d\n",
3759 pipe, queue);
3760 seq_reg_dump(m, dump, n_regs);
3761
3762 kfree(objp: dump);
3763 }
3764 }
3765
3766 return r;
3767}
3768
3769int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
3770{
3771 int r = 0;
3772
3773 dqm_lock(dqm);
3774 r = pm_debugfs_hang_hws(pm: &dqm->packet_mgr);
3775 if (r) {
3776 dqm_unlock(dqm);
3777 return r;
3778 }
3779 dqm->active_runlist = true;
3780 r = execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
3781 filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
3782 dqm_unlock(dqm);
3783
3784 return r;
3785}
3786
3787#endif
3788

source code of linux/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c