1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24#ifndef KFD_PRIV_H_INCLUDED
25#define KFD_PRIV_H_INCLUDED
26
27#include <linux/hashtable.h>
28#include <linux/mmu_notifier.h>
29#include <linux/memremap.h>
30#include <linux/mutex.h>
31#include <linux/types.h>
32#include <linux/atomic.h>
33#include <linux/workqueue.h>
34#include <linux/spinlock.h>
35#include <linux/kfd_ioctl.h>
36#include <linux/idr.h>
37#include <linux/kfifo.h>
38#include <linux/seq_file.h>
39#include <linux/kref.h>
40#include <linux/sysfs.h>
41#include <linux/device_cgroup.h>
42#include <drm/drm_file.h>
43#include <drm/drm_drv.h>
44#include <drm/drm_device.h>
45#include <drm/drm_ioctl.h>
46#include <kgd_kfd_interface.h>
47#include <linux/swap.h>
48
49#include "amd_shared.h"
50#include "amdgpu.h"
51
52#define KFD_MAX_RING_ENTRY_SIZE 8
53
54#define KFD_SYSFS_FILE_MODE 0444
55
56/* GPU ID hash width in bits */
57#define KFD_GPU_ID_HASH_WIDTH 16
58
59/* Use upper bits of mmap offset to store KFD driver specific information.
60 * BITS[63:62] - Encode MMAP type
61 * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
62 * BITS[45:0] - MMAP offset value
63 *
64 * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
65 * defines are w.r.t to PAGE_SIZE
66 */
67#define KFD_MMAP_TYPE_SHIFT 62
68#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT)
69#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
70#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
71#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT)
72#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT)
73
74#define KFD_MMAP_GPU_ID_SHIFT 46
75#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
76 << KFD_MMAP_GPU_ID_SHIFT)
77#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
78 & KFD_MMAP_GPU_ID_MASK)
79#define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \
80 >> KFD_MMAP_GPU_ID_SHIFT)
81
82/*
83 * When working with cp scheduler we should assign the HIQ manually or via
84 * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
85 * definitions for Kaveri. In Kaveri only the first ME queues participates
86 * in the cp scheduling taking that in mind we set the HIQ slot in the
87 * second ME.
88 */
89#define KFD_CIK_HIQ_PIPE 4
90#define KFD_CIK_HIQ_QUEUE 0
91
92/* Macro for allocating structures */
93#define kfd_alloc_struct(ptr_to_struct) \
94 ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
95
96#define KFD_MAX_NUM_OF_PROCESSES 512
97#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
98
99/*
100 * Size of the per-process TBA+TMA buffer: 2 pages
101 *
102 * The first page is the TBA used for the CWSR ISA code. The second
103 * page is used as TMA for user-mode trap handler setup in daisy-chain mode.
104 */
105#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
106#define KFD_CWSR_TMA_OFFSET PAGE_SIZE
107
108#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
109 (KFD_MAX_NUM_OF_PROCESSES * \
110 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
111
112#define KFD_KERNEL_QUEUE_SIZE 2048
113
114#define KFD_UNMAP_LATENCY_MS (4000)
115
116/*
117 * 512 = 0x200
118 * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the
119 * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA.
120 * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC
121 * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in
122 * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE.
123 */
124#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512
125
126/**
127 * enum kfd_ioctl_flags - KFD ioctl flags
128 * Various flags that can be set in &amdkfd_ioctl_desc.flags to control how
129 * userspace can use a given ioctl.
130 */
131enum kfd_ioctl_flags {
132 /*
133 * @KFD_IOC_FLAG_CHECKPOINT_RESTORE:
134 * Certain KFD ioctls such as AMDKFD_IOC_CRIU_OP can potentially
135 * perform privileged operations and load arbitrary data into MQDs and
136 * eventually HQD registers when the queue is mapped by HWS. In order to
137 * prevent this we should perform additional security checks.
138 *
139 * This is equivalent to callers with the CHECKPOINT_RESTORE capability.
140 *
141 * Note: Since earlier versions of docker do not support CHECKPOINT_RESTORE,
142 * we also allow ioctls with SYS_ADMIN capability.
143 */
144 KFD_IOC_FLAG_CHECKPOINT_RESTORE = BIT(0),
145};
146/*
147 * Kernel module parameter to specify maximum number of supported queues per
148 * device
149 */
150extern int max_num_of_queues_per_device;
151
152
153/* Kernel module parameter to specify the scheduling policy */
154extern int sched_policy;
155
156/*
157 * Kernel module parameter to specify the maximum process
158 * number per HW scheduler
159 */
160extern int hws_max_conc_proc;
161
162extern int cwsr_enable;
163
164/*
165 * Kernel module parameter to specify whether to send sigterm to HSA process on
166 * unhandled exception
167 */
168extern int send_sigterm;
169
170/*
171 * This kernel module is used to simulate large bar machine on non-large bar
172 * enabled machines.
173 */
174extern int debug_largebar;
175
176/*
177 * Ignore CRAT table during KFD initialization, can be used to work around
178 * broken CRAT tables on some AMD systems
179 */
180extern int ignore_crat;
181
182/* Set sh_mem_config.retry_disable on GFX v9 */
183extern int amdgpu_noretry;
184
185/* Halt if HWS hang is detected */
186extern int halt_if_hws_hang;
187
188/* Whether MEC FW support GWS barriers */
189extern bool hws_gws_support;
190
191/* Queue preemption timeout in ms */
192extern int queue_preemption_timeout_ms;
193
194/*
195 * Don't evict process queues on vm fault
196 */
197extern int amdgpu_no_queue_eviction_on_vm_fault;
198
199/* Enable eviction debug messages */
200extern bool debug_evictions;
201
202enum cache_policy {
203 cache_policy_coherent,
204 cache_policy_noncoherent
205};
206
207#define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])
208#define KFD_IS_SOC15(dev) ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
209#define KFD_SUPPORT_XNACK_PER_PROCESS(dev)\
210 ((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) || \
211 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)))
212
213struct kfd_event_interrupt_class {
214 bool (*interrupt_isr)(struct kfd_dev *dev,
215 const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
216 bool *patched_flag);
217 void (*interrupt_wq)(struct kfd_dev *dev,
218 const uint32_t *ih_ring_entry);
219};
220
221struct kfd_device_info {
222 uint32_t gfx_target_version;
223 const struct kfd_event_interrupt_class *event_interrupt_class;
224 unsigned int max_pasid_bits;
225 unsigned int max_no_of_hqd;
226 unsigned int doorbell_size;
227 size_t ih_ring_entry_size;
228 uint8_t num_of_watch_points;
229 uint16_t mqd_size_aligned;
230 bool supports_cwsr;
231 bool needs_iommu_device;
232 bool needs_pci_atomics;
233 uint32_t no_atomic_fw_version;
234 unsigned int num_sdma_queues_per_engine;
235 unsigned int num_reserved_sdma_queues_per_engine;
236 uint64_t reserved_sdma_queues_bitmap;
237};
238
239unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev);
240unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev);
241
242struct kfd_mem_obj {
243 uint32_t range_start;
244 uint32_t range_end;
245 uint64_t gpu_addr;
246 uint32_t *cpu_ptr;
247 void *gtt_mem;
248};
249
250struct kfd_vmid_info {
251 uint32_t first_vmid_kfd;
252 uint32_t last_vmid_kfd;
253 uint32_t vmid_num_kfd;
254};
255
256struct kfd_dev {
257 struct amdgpu_device *adev;
258
259 struct kfd_device_info device_info;
260
261 unsigned int id; /* topology stub index */
262
263 phys_addr_t doorbell_base; /* Start of actual doorbells used by
264 * KFD. It is aligned for mapping
265 * into user mode
266 */
267 size_t doorbell_base_dw_offset; /* Offset from the start of the PCI
268 * doorbell BAR to the first KFD
269 * doorbell in dwords. GFX reserves
270 * the segment before this offset.
271 */
272 u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
273 * page used by kernel queue
274 */
275
276 struct kgd2kfd_shared_resources shared_resources;
277 struct kfd_vmid_info vm_info;
278 struct kfd_local_mem_info local_mem_info;
279
280 const struct kfd2kgd_calls *kfd2kgd;
281 struct mutex doorbell_mutex;
282 DECLARE_BITMAP(doorbell_available_index,
283 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
284
285 void *gtt_mem;
286 uint64_t gtt_start_gpu_addr;
287 void *gtt_start_cpu_ptr;
288 void *gtt_sa_bitmap;
289 struct mutex gtt_sa_lock;
290 unsigned int gtt_sa_chunk_size;
291 unsigned int gtt_sa_num_of_chunks;
292
293 /* Interrupts */
294 struct kfifo ih_fifo;
295 struct workqueue_struct *ih_wq;
296 struct work_struct interrupt_work;
297 spinlock_t interrupt_lock;
298
299 /* QCM Device instance */
300 struct device_queue_manager *dqm;
301
302 bool init_complete;
303 /*
304 * Interrupts of interest to KFD are copied
305 * from the HW ring into a SW ring.
306 */
307 bool interrupts_active;
308
309 /* Firmware versions */
310 uint16_t mec_fw_version;
311 uint16_t mec2_fw_version;
312 uint16_t sdma_fw_version;
313
314 /* Maximum process number mapped to HW scheduler */
315 unsigned int max_proc_per_quantum;
316
317 /* CWSR */
318 bool cwsr_enabled;
319 const void *cwsr_isa;
320 unsigned int cwsr_isa_size;
321
322 /* xGMI */
323 uint64_t hive_id;
324
325 bool pci_atomic_requested;
326
327 /* Use IOMMU v2 flag */
328 bool use_iommu_v2;
329
330 /* SRAM ECC flag */
331 atomic_t sram_ecc_flag;
332
333 /* Compute Profile ref. count */
334 atomic_t compute_profile;
335
336 /* Global GWS resource shared between processes */
337 void *gws;
338
339 /* Clients watching SMI events */
340 struct list_head smi_clients;
341 spinlock_t smi_lock;
342
343 uint32_t reset_seq_num;
344
345 struct ida doorbell_ida;
346 unsigned int max_doorbell_slices;
347
348 int noretry;
349
350 /* HMM page migration MEMORY_DEVICE_PRIVATE mapping */
351 struct dev_pagemap pgmap;
352};
353
354enum kfd_mempool {
355 KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
356 KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
357 KFD_MEMPOOL_FRAMEBUFFER = 3,
358};
359
360/* Character device interface */
361int kfd_chardev_init(void);
362void kfd_chardev_exit(void);
363
364/**
365 * enum kfd_unmap_queues_filter - Enum for queue filters.
366 *
367 * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
368 * running queues list.
369 *
370 * @KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES: Preempts all non-static queues
371 * in the run list.
372 *
373 * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
374 * specific process.
375 *
376 */
377enum kfd_unmap_queues_filter {
378 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES = 1,
379 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES = 2,
380 KFD_UNMAP_QUEUES_FILTER_BY_PASID = 3
381};
382
383/**
384 * enum kfd_queue_type - Enum for various queue types.
385 *
386 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
387 *
388 * @KFD_QUEUE_TYPE_SDMA: SDMA user mode queue type.
389 *
390 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
391 *
392 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
393 *
394 * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface.
395 */
396enum kfd_queue_type {
397 KFD_QUEUE_TYPE_COMPUTE,
398 KFD_QUEUE_TYPE_SDMA,
399 KFD_QUEUE_TYPE_HIQ,
400 KFD_QUEUE_TYPE_DIQ,
401 KFD_QUEUE_TYPE_SDMA_XGMI
402};
403
404enum kfd_queue_format {
405 KFD_QUEUE_FORMAT_PM4,
406 KFD_QUEUE_FORMAT_AQL
407};
408
409enum KFD_QUEUE_PRIORITY {
410 KFD_QUEUE_PRIORITY_MINIMUM = 0,
411 KFD_QUEUE_PRIORITY_MAXIMUM = 15
412};
413
414/**
415 * struct queue_properties
416 *
417 * @type: The queue type.
418 *
419 * @queue_id: Queue identifier.
420 *
421 * @queue_address: Queue ring buffer address.
422 *
423 * @queue_size: Queue ring buffer size.
424 *
425 * @priority: Defines the queue priority relative to other queues in the
426 * process.
427 * This is just an indication and HW scheduling may override the priority as
428 * necessary while keeping the relative prioritization.
429 * the priority granularity is from 0 to f which f is the highest priority.
430 * currently all queues are initialized with the highest priority.
431 *
432 * @queue_percent: This field is partially implemented and currently a zero in
433 * this field defines that the queue is non active.
434 *
435 * @read_ptr: User space address which points to the number of dwords the
436 * cp read from the ring buffer. This field updates automatically by the H/W.
437 *
438 * @write_ptr: Defines the number of dwords written to the ring buffer.
439 *
440 * @doorbell_ptr: Notifies the H/W of new packet written to the queue ring
441 * buffer. This field should be similar to write_ptr and the user should
442 * update this field after updating the write_ptr.
443 *
444 * @doorbell_off: The doorbell offset in the doorbell pci-bar.
445 *
446 * @is_interop: Defines if this is a interop queue. Interop queue means that
447 * the queue can access both graphics and compute resources.
448 *
449 * @is_evicted: Defines if the queue is evicted. Only active queues
450 * are evicted, rendering them inactive.
451 *
452 * @is_active: Defines if the queue is active or not. @is_active and
453 * @is_evicted are protected by the DQM lock.
454 *
455 * @is_gws: Defines if the queue has been updated to be GWS-capable or not.
456 * @is_gws should be protected by the DQM lock, since changing it can yield the
457 * possibility of updating DQM state on number of GWS queues.
458 *
459 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
460 * of the queue.
461 *
462 * This structure represents the queue properties for each queue no matter if
463 * it's user mode or kernel mode queue.
464 *
465 */
466
467struct queue_properties {
468 enum kfd_queue_type type;
469 enum kfd_queue_format format;
470 unsigned int queue_id;
471 uint64_t queue_address;
472 uint64_t queue_size;
473 uint32_t priority;
474 uint32_t queue_percent;
475 uint32_t *read_ptr;
476 uint32_t *write_ptr;
477 void __iomem *doorbell_ptr;
478 uint32_t doorbell_off;
479 bool is_interop;
480 bool is_evicted;
481 bool is_active;
482 bool is_gws;
483 /* Not relevant for user mode queues in cp scheduling */
484 unsigned int vmid;
485 /* Relevant only for sdma queues*/
486 uint32_t sdma_engine_id;
487 uint32_t sdma_queue_id;
488 uint32_t sdma_vm_addr;
489 /* Relevant only for VI */
490 uint64_t eop_ring_buffer_address;
491 uint32_t eop_ring_buffer_size;
492 uint64_t ctx_save_restore_area_address;
493 uint32_t ctx_save_restore_area_size;
494 uint32_t ctl_stack_size;
495 uint64_t tba_addr;
496 uint64_t tma_addr;
497};
498
499#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
500 (q).queue_address != 0 && \
501 (q).queue_percent > 0 && \
502 !(q).is_evicted)
503
504enum mqd_update_flag {
505 UPDATE_FLAG_CU_MASK = 0,
506};
507
508struct mqd_update_info {
509 union {
510 struct {
511 uint32_t count; /* Must be a multiple of 32 */
512 uint32_t *ptr;
513 } cu_mask;
514 };
515 enum mqd_update_flag update_flag;
516};
517
518/**
519 * struct queue
520 *
521 * @list: Queue linked list.
522 *
523 * @mqd: The queue MQD (memory queue descriptor).
524 *
525 * @mqd_mem_obj: The MQD local gpu memory object.
526 *
527 * @gart_mqd_addr: The MQD gart mc address.
528 *
529 * @properties: The queue properties.
530 *
531 * @mec: Used only in no cp scheduling mode and identifies to micro engine id
532 * that the queue should be executed on.
533 *
534 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
535 * id.
536 *
537 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
538 *
539 * @process: The kfd process that created this queue.
540 *
541 * @device: The kfd device that created this queue.
542 *
543 * @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL
544 * otherwise.
545 *
546 * This structure represents user mode compute queues.
547 * It contains all the necessary data to handle such queues.
548 *
549 */
550
551struct queue {
552 struct list_head list;
553 void *mqd;
554 struct kfd_mem_obj *mqd_mem_obj;
555 uint64_t gart_mqd_addr;
556 struct queue_properties properties;
557
558 uint32_t mec;
559 uint32_t pipe;
560 uint32_t queue;
561
562 unsigned int sdma_id;
563 unsigned int doorbell_id;
564
565 struct kfd_process *process;
566 struct kfd_dev *device;
567 void *gws;
568
569 /* procfs */
570 struct kobject kobj;
571
572 void *gang_ctx_bo;
573 uint64_t gang_ctx_gpu_addr;
574 void *gang_ctx_cpu_ptr;
575
576 struct amdgpu_bo *wptr_bo;
577};
578
579enum KFD_MQD_TYPE {
580 KFD_MQD_TYPE_HIQ = 0, /* for hiq */
581 KFD_MQD_TYPE_CP, /* for cp queues and diq */
582 KFD_MQD_TYPE_SDMA, /* for sdma queues */
583 KFD_MQD_TYPE_DIQ, /* for diq */
584 KFD_MQD_TYPE_MAX
585};
586
587enum KFD_PIPE_PRIORITY {
588 KFD_PIPE_PRIORITY_CS_LOW = 0,
589 KFD_PIPE_PRIORITY_CS_MEDIUM,
590 KFD_PIPE_PRIORITY_CS_HIGH
591};
592
593struct scheduling_resources {
594 unsigned int vmid_mask;
595 enum kfd_queue_type type;
596 uint64_t queue_mask;
597 uint64_t gws_mask;
598 uint32_t oac_mask;
599 uint32_t gds_heap_base;
600 uint32_t gds_heap_size;
601};
602
603struct process_queue_manager {
604 /* data */
605 struct kfd_process *process;
606 struct list_head queues;
607 unsigned long *queue_slot_bitmap;
608};
609
610struct qcm_process_device {
611 /* The Device Queue Manager that owns this data */
612 struct device_queue_manager *dqm;
613 struct process_queue_manager *pqm;
614 /* Queues list */
615 struct list_head queues_list;
616 struct list_head priv_queue_list;
617
618 unsigned int queue_count;
619 unsigned int vmid;
620 bool is_debug;
621 unsigned int evicted; /* eviction counter, 0=active */
622
623 /* This flag tells if we should reset all wavefronts on
624 * process termination
625 */
626 bool reset_wavefronts;
627
628 /* This flag tells us if this process has a GWS-capable
629 * queue that will be mapped into the runlist. It's
630 * possible to request a GWS BO, but not have the queue
631 * currently mapped, and this changes how the MAP_PROCESS
632 * PM4 packet is configured.
633 */
634 bool mapped_gws_queue;
635
636 /* All the memory management data should be here too */
637 uint64_t gds_context_area;
638 /* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */
639 uint64_t page_table_base;
640 uint32_t sh_mem_config;
641 uint32_t sh_mem_bases;
642 uint32_t sh_mem_ape1_base;
643 uint32_t sh_mem_ape1_limit;
644 uint32_t gds_size;
645 uint32_t num_gws;
646 uint32_t num_oac;
647 uint32_t sh_hidden_private_base;
648
649 /* CWSR memory */
650 struct kgd_mem *cwsr_mem;
651 void *cwsr_kaddr;
652 uint64_t cwsr_base;
653 uint64_t tba_addr;
654 uint64_t tma_addr;
655
656 /* IB memory */
657 struct kgd_mem *ib_mem;
658 uint64_t ib_base;
659 void *ib_kaddr;
660
661 /* doorbell resources per process per device */
662 unsigned long *doorbell_bitmap;
663};
664
665/* KFD Memory Eviction */
666
667/* Approx. wait time before attempting to restore evicted BOs */
668#define PROCESS_RESTORE_TIME_MS 100
669/* Approx. back off time if restore fails due to lack of memory */
670#define PROCESS_BACK_OFF_TIME_MS 100
671/* Approx. time before evicting the process again */
672#define PROCESS_ACTIVE_TIME_MS 10
673
674/* 8 byte handle containing GPU ID in the most significant 4 bytes and
675 * idr_handle in the least significant 4 bytes
676 */
677#define MAKE_HANDLE(gpu_id, idr_handle) \
678 (((uint64_t)(gpu_id) << 32) + idr_handle)
679#define GET_GPU_ID(handle) (handle >> 32)
680#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
681
682enum kfd_pdd_bound {
683 PDD_UNBOUND = 0,
684 PDD_BOUND,
685 PDD_BOUND_SUSPENDED,
686};
687
688#define MAX_SYSFS_FILENAME_LEN 15
689
690/*
691 * SDMA counter runs at 100MHz frequency.
692 * We display SDMA activity in microsecond granularity in sysfs.
693 * As a result, the divisor is 100.
694 */
695#define SDMA_ACTIVITY_DIVISOR 100
696
697/* Data that is per-process-per device. */
698struct kfd_process_device {
699 /* The device that owns this data. */
700 struct kfd_dev *dev;
701
702 /* The process that owns this kfd_process_device. */
703 struct kfd_process *process;
704
705 /* per-process-per device QCM data structure */
706 struct qcm_process_device qpd;
707
708 /*Apertures*/
709 uint64_t lds_base;
710 uint64_t lds_limit;
711 uint64_t gpuvm_base;
712 uint64_t gpuvm_limit;
713 uint64_t scratch_base;
714 uint64_t scratch_limit;
715
716 /* VM context for GPUVM allocations */
717 struct file *drm_file;
718 void *drm_priv;
719 atomic64_t tlb_seq;
720
721 /* GPUVM allocations storage */
722 struct idr alloc_idr;
723
724 /* Flag used to tell the pdd has dequeued from the dqm.
725 * This is used to prevent dev->dqm->ops.process_termination() from
726 * being called twice when it is already called in IOMMU callback
727 * function.
728 */
729 bool already_dequeued;
730 bool runtime_inuse;
731
732 /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
733 enum kfd_pdd_bound bound;
734
735 /* VRAM usage */
736 uint64_t vram_usage;
737 struct attribute attr_vram;
738 char vram_filename[MAX_SYSFS_FILENAME_LEN];
739
740 /* SDMA activity tracking */
741 uint64_t sdma_past_activity_counter;
742 struct attribute attr_sdma;
743 char sdma_filename[MAX_SYSFS_FILENAME_LEN];
744
745 /* Eviction activity tracking */
746 uint64_t last_evict_timestamp;
747 atomic64_t evict_duration_counter;
748 struct attribute attr_evict;
749
750 struct kobject *kobj_stats;
751 unsigned int doorbell_index;
752
753 /*
754 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
755 * that is associated with device encoded by "this" struct instance. The
756 * value reflects CU usage by all of the waves launched by this process
757 * on this device. A very important property of occupancy parameter is
758 * that its value is a snapshot of current use.
759 *
760 * Following is to be noted regarding how this parameter is reported:
761 *
762 * The number of waves that a CU can launch is limited by couple of
763 * parameters. These are encoded by struct amdgpu_cu_info instance
764 * that is part of every device definition. For GFX9 devices this
765 * translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
766 * do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
767 * when they do use scratch memory. This could change for future
768 * devices and therefore this example should be considered as a guide.
769 *
770 * All CU's of a device are available for the process. This may not be true
771 * under certain conditions - e.g. CU masking.
772 *
773 * Finally number of CU's that are occupied by a process is affected by both
774 * number of CU's a device has along with number of other competing processes
775 */
776 struct attribute attr_cu_occupancy;
777
778 /* sysfs counters for GPU retry fault and page migration tracking */
779 struct kobject *kobj_counters;
780 struct attribute attr_faults;
781 struct attribute attr_page_in;
782 struct attribute attr_page_out;
783 uint64_t faults;
784 uint64_t page_in;
785 uint64_t page_out;
786 /*
787 * If this process has been checkpointed before, then the user
788 * application will use the original gpu_id on the
789 * checkpointed node to refer to this device.
790 */
791 uint32_t user_gpu_id;
792
793 void *proc_ctx_bo;
794 uint64_t proc_ctx_gpu_addr;
795 void *proc_ctx_cpu_ptr;
796};
797
798#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
799
800struct svm_range_list {
801 struct mutex lock;
802 struct rb_root_cached objects;
803 struct list_head list;
804 struct work_struct deferred_list_work;
805 struct list_head deferred_range_list;
806 struct list_head criu_svm_metadata_list;
807 spinlock_t deferred_list_lock;
808 atomic_t evicted_ranges;
809 atomic_t drain_pagefaults;
810 struct delayed_work restore_work;
811 DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
812 struct task_struct *faulting_task;
813};
814
815/* Process data */
816struct kfd_process {
817 /*
818 * kfd_process are stored in an mm_struct*->kfd_process*
819 * hash table (kfd_processes in kfd_process.c)
820 */
821 struct hlist_node kfd_processes;
822
823 /*
824 * Opaque pointer to mm_struct. We don't hold a reference to
825 * it so it should never be dereferenced from here. This is
826 * only used for looking up processes by their mm.
827 */
828 void *mm;
829
830 struct kref ref;
831 struct work_struct release_work;
832
833 struct mutex mutex;
834
835 /*
836 * In any process, the thread that started main() is the lead
837 * thread and outlives the rest.
838 * It is here because amd_iommu_bind_pasid wants a task_struct.
839 * It can also be used for safely getting a reference to the
840 * mm_struct of the process.
841 */
842 struct task_struct *lead_thread;
843
844 /* We want to receive a notification when the mm_struct is destroyed */
845 struct mmu_notifier mmu_notifier;
846
847 u32 pasid;
848
849 /*
850 * Array of kfd_process_device pointers,
851 * one for each device the process is using.
852 */
853 struct kfd_process_device *pdds[MAX_GPU_INSTANCE];
854 uint32_t n_pdds;
855
856 struct process_queue_manager pqm;
857
858 /*Is the user space process 32 bit?*/
859 bool is_32bit_user_mode;
860
861 /* Event-related data */
862 struct mutex event_mutex;
863 /* Event ID allocator and lookup */
864 struct idr event_idr;
865 /* Event page */
866 u64 signal_handle;
867 struct kfd_signal_page *signal_page;
868 size_t signal_mapped_size;
869 size_t signal_event_count;
870 bool signal_event_limit_reached;
871
872 /* Information used for memory eviction */
873 void *kgd_process_info;
874 /* Eviction fence that is attached to all the BOs of this process. The
875 * fence will be triggered during eviction and new one will be created
876 * during restore
877 */
878 struct dma_fence *ef;
879
880 /* Work items for evicting and restoring BOs */
881 struct delayed_work eviction_work;
882 struct delayed_work restore_work;
883 /* seqno of the last scheduled eviction */
884 unsigned int last_eviction_seqno;
885 /* Approx. the last timestamp (in jiffies) when the process was
886 * restored after an eviction
887 */
888 unsigned long last_restore_timestamp;
889
890 /* Kobj for our procfs */
891 struct kobject *kobj;
892 struct kobject *kobj_queues;
893 struct attribute attr_pasid;
894
895 /* shared virtual memory registered by this process */
896 struct svm_range_list svms;
897
898 bool xnack_enabled;
899
900 atomic_t poison;
901 /* Queues are in paused stated because we are in the process of doing a CRIU checkpoint */
902 bool queues_paused;
903};
904
905#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
906extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
907extern struct srcu_struct kfd_processes_srcu;
908
909/**
910 * typedef amdkfd_ioctl_t - typedef for ioctl function pointer.
911 *
912 * @filep: pointer to file structure.
913 * @p: amdkfd process pointer.
914 * @data: pointer to arg that was copied from user.
915 *
916 * Return: returns ioctl completion code.
917 */
918typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
919 void *data);
920
921struct amdkfd_ioctl_desc {
922 unsigned int cmd;
923 int flags;
924 amdkfd_ioctl_t *func;
925 unsigned int cmd_drv;
926 const char *name;
927};
928bool kfd_dev_is_large_bar(struct kfd_dev *dev);
929
930int kfd_process_create_wq(void);
931void kfd_process_destroy_wq(void);
932void kfd_cleanup_processes(void);
933struct kfd_process *kfd_create_process(struct file *filep);
934struct kfd_process *kfd_get_process(const struct task_struct *task);
935struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
936struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
937
938int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
939int kfd_process_gpuid_from_adev(struct kfd_process *p,
940 struct amdgpu_device *adev, uint32_t *gpuid,
941 uint32_t *gpuidx);
942static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
943 uint32_t gpuidx, uint32_t *gpuid) {
944 return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL;
945}
946static inline struct kfd_process_device *kfd_process_device_from_gpuidx(
947 struct kfd_process *p, uint32_t gpuidx) {
948 return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL;
949}
950
951void kfd_unref_process(struct kfd_process *p);
952int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger);
953int kfd_process_restore_queues(struct kfd_process *p);
954void kfd_suspend_all_processes(void);
955int kfd_resume_all_processes(void);
956
957struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *process,
958 uint32_t gpu_id);
959
960int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id);
961
962int kfd_process_device_init_vm(struct kfd_process_device *pdd,
963 struct file *drm_file);
964struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
965 struct kfd_process *p);
966struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
967 struct kfd_process *p);
968struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
969 struct kfd_process *p);
970
971bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
972
973int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
974 struct vm_area_struct *vma);
975
976/* KFD process API for creating and translating handles */
977int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
978 void *mem);
979void *kfd_process_device_translate_handle(struct kfd_process_device *p,
980 int handle);
981void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
982 int handle);
983struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid);
984
985/* PASIDs */
986int kfd_pasid_init(void);
987void kfd_pasid_exit(void);
988bool kfd_set_pasid_limit(unsigned int new_limit);
989unsigned int kfd_get_pasid_limit(void);
990u32 kfd_pasid_alloc(void);
991void kfd_pasid_free(u32 pasid);
992
993/* Doorbells */
994size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
995int kfd_doorbell_init(struct kfd_dev *kfd);
996void kfd_doorbell_fini(struct kfd_dev *kfd);
997int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
998 struct vm_area_struct *vma);
999void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
1000 unsigned int *doorbell_off);
1001void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
1002u32 read_kernel_doorbell(u32 __iomem *db);
1003void write_kernel_doorbell(void __iomem *db, u32 value);
1004void write_kernel_doorbell64(void __iomem *db, u64 value);
1005unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
1006 struct kfd_process_device *pdd,
1007 unsigned int doorbell_id);
1008phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd);
1009int kfd_alloc_process_doorbells(struct kfd_dev *kfd,
1010 unsigned int *doorbell_index);
1011void kfd_free_process_doorbells(struct kfd_dev *kfd,
1012 unsigned int doorbell_index);
1013/* GTT Sub-Allocator */
1014
1015int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
1016 struct kfd_mem_obj **mem_obj);
1017
1018int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
1019
1020extern struct device *kfd_device;
1021
1022/* KFD's procfs */
1023void kfd_procfs_init(void);
1024void kfd_procfs_shutdown(void);
1025int kfd_procfs_add_queue(struct queue *q);
1026void kfd_procfs_del_queue(struct queue *q);
1027
1028/* Topology */
1029int kfd_topology_init(void);
1030void kfd_topology_shutdown(void);
1031int kfd_topology_add_device(struct kfd_dev *gpu);
1032int kfd_topology_remove_device(struct kfd_dev *gpu);
1033struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
1034 uint32_t proximity_domain);
1035struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
1036 uint32_t proximity_domain);
1037struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
1038struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
1039struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
1040struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev);
1041int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
1042int kfd_numa_node_to_apic_id(int numa_node_id);
1043void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
1044
1045/* Interrupts */
1046int kfd_interrupt_init(struct kfd_dev *dev);
1047void kfd_interrupt_exit(struct kfd_dev *dev);
1048bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
1049bool interrupt_is_wanted(struct kfd_dev *dev,
1050 const uint32_t *ih_ring_entry,
1051 uint32_t *patched_ihre, bool *flag);
1052
1053/* amdkfd Apertures */
1054int kfd_init_apertures(struct kfd_process *process);
1055
1056void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1057 uint64_t tba_addr,
1058 uint64_t tma_addr);
1059
1060/* CRIU */
1061/*
1062 * Need to increment KFD_CRIU_PRIV_VERSION each time a change is made to any of the CRIU private
1063 * structures:
1064 * kfd_criu_process_priv_data
1065 * kfd_criu_device_priv_data
1066 * kfd_criu_bo_priv_data
1067 * kfd_criu_queue_priv_data
1068 * kfd_criu_event_priv_data
1069 * kfd_criu_svm_range_priv_data
1070 */
1071
1072#define KFD_CRIU_PRIV_VERSION 1
1073
1074struct kfd_criu_process_priv_data {
1075 uint32_t version;
1076 uint32_t xnack_mode;
1077};
1078
1079struct kfd_criu_device_priv_data {
1080 /* For future use */
1081 uint64_t reserved;
1082};
1083
1084struct kfd_criu_bo_priv_data {
1085 uint64_t user_addr;
1086 uint32_t idr_handle;
1087 uint32_t mapped_gpuids[MAX_GPU_INSTANCE];
1088};
1089
1090/*
1091 * The first 4 bytes of kfd_criu_queue_priv_data, kfd_criu_event_priv_data,
1092 * kfd_criu_svm_range_priv_data is the object type
1093 */
1094enum kfd_criu_object_type {
1095 KFD_CRIU_OBJECT_TYPE_QUEUE,
1096 KFD_CRIU_OBJECT_TYPE_EVENT,
1097 KFD_CRIU_OBJECT_TYPE_SVM_RANGE,
1098};
1099
1100struct kfd_criu_svm_range_priv_data {
1101 uint32_t object_type;
1102 uint64_t start_addr;
1103 uint64_t size;
1104 /* Variable length array of attributes */
1105 struct kfd_ioctl_svm_attribute attrs[];
1106};
1107
1108struct kfd_criu_queue_priv_data {
1109 uint32_t object_type;
1110 uint64_t q_address;
1111 uint64_t q_size;
1112 uint64_t read_ptr_addr;
1113 uint64_t write_ptr_addr;
1114 uint64_t doorbell_off;
1115 uint64_t eop_ring_buffer_address;
1116 uint64_t ctx_save_restore_area_address;
1117 uint32_t gpu_id;
1118 uint32_t type;
1119 uint32_t format;
1120 uint32_t q_id;
1121 uint32_t priority;
1122 uint32_t q_percent;
1123 uint32_t doorbell_id;
1124 uint32_t gws;
1125 uint32_t sdma_id;
1126 uint32_t eop_ring_buffer_size;
1127 uint32_t ctx_save_restore_area_size;
1128 uint32_t ctl_stack_size;
1129 uint32_t mqd_size;
1130};
1131
1132struct kfd_criu_event_priv_data {
1133 uint32_t object_type;
1134 uint64_t user_handle;
1135 uint32_t event_id;
1136 uint32_t auto_reset;
1137 uint32_t type;
1138 uint32_t signaled;
1139
1140 union {
1141 struct kfd_hsa_memory_exception_data memory_exception_data;
1142 struct kfd_hsa_hw_exception_data hw_exception_data;
1143 };
1144};
1145
1146int kfd_process_get_queue_info(struct kfd_process *p,
1147 uint32_t *num_queues,
1148 uint64_t *priv_data_sizes);
1149
1150int kfd_criu_checkpoint_queues(struct kfd_process *p,
1151 uint8_t __user *user_priv_data,
1152 uint64_t *priv_data_offset);
1153
1154int kfd_criu_restore_queue(struct kfd_process *p,
1155 uint8_t __user *user_priv_data,
1156 uint64_t *priv_data_offset,
1157 uint64_t max_priv_data_size);
1158
1159int kfd_criu_checkpoint_events(struct kfd_process *p,
1160 uint8_t __user *user_priv_data,
1161 uint64_t *priv_data_offset);
1162
1163int kfd_criu_restore_event(struct file *devkfd,
1164 struct kfd_process *p,
1165 uint8_t __user *user_priv_data,
1166 uint64_t *priv_data_offset,
1167 uint64_t max_priv_data_size);
1168/* CRIU - End */
1169
1170/* Queue Context Management */
1171int init_queue(struct queue **q, const struct queue_properties *properties);
1172void uninit_queue(struct queue *q);
1173void print_queue_properties(struct queue_properties *q);
1174void print_queue(struct queue *q);
1175
1176struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
1177 struct kfd_dev *dev);
1178struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
1179 struct kfd_dev *dev);
1180struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
1181 struct kfd_dev *dev);
1182struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
1183 struct kfd_dev *dev);
1184struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
1185 struct kfd_dev *dev);
1186struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
1187 struct kfd_dev *dev);
1188struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
1189 struct kfd_dev *dev);
1190struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
1191void device_queue_manager_uninit(struct device_queue_manager *dqm);
1192struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
1193 enum kfd_queue_type type);
1194void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
1195int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid);
1196
1197/* Process Queue Manager */
1198struct process_queue_node {
1199 struct queue *q;
1200 struct kernel_queue *kq;
1201 struct list_head process_queue_list;
1202};
1203
1204void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
1205void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
1206int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
1207void pqm_uninit(struct process_queue_manager *pqm);
1208int pqm_create_queue(struct process_queue_manager *pqm,
1209 struct kfd_dev *dev,
1210 struct file *f,
1211 struct queue_properties *properties,
1212 unsigned int *qid,
1213 struct amdgpu_bo *wptr_bo,
1214 const struct kfd_criu_queue_priv_data *q_data,
1215 const void *restore_mqd,
1216 const void *restore_ctl_stack,
1217 uint32_t *p_doorbell_offset_in_process);
1218int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
1219int pqm_update_queue_properties(struct process_queue_manager *pqm, unsigned int qid,
1220 struct queue_properties *p);
1221int pqm_update_mqd(struct process_queue_manager *pqm, unsigned int qid,
1222 struct mqd_update_info *minfo);
1223int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
1224 void *gws);
1225struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
1226 unsigned int qid);
1227struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
1228 unsigned int qid);
1229int pqm_get_wave_state(struct process_queue_manager *pqm,
1230 unsigned int qid,
1231 void __user *ctl_stack,
1232 u32 *ctl_stack_used_size,
1233 u32 *save_area_used_size);
1234
1235int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
1236 uint64_t fence_value,
1237 unsigned int timeout_ms);
1238
1239int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
1240 unsigned int qid,
1241 u32 *mqd_size,
1242 u32 *ctl_stack_size);
1243/* Packet Manager */
1244
1245#define KFD_FENCE_COMPLETED (100)
1246#define KFD_FENCE_INIT (10)
1247
1248struct packet_manager {
1249 struct device_queue_manager *dqm;
1250 struct kernel_queue *priv_queue;
1251 struct mutex lock;
1252 bool allocated;
1253 struct kfd_mem_obj *ib_buffer_obj;
1254 unsigned int ib_size_bytes;
1255 bool is_over_subscription;
1256
1257 const struct packet_manager_funcs *pmf;
1258};
1259
1260struct packet_manager_funcs {
1261 /* Support ASIC-specific packet formats for PM4 packets */
1262 int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
1263 struct qcm_process_device *qpd);
1264 int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
1265 uint64_t ib, size_t ib_size_in_dwords, bool chain);
1266 int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
1267 struct scheduling_resources *res);
1268 int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
1269 struct queue *q, bool is_static);
1270 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
1271 enum kfd_unmap_queues_filter mode,
1272 uint32_t filter_param, bool reset);
1273 int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
1274 uint64_t fence_address, uint64_t fence_value);
1275 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
1276
1277 /* Packet sizes */
1278 int map_process_size;
1279 int runlist_size;
1280 int set_resources_size;
1281 int map_queues_size;
1282 int unmap_queues_size;
1283 int query_status_size;
1284 int release_mem_size;
1285};
1286
1287extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1288extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1289extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
1290
1291int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1292void pm_uninit(struct packet_manager *pm, bool hanging);
1293int pm_send_set_resources(struct packet_manager *pm,
1294 struct scheduling_resources *res);
1295int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
1296int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
1297 uint64_t fence_value);
1298
1299int pm_send_unmap_queue(struct packet_manager *pm,
1300 enum kfd_unmap_queues_filter mode,
1301 uint32_t filter_param, bool reset);
1302
1303void pm_release_ib(struct packet_manager *pm);
1304
1305/* Following PM funcs can be shared among VI and AI */
1306unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
1307
1308uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
1309
1310/* Events */
1311extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
1312extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
1313extern const struct kfd_event_interrupt_class event_interrupt_class_v11;
1314
1315extern const struct kfd_device_global_init_class device_global_init_class_cik;
1316
1317int kfd_event_init_process(struct kfd_process *p);
1318void kfd_event_free_process(struct kfd_process *p);
1319int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
1320int kfd_wait_on_events(struct kfd_process *p,
1321 uint32_t num_events, void __user *data,
1322 bool all, uint32_t *user_timeout_ms,
1323 uint32_t *wait_result);
1324void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
1325 uint32_t valid_id_bits);
1326void kfd_signal_iommu_event(struct kfd_dev *dev,
1327 u32 pasid, unsigned long address,
1328 bool is_write_requested, bool is_execute_requested);
1329void kfd_signal_hw_exception_event(u32 pasid);
1330int kfd_set_event(struct kfd_process *p, uint32_t event_id);
1331int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
1332int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset);
1333
1334int kfd_event_create(struct file *devkfd, struct kfd_process *p,
1335 uint32_t event_type, bool auto_reset, uint32_t node_id,
1336 uint32_t *event_id, uint32_t *event_trigger_data,
1337 uint64_t *event_page_offset, uint32_t *event_slot_index);
1338
1339int kfd_get_num_events(struct kfd_process *p);
1340int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
1341
1342void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
1343 struct kfd_vm_fault_info *info);
1344
1345void kfd_signal_reset_event(struct kfd_dev *dev);
1346
1347void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid);
1348
1349void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);
1350
1351static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)
1352{
1353 return KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2) ||
1354 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 1) &&
1355 dev->adev->sdma.instance[0].fw_version >= 18) ||
1356 KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 0);
1357}
1358
1359bool kfd_is_locked(void);
1360
1361/* Compute profile */
1362void kfd_inc_compute_active(struct kfd_dev *dev);
1363void kfd_dec_compute_active(struct kfd_dev *dev);
1364
1365/* Cgroup Support */
1366/* Check with device cgroup if @kfd device is accessible */
1367static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
1368{
1369#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
1370 struct drm_device *ddev = adev_to_drm(kfd->adev);
1371
1372 return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
1373 ddev->render->index,
1374 DEVCG_ACC_WRITE | DEVCG_ACC_READ);
1375#else
1376 return 0;
1377#endif
1378}
1379
1380/* Debugfs */
1381#if defined(CONFIG_DEBUG_FS)
1382
1383void kfd_debugfs_init(void);
1384void kfd_debugfs_fini(void);
1385int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
1386int pqm_debugfs_mqds(struct seq_file *m, void *data);
1387int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
1388int dqm_debugfs_hqds(struct seq_file *m, void *data);
1389int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
1390int pm_debugfs_runlist(struct seq_file *m, void *data);
1391
1392int kfd_debugfs_hang_hws(struct kfd_dev *dev);
1393int pm_debugfs_hang_hws(struct packet_manager *pm);
1394int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
1395
1396#else
1397
1398static inline void kfd_debugfs_init(void) {}
1399static inline void kfd_debugfs_fini(void) {}
1400
1401#endif
1402
1403#endif
1404

source code of linux/drivers/gpu/drm/amd/amdkfd/kfd_priv.h