1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
2 | /* |
3 | * Copyright 2014-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #ifndef KFD_DEVICE_QUEUE_MANAGER_H_ |
26 | #define KFD_DEVICE_QUEUE_MANAGER_H_ |
27 | |
28 | #include <linux/rwsem.h> |
29 | #include <linux/list.h> |
30 | #include <linux/mutex.h> |
31 | #include <linux/sched/mm.h> |
32 | #include "kfd_priv.h" |
33 | #include "kfd_mqd_manager.h" |
34 | |
35 | |
36 | #define VMID_NUM 16 |
37 | |
38 | #define KFD_MES_PROCESS_QUANTUM 100000 |
39 | #define KFD_MES_GANG_QUANTUM 10000 |
40 | #define USE_DEFAULT_GRACE_PERIOD 0xffffffff |
41 | |
42 | struct device_process_node { |
43 | struct qcm_process_device *qpd; |
44 | struct list_head list; |
45 | }; |
46 | |
47 | union SQ_CMD_BITS { |
48 | struct { |
49 | uint32_t cmd:3; |
50 | uint32_t:1; |
51 | uint32_t mode:3; |
52 | uint32_t check_vmid:1; |
53 | uint32_t trap_id:3; |
54 | uint32_t:5; |
55 | uint32_t wave_id:4; |
56 | uint32_t simd_id:2; |
57 | uint32_t:2; |
58 | uint32_t queue_id:3; |
59 | uint32_t:1; |
60 | uint32_t vm_id:4; |
61 | } bitfields, bits; |
62 | uint32_t u32All; |
63 | signed int i32All; |
64 | float f32All; |
65 | }; |
66 | |
67 | union GRBM_GFX_INDEX_BITS { |
68 | struct { |
69 | uint32_t instance_index:8; |
70 | uint32_t sh_index:8; |
71 | uint32_t se_index:8; |
72 | uint32_t:5; |
73 | uint32_t sh_broadcast_writes:1; |
74 | uint32_t instance_broadcast_writes:1; |
75 | uint32_t se_broadcast_writes:1; |
76 | } bitfields, bits; |
77 | uint32_t u32All; |
78 | signed int i32All; |
79 | float f32All; |
80 | }; |
81 | |
82 | /** |
83 | * struct device_queue_manager_ops |
84 | * |
85 | * @create_queue: Queue creation routine. |
86 | * |
87 | * @destroy_queue: Queue destruction routine. |
88 | * |
89 | * @update_queue: Queue update routine. |
90 | * |
91 | * @exeute_queues: Dispatches the queues list to the H/W. |
92 | * |
93 | * @register_process: This routine associates a specific process with device. |
94 | * |
95 | * @unregister_process: destroys the associations between process to device. |
96 | * |
97 | * @initialize: Initializes the pipelines and memory module for that device. |
98 | * |
99 | * @start: Initializes the resources/modules the device needs for queues |
100 | * execution. This function is called on device initialization and after the |
101 | * system woke up after suspension. |
102 | * |
103 | * @stop: This routine stops execution of all the active queue running on the |
104 | * H/W and basically this function called on system suspend. |
105 | * |
106 | * @uninitialize: Destroys all the device queue manager resources allocated in |
107 | * initialize routine. |
108 | * |
109 | * @create_kernel_queue: Creates kernel queue. Used for debug queue. |
110 | * |
111 | * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. |
112 | * |
113 | * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the |
114 | * memory apertures. |
115 | * |
116 | * @process_termination: Clears all process queues belongs to that device. |
117 | * |
118 | * @evict_process_queues: Evict all active queues of a process |
119 | * |
120 | * @restore_process_queues: Restore all evicted queues of a process |
121 | * |
122 | * @get_wave_state: Retrieves context save state and optionally copies the |
123 | * control stack, if kept in the MQD, to the given userspace address. |
124 | * |
125 | * @reset_queues: reset queues which consume RAS poison |
126 | * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint. |
127 | * |
128 | * @checkpoint_mqd: checkpoint queue MQD contents for CRIU. |
129 | */ |
130 | |
131 | struct device_queue_manager_ops { |
132 | int (*create_queue)(struct device_queue_manager *dqm, |
133 | struct queue *q, |
134 | struct qcm_process_device *qpd, |
135 | const struct kfd_criu_queue_priv_data *qd, |
136 | const void *restore_mqd, |
137 | const void *restore_ctl_stack); |
138 | |
139 | int (*destroy_queue)(struct device_queue_manager *dqm, |
140 | struct qcm_process_device *qpd, |
141 | struct queue *q); |
142 | |
143 | int (*update_queue)(struct device_queue_manager *dqm, |
144 | struct queue *q, struct mqd_update_info *minfo); |
145 | |
146 | int (*register_process)(struct device_queue_manager *dqm, |
147 | struct qcm_process_device *qpd); |
148 | |
149 | int (*unregister_process)(struct device_queue_manager *dqm, |
150 | struct qcm_process_device *qpd); |
151 | |
152 | int (*initialize)(struct device_queue_manager *dqm); |
153 | int (*start)(struct device_queue_manager *dqm); |
154 | int (*stop)(struct device_queue_manager *dqm); |
155 | void (*pre_reset)(struct device_queue_manager *dqm); |
156 | void (*uninitialize)(struct device_queue_manager *dqm); |
157 | int (*create_kernel_queue)(struct device_queue_manager *dqm, |
158 | struct kernel_queue *kq, |
159 | struct qcm_process_device *qpd); |
160 | |
161 | void (*destroy_kernel_queue)(struct device_queue_manager *dqm, |
162 | struct kernel_queue *kq, |
163 | struct qcm_process_device *qpd); |
164 | |
165 | bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, |
166 | struct qcm_process_device *qpd, |
167 | enum cache_policy default_policy, |
168 | enum cache_policy alternate_policy, |
169 | void __user *alternate_aperture_base, |
170 | uint64_t alternate_aperture_size); |
171 | |
172 | int (*process_termination)(struct device_queue_manager *dqm, |
173 | struct qcm_process_device *qpd); |
174 | |
175 | int (*evict_process_queues)(struct device_queue_manager *dqm, |
176 | struct qcm_process_device *qpd); |
177 | int (*restore_process_queues)(struct device_queue_manager *dqm, |
178 | struct qcm_process_device *qpd); |
179 | |
180 | int (*get_wave_state)(struct device_queue_manager *dqm, |
181 | struct queue *q, |
182 | void __user *ctl_stack, |
183 | u32 *ctl_stack_used_size, |
184 | u32 *save_area_used_size); |
185 | |
186 | int (*reset_queues)(struct device_queue_manager *dqm, |
187 | uint16_t pasid); |
188 | void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm, |
189 | const struct queue *q, u32 *mqd_size, |
190 | u32 *ctl_stack_size); |
191 | |
192 | int (*checkpoint_mqd)(struct device_queue_manager *dqm, |
193 | const struct queue *q, |
194 | void *mqd, |
195 | void *ctl_stack); |
196 | }; |
197 | |
198 | struct device_queue_manager_asic_ops { |
199 | int (*update_qpd)(struct device_queue_manager *dqm, |
200 | struct qcm_process_device *qpd); |
201 | bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, |
202 | struct qcm_process_device *qpd, |
203 | enum cache_policy default_policy, |
204 | enum cache_policy alternate_policy, |
205 | void __user *alternate_aperture_base, |
206 | uint64_t alternate_aperture_size); |
207 | void (*init_sdma_vm)(struct device_queue_manager *dqm, |
208 | struct queue *q, |
209 | struct qcm_process_device *qpd); |
210 | struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type, |
211 | struct kfd_node *dev); |
212 | }; |
213 | |
214 | /** |
215 | * struct device_queue_manager |
216 | * |
217 | * This struct is a base class for the kfd queues scheduler in the |
218 | * device level. The device base class should expose the basic operations |
219 | * for queue creation and queue destruction. This base class hides the |
220 | * scheduling mode of the driver and the specific implementation of the |
221 | * concrete device. This class is the only class in the queues scheduler |
222 | * that configures the H/W. |
223 | * |
224 | */ |
225 | |
226 | struct device_queue_manager { |
227 | struct device_queue_manager_ops ops; |
228 | struct device_queue_manager_asic_ops asic_ops; |
229 | |
230 | struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX]; |
231 | struct packet_manager packet_mgr; |
232 | struct kfd_node *dev; |
233 | struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */ |
234 | struct list_head queues; |
235 | unsigned int saved_flags; |
236 | unsigned int processes_count; |
237 | unsigned int active_queue_count; |
238 | unsigned int active_cp_queue_count; |
239 | unsigned int gws_queue_count; |
240 | unsigned int total_queue_count; |
241 | unsigned int next_pipe_to_allocate; |
242 | unsigned int *allocated_queues; |
243 | DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES); |
244 | DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); |
245 | /* the pasid mapping for each kfd vmid */ |
246 | uint16_t vmid_pasid[VMID_NUM]; |
247 | uint64_t pipelines_addr; |
248 | uint64_t fence_gpu_addr; |
249 | uint64_t *fence_addr; |
250 | struct kfd_mem_obj *fence_mem; |
251 | bool active_runlist; |
252 | int sched_policy; |
253 | uint32_t trap_debug_vmid; |
254 | |
255 | /* hw exception */ |
256 | bool is_hws_hang; |
257 | bool is_resetting; |
258 | struct work_struct hw_exception_work; |
259 | struct kfd_mem_obj hiq_sdma_mqd; |
260 | bool sched_running; |
261 | |
262 | /* used for GFX 9.4.3 only */ |
263 | uint32_t current_logical_xcc_start; |
264 | |
265 | uint32_t wait_times; |
266 | |
267 | wait_queue_head_t destroy_wait; |
268 | }; |
269 | |
270 | void device_queue_manager_init_cik( |
271 | struct device_queue_manager_asic_ops *asic_ops); |
272 | void device_queue_manager_init_vi( |
273 | struct device_queue_manager_asic_ops *asic_ops); |
274 | void device_queue_manager_init_v9( |
275 | struct device_queue_manager_asic_ops *asic_ops); |
276 | void device_queue_manager_init_v10( |
277 | struct device_queue_manager_asic_ops *asic_ops); |
278 | void device_queue_manager_init_v11( |
279 | struct device_queue_manager_asic_ops *asic_ops); |
280 | void program_sh_mem_settings(struct device_queue_manager *dqm, |
281 | struct qcm_process_device *qpd); |
282 | unsigned int get_cp_queues_num(struct device_queue_manager *dqm); |
283 | unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); |
284 | unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); |
285 | unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); |
286 | unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); |
287 | int reserve_debug_trap_vmid(struct device_queue_manager *dqm, |
288 | struct qcm_process_device *qpd); |
289 | int release_debug_trap_vmid(struct device_queue_manager *dqm, |
290 | struct qcm_process_device *qpd); |
291 | int suspend_queues(struct kfd_process *p, |
292 | uint32_t num_queues, |
293 | uint32_t grace_period, |
294 | uint64_t exception_clear_mask, |
295 | uint32_t *usr_queue_id_array); |
296 | int resume_queues(struct kfd_process *p, |
297 | uint32_t num_queues, |
298 | uint32_t *usr_queue_id_array); |
299 | void set_queue_snapshot_entry(struct queue *q, |
300 | uint64_t exception_clear_mask, |
301 | struct kfd_queue_snapshot_entry *qss_entry); |
302 | int debug_lock_and_unmap(struct device_queue_manager *dqm); |
303 | int debug_map_and_unlock(struct device_queue_manager *dqm); |
304 | int debug_refresh_runlist(struct device_queue_manager *dqm); |
305 | |
306 | static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) |
307 | { |
308 | return (pdd->lds_base >> 16) & 0xFF; |
309 | } |
310 | |
311 | static inline unsigned int |
312 | get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) |
313 | { |
314 | return (pdd->lds_base >> 60) & 0x0E; |
315 | } |
316 | |
317 | /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS |
318 | * happens while holding this lock anywhere to prevent deadlocks when |
319 | * an MMU notifier runs in reclaim-FS context. |
320 | */ |
321 | static inline void dqm_lock(struct device_queue_manager *dqm) |
322 | { |
323 | mutex_lock(&dqm->lock_hidden); |
324 | dqm->saved_flags = memalloc_noreclaim_save(); |
325 | } |
326 | static inline void dqm_unlock(struct device_queue_manager *dqm) |
327 | { |
328 | memalloc_noreclaim_restore(flags: dqm->saved_flags); |
329 | mutex_unlock(lock: &dqm->lock_hidden); |
330 | } |
331 | |
332 | static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) |
333 | { |
334 | /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */ |
335 | return get_user(*val, q_rptr + 1); |
336 | } |
337 | #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ |
338 | |