1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2014-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include <linux/printk.h> |
26 | #include <linux/slab.h> |
27 | #include <linux/mm_types.h> |
28 | |
29 | #include "kfd_priv.h" |
30 | #include "kfd_mqd_manager.h" |
31 | #include "vi_structs.h" |
32 | #include "gca/gfx_8_0_sh_mask.h" |
33 | #include "gca/gfx_8_0_enum.h" |
34 | #include "oss/oss_3_0_sh_mask.h" |
35 | |
36 | #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8 |
37 | |
38 | static inline struct vi_mqd *get_mqd(void *mqd) |
39 | { |
40 | return (struct vi_mqd *)mqd; |
41 | } |
42 | |
43 | static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd) |
44 | { |
45 | return (struct vi_sdma_mqd *)mqd; |
46 | } |
47 | |
48 | static void update_cu_mask(struct mqd_manager *mm, void *mqd, |
49 | struct mqd_update_info *minfo) |
50 | { |
51 | struct vi_mqd *m; |
52 | uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */ |
53 | |
54 | if (!minfo || !minfo->cu_mask.ptr) |
55 | return; |
56 | |
57 | mqd_symmetrically_map_cu_mask(mm, |
58 | cu_mask: minfo->cu_mask.ptr, cu_mask_count: minfo->cu_mask.count, se_mask, inst: 0); |
59 | |
60 | m = get_mqd(mqd); |
61 | m->compute_static_thread_mgmt_se0 = se_mask[0]; |
62 | m->compute_static_thread_mgmt_se1 = se_mask[1]; |
63 | m->compute_static_thread_mgmt_se2 = se_mask[2]; |
64 | m->compute_static_thread_mgmt_se3 = se_mask[3]; |
65 | |
66 | pr_debug("Update cu mask to %#x %#x %#x %#x\n" , |
67 | m->compute_static_thread_mgmt_se0, |
68 | m->compute_static_thread_mgmt_se1, |
69 | m->compute_static_thread_mgmt_se2, |
70 | m->compute_static_thread_mgmt_se3); |
71 | } |
72 | |
73 | static void set_priority(struct vi_mqd *m, struct queue_properties *q) |
74 | { |
75 | m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; |
76 | m->cp_hqd_queue_priority = q->priority; |
77 | } |
78 | |
79 | static struct kfd_mem_obj *allocate_mqd(struct kfd_node *kfd, |
80 | struct queue_properties *q) |
81 | { |
82 | struct kfd_mem_obj *mqd_mem_obj; |
83 | |
84 | if (kfd_gtt_sa_allocate(node: kfd, size: sizeof(struct vi_mqd), |
85 | mem_obj: &mqd_mem_obj)) |
86 | return NULL; |
87 | |
88 | return mqd_mem_obj; |
89 | } |
90 | |
91 | static void init_mqd(struct mqd_manager *mm, void **mqd, |
92 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
93 | struct queue_properties *q) |
94 | { |
95 | uint64_t addr; |
96 | struct vi_mqd *m; |
97 | |
98 | m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr; |
99 | addr = mqd_mem_obj->gpu_addr; |
100 | |
101 | memset(m, 0, sizeof(struct vi_mqd)); |
102 | |
103 | m->header = 0xC0310800; |
104 | m->compute_pipelinestat_enable = 1; |
105 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; |
106 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; |
107 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; |
108 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; |
109 | |
110 | m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK | |
111 | 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT; |
112 | |
113 | m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT | |
114 | MTYPE_UC << CP_MQD_CONTROL__MTYPE__SHIFT; |
115 | |
116 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); |
117 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); |
118 | |
119 | m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT | |
120 | 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT | |
121 | 1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT; |
122 | |
123 | set_priority(m, q); |
124 | m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT; |
125 | |
126 | if (q->format == KFD_QUEUE_FORMAT_AQL) |
127 | m->cp_hqd_iq_rptr = 1; |
128 | |
129 | if (q->tba_addr) { |
130 | m->compute_tba_lo = lower_32_bits(q->tba_addr >> 8); |
131 | m->compute_tba_hi = upper_32_bits(q->tba_addr >> 8); |
132 | m->compute_tma_lo = lower_32_bits(q->tma_addr >> 8); |
133 | m->compute_tma_hi = upper_32_bits(q->tma_addr >> 8); |
134 | m->compute_pgm_rsrc2 |= |
135 | (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT); |
136 | } |
137 | |
138 | if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) { |
139 | m->cp_hqd_persistent_state |= |
140 | (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT); |
141 | m->cp_hqd_ctx_save_base_addr_lo = |
142 | lower_32_bits(q->ctx_save_restore_area_address); |
143 | m->cp_hqd_ctx_save_base_addr_hi = |
144 | upper_32_bits(q->ctx_save_restore_area_address); |
145 | m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size; |
146 | m->cp_hqd_cntl_stack_size = q->ctl_stack_size; |
147 | m->cp_hqd_cntl_stack_offset = q->ctl_stack_size; |
148 | m->cp_hqd_wg_state_offset = q->ctl_stack_size; |
149 | } |
150 | |
151 | *mqd = m; |
152 | if (gart_addr) |
153 | *gart_addr = addr; |
154 | mm->update_mqd(mm, m, q, NULL); |
155 | } |
156 | |
157 | static int load_mqd(struct mqd_manager *mm, void *mqd, |
158 | uint32_t pipe_id, uint32_t queue_id, |
159 | struct queue_properties *p, struct mm_struct *mms) |
160 | { |
161 | /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */ |
162 | uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0); |
163 | uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1); |
164 | |
165 | return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id, |
166 | (uint32_t __user *)p->write_ptr, |
167 | wptr_shift, wptr_mask, mms, 0); |
168 | } |
169 | |
170 | static void __update_mqd(struct mqd_manager *mm, void *mqd, |
171 | struct queue_properties *q, struct mqd_update_info *minfo, |
172 | unsigned int mtype, unsigned int atc_bit) |
173 | { |
174 | struct vi_mqd *m; |
175 | |
176 | m = get_mqd(mqd); |
177 | |
178 | m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT | |
179 | atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT | |
180 | mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT; |
181 | m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1; |
182 | pr_debug("cp_hqd_pq_control 0x%x\n" , m->cp_hqd_pq_control); |
183 | |
184 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); |
185 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); |
186 | |
187 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); |
188 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); |
189 | m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr); |
190 | m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); |
191 | |
192 | m->cp_hqd_pq_doorbell_control = |
193 | q->doorbell_off << |
194 | CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; |
195 | pr_debug("cp_hqd_pq_doorbell_control 0x%x\n" , |
196 | m->cp_hqd_pq_doorbell_control); |
197 | |
198 | m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT | |
199 | mtype << CP_HQD_EOP_CONTROL__MTYPE__SHIFT; |
200 | |
201 | m->cp_hqd_ib_control = atc_bit << CP_HQD_IB_CONTROL__IB_ATC__SHIFT | |
202 | 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT | |
203 | mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT; |
204 | |
205 | /* |
206 | * HW does not clamp this field correctly. Maximum EOP queue size |
207 | * is constrained by per-SE EOP done signal count, which is 8-bit. |
208 | * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit |
209 | * more than (EOP entry count - 1) so a queue size of 0x800 dwords |
210 | * is safe, giving a maximum field value of 0xA. |
211 | */ |
212 | m->cp_hqd_eop_control |= min(0xA, |
213 | order_base_2(q->eop_ring_buffer_size / 4) - 1); |
214 | m->cp_hqd_eop_base_addr_lo = |
215 | lower_32_bits(q->eop_ring_buffer_address >> 8); |
216 | m->cp_hqd_eop_base_addr_hi = |
217 | upper_32_bits(q->eop_ring_buffer_address >> 8); |
218 | |
219 | m->cp_hqd_iq_timer = atc_bit << CP_HQD_IQ_TIMER__IQ_ATC__SHIFT | |
220 | mtype << CP_HQD_IQ_TIMER__MTYPE__SHIFT; |
221 | |
222 | m->cp_hqd_vmid = q->vmid; |
223 | |
224 | if (q->format == KFD_QUEUE_FORMAT_AQL) { |
225 | m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK | |
226 | 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT; |
227 | } |
228 | |
229 | if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) |
230 | m->cp_hqd_ctx_save_control = |
231 | atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT | |
232 | mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT; |
233 | |
234 | update_cu_mask(mm, mqd, minfo); |
235 | set_priority(m, q); |
236 | |
237 | q->is_active = QUEUE_IS_ACTIVE(*q); |
238 | } |
239 | |
240 | static uint32_t read_doorbell_id(void *mqd) |
241 | { |
242 | struct vi_mqd *m = (struct vi_mqd *)mqd; |
243 | |
244 | return m->queue_doorbell_id0; |
245 | } |
246 | |
247 | static void update_mqd(struct mqd_manager *mm, void *mqd, |
248 | struct queue_properties *q, |
249 | struct mqd_update_info *minfo) |
250 | { |
251 | __update_mqd(mm, mqd, q, minfo, mtype: MTYPE_UC, atc_bit: 0); |
252 | } |
253 | |
254 | static int get_wave_state(struct mqd_manager *mm, void *mqd, |
255 | struct queue_properties *q, |
256 | void __user *ctl_stack, |
257 | u32 *ctl_stack_used_size, |
258 | u32 *save_area_used_size) |
259 | { |
260 | struct vi_mqd *m; |
261 | |
262 | m = get_mqd(mqd); |
263 | |
264 | *ctl_stack_used_size = m->cp_hqd_cntl_stack_size - |
265 | m->cp_hqd_cntl_stack_offset; |
266 | *save_area_used_size = m->cp_hqd_wg_state_offset - |
267 | m->cp_hqd_cntl_stack_size; |
268 | |
269 | /* Control stack is not copied to user mode for GFXv8 because |
270 | * it's part of the context save area that is already |
271 | * accessible to user mode |
272 | */ |
273 | |
274 | return 0; |
275 | } |
276 | |
277 | static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size) |
278 | { |
279 | /* Control stack is stored in user mode */ |
280 | *ctl_stack_size = 0; |
281 | } |
282 | |
283 | static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst) |
284 | { |
285 | struct vi_mqd *m; |
286 | |
287 | m = get_mqd(mqd); |
288 | |
289 | memcpy(mqd_dst, m, sizeof(struct vi_mqd)); |
290 | } |
291 | |
292 | static void restore_mqd(struct mqd_manager *mm, void **mqd, |
293 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
294 | struct queue_properties *qp, |
295 | const void *mqd_src, |
296 | const void *ctl_stack_src, const u32 ctl_stack_size) |
297 | { |
298 | uint64_t addr; |
299 | struct vi_mqd *m; |
300 | |
301 | m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr; |
302 | addr = mqd_mem_obj->gpu_addr; |
303 | |
304 | memcpy(m, mqd_src, sizeof(*m)); |
305 | |
306 | *mqd = m; |
307 | if (gart_addr) |
308 | *gart_addr = addr; |
309 | |
310 | m->cp_hqd_pq_doorbell_control = |
311 | qp->doorbell_off << |
312 | CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; |
313 | pr_debug("cp_hqd_pq_doorbell_control 0x%x\n" , |
314 | m->cp_hqd_pq_doorbell_control); |
315 | |
316 | qp->is_active = 0; |
317 | } |
318 | |
319 | static void init_mqd_hiq(struct mqd_manager *mm, void **mqd, |
320 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
321 | struct queue_properties *q) |
322 | { |
323 | struct vi_mqd *m; |
324 | |
325 | init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); |
326 | |
327 | m = get_mqd(mqd: *mqd); |
328 | |
329 | m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT | |
330 | 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT; |
331 | } |
332 | |
333 | static void update_mqd_hiq(struct mqd_manager *mm, void *mqd, |
334 | struct queue_properties *q, |
335 | struct mqd_update_info *minfo) |
336 | { |
337 | __update_mqd(mm, mqd, q, minfo, mtype: MTYPE_UC, atc_bit: 0); |
338 | } |
339 | |
340 | static void init_mqd_sdma(struct mqd_manager *mm, void **mqd, |
341 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
342 | struct queue_properties *q) |
343 | { |
344 | struct vi_sdma_mqd *m; |
345 | |
346 | m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr; |
347 | |
348 | memset(m, 0, sizeof(struct vi_sdma_mqd)); |
349 | |
350 | *mqd = m; |
351 | if (gart_addr) |
352 | *gart_addr = mqd_mem_obj->gpu_addr; |
353 | |
354 | mm->update_mqd(mm, m, q, NULL); |
355 | } |
356 | |
357 | static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, |
358 | struct queue_properties *q, |
359 | struct mqd_update_info *minfo) |
360 | { |
361 | struct vi_sdma_mqd *m; |
362 | |
363 | m = get_sdma_mqd(mqd); |
364 | m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4) |
365 | << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | |
366 | q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT | |
367 | 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | |
368 | 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; |
369 | |
370 | m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8); |
371 | m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); |
372 | m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); |
373 | m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); |
374 | m->sdmax_rlcx_doorbell = |
375 | q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; |
376 | |
377 | m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr; |
378 | |
379 | m->sdma_engine_id = q->sdma_engine_id; |
380 | m->sdma_queue_id = q->sdma_queue_id; |
381 | |
382 | q->is_active = QUEUE_IS_ACTIVE(*q); |
383 | } |
384 | |
385 | static void checkpoint_mqd_sdma(struct mqd_manager *mm, |
386 | void *mqd, |
387 | void *mqd_dst, |
388 | void *ctl_stack_dst) |
389 | { |
390 | struct vi_sdma_mqd *m; |
391 | |
392 | m = get_sdma_mqd(mqd); |
393 | |
394 | memcpy(mqd_dst, m, sizeof(struct vi_sdma_mqd)); |
395 | } |
396 | |
397 | static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd, |
398 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, |
399 | struct queue_properties *qp, |
400 | const void *mqd_src, |
401 | const void *ctl_stack_src, const u32 ctl_stack_size) |
402 | { |
403 | uint64_t addr; |
404 | struct vi_sdma_mqd *m; |
405 | |
406 | m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr; |
407 | addr = mqd_mem_obj->gpu_addr; |
408 | |
409 | memcpy(m, mqd_src, sizeof(*m)); |
410 | |
411 | m->sdmax_rlcx_doorbell = |
412 | qp->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; |
413 | |
414 | *mqd = m; |
415 | if (gart_addr) |
416 | *gart_addr = addr; |
417 | |
418 | qp->is_active = 0; |
419 | } |
420 | |
421 | #if defined(CONFIG_DEBUG_FS) |
422 | |
423 | |
424 | static int debugfs_show_mqd(struct seq_file *m, void *data) |
425 | { |
426 | seq_hex_dump(m, prefix_str: " " , prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4, |
427 | buf: data, len: sizeof(struct vi_mqd), ascii: false); |
428 | return 0; |
429 | } |
430 | |
431 | static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) |
432 | { |
433 | seq_hex_dump(m, prefix_str: " " , prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4, |
434 | buf: data, len: sizeof(struct vi_sdma_mqd), ascii: false); |
435 | return 0; |
436 | } |
437 | |
438 | #endif |
439 | |
440 | struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, |
441 | struct kfd_node *dev) |
442 | { |
443 | struct mqd_manager *mqd; |
444 | |
445 | if (WARN_ON(type >= KFD_MQD_TYPE_MAX)) |
446 | return NULL; |
447 | |
448 | mqd = kzalloc(size: sizeof(*mqd), GFP_KERNEL); |
449 | if (!mqd) |
450 | return NULL; |
451 | |
452 | mqd->dev = dev; |
453 | |
454 | switch (type) { |
455 | case KFD_MQD_TYPE_CP: |
456 | mqd->allocate_mqd = allocate_mqd; |
457 | mqd->init_mqd = init_mqd; |
458 | mqd->free_mqd = kfd_free_mqd_cp; |
459 | mqd->load_mqd = load_mqd; |
460 | mqd->update_mqd = update_mqd; |
461 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
462 | mqd->is_occupied = kfd_is_occupied_cp; |
463 | mqd->get_wave_state = get_wave_state; |
464 | mqd->get_checkpoint_info = get_checkpoint_info; |
465 | mqd->checkpoint_mqd = checkpoint_mqd; |
466 | mqd->restore_mqd = restore_mqd; |
467 | mqd->mqd_size = sizeof(struct vi_mqd); |
468 | #if defined(CONFIG_DEBUG_FS) |
469 | mqd->debugfs_show_mqd = debugfs_show_mqd; |
470 | #endif |
471 | break; |
472 | case KFD_MQD_TYPE_HIQ: |
473 | mqd->allocate_mqd = allocate_hiq_mqd; |
474 | mqd->init_mqd = init_mqd_hiq; |
475 | mqd->free_mqd = free_mqd_hiq_sdma; |
476 | mqd->load_mqd = load_mqd; |
477 | mqd->update_mqd = update_mqd_hiq; |
478 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
479 | mqd->is_occupied = kfd_is_occupied_cp; |
480 | mqd->mqd_size = sizeof(struct vi_mqd); |
481 | mqd->mqd_stride = kfd_mqd_stride; |
482 | #if defined(CONFIG_DEBUG_FS) |
483 | mqd->debugfs_show_mqd = debugfs_show_mqd; |
484 | #endif |
485 | mqd->read_doorbell_id = read_doorbell_id; |
486 | break; |
487 | case KFD_MQD_TYPE_DIQ: |
488 | mqd->allocate_mqd = allocate_mqd; |
489 | mqd->init_mqd = init_mqd_hiq; |
490 | mqd->free_mqd = kfd_free_mqd_cp; |
491 | mqd->load_mqd = load_mqd; |
492 | mqd->update_mqd = update_mqd_hiq; |
493 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
494 | mqd->is_occupied = kfd_is_occupied_cp; |
495 | mqd->mqd_size = sizeof(struct vi_mqd); |
496 | mqd->mqd_stride = kfd_mqd_stride; |
497 | #if defined(CONFIG_DEBUG_FS) |
498 | mqd->debugfs_show_mqd = debugfs_show_mqd; |
499 | #endif |
500 | break; |
501 | case KFD_MQD_TYPE_SDMA: |
502 | mqd->allocate_mqd = allocate_sdma_mqd; |
503 | mqd->init_mqd = init_mqd_sdma; |
504 | mqd->free_mqd = free_mqd_hiq_sdma; |
505 | mqd->load_mqd = kfd_load_mqd_sdma; |
506 | mqd->update_mqd = update_mqd_sdma; |
507 | mqd->destroy_mqd = kfd_destroy_mqd_sdma; |
508 | mqd->is_occupied = kfd_is_occupied_sdma; |
509 | mqd->checkpoint_mqd = checkpoint_mqd_sdma; |
510 | mqd->restore_mqd = restore_mqd_sdma; |
511 | mqd->mqd_size = sizeof(struct vi_sdma_mqd); |
512 | mqd->mqd_stride = kfd_mqd_stride; |
513 | #if defined(CONFIG_DEBUG_FS) |
514 | mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; |
515 | #endif |
516 | break; |
517 | default: |
518 | kfree(objp: mqd); |
519 | return NULL; |
520 | } |
521 | |
522 | return mqd; |
523 | } |
524 | |