1 | /* |
2 | * Copyright 2019 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/firmware.h> |
25 | #include <linux/module.h> |
26 | #include "amdgpu.h" |
27 | #include "soc15_common.h" |
28 | #include "soc21.h" |
29 | #include "gc/gc_11_0_0_offset.h" |
30 | #include "gc/gc_11_0_0_sh_mask.h" |
31 | #include "gc/gc_11_0_0_default.h" |
32 | #include "v11_structs.h" |
33 | #include "mes_v11_api_def.h" |
34 | |
35 | MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin" ); |
36 | MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes_2.bin" ); |
37 | MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin" ); |
38 | MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin" ); |
39 | MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes_2.bin" ); |
40 | MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin" ); |
41 | MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin" ); |
42 | MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes_2.bin" ); |
43 | MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin" ); |
44 | MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin" ); |
45 | MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes_2.bin" ); |
46 | MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin" ); |
47 | MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes.bin" ); |
48 | MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes_2.bin" ); |
49 | MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes1.bin" ); |
50 | MODULE_FIRMWARE("amdgpu/gc_11_5_0_mes_2.bin" ); |
51 | MODULE_FIRMWARE("amdgpu/gc_11_5_0_mes1.bin" ); |
52 | MODULE_FIRMWARE("amdgpu/gc_11_5_1_mes_2.bin" ); |
53 | MODULE_FIRMWARE("amdgpu/gc_11_5_1_mes1.bin" ); |
54 | |
55 | |
56 | static int mes_v11_0_hw_fini(void *handle); |
57 | static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev); |
58 | static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev); |
59 | |
60 | #define MES_EOP_SIZE 2048 |
61 | #define GFX_MES_DRAM_SIZE 0x80000 |
62 | |
63 | static void mes_v11_0_ring_set_wptr(struct amdgpu_ring *ring) |
64 | { |
65 | struct amdgpu_device *adev = ring->adev; |
66 | |
67 | if (ring->use_doorbell) { |
68 | atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, |
69 | i: ring->wptr); |
70 | WDOORBELL64(ring->doorbell_index, ring->wptr); |
71 | } else { |
72 | BUG(); |
73 | } |
74 | } |
75 | |
76 | static u64 mes_v11_0_ring_get_rptr(struct amdgpu_ring *ring) |
77 | { |
78 | return *ring->rptr_cpu_addr; |
79 | } |
80 | |
81 | static u64 mes_v11_0_ring_get_wptr(struct amdgpu_ring *ring) |
82 | { |
83 | u64 wptr; |
84 | |
85 | if (ring->use_doorbell) |
86 | wptr = atomic64_read(v: (atomic64_t *)ring->wptr_cpu_addr); |
87 | else |
88 | BUG(); |
89 | return wptr; |
90 | } |
91 | |
92 | static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = { |
93 | .type = AMDGPU_RING_TYPE_MES, |
94 | .align_mask = 1, |
95 | .nop = 0, |
96 | .support_64bit_ptrs = true, |
97 | .get_rptr = mes_v11_0_ring_get_rptr, |
98 | .get_wptr = mes_v11_0_ring_get_wptr, |
99 | .set_wptr = mes_v11_0_ring_set_wptr, |
100 | .insert_nop = amdgpu_ring_insert_nop, |
101 | }; |
102 | |
103 | static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, |
104 | void *pkt, int size, |
105 | int api_status_off) |
106 | { |
107 | int ndw = size / 4; |
108 | signed long r; |
109 | union MESAPI__ADD_QUEUE *x_pkt = pkt; |
110 | struct MES_API_STATUS *api_status; |
111 | struct amdgpu_device *adev = mes->adev; |
112 | struct amdgpu_ring *ring = &mes->ring; |
113 | unsigned long flags; |
114 | signed long timeout = adev->usec_timeout; |
115 | |
116 | if (amdgpu_emu_mode) { |
117 | timeout *= 100; |
118 | } else if (amdgpu_sriov_vf(adev)) { |
119 | /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */ |
120 | timeout = 15 * 600 * 1000; |
121 | } |
122 | BUG_ON(size % 4 != 0); |
123 | |
124 | spin_lock_irqsave(&mes->ring_lock, flags); |
125 | if (amdgpu_ring_alloc(ring, ndw)) { |
126 | spin_unlock_irqrestore(lock: &mes->ring_lock, flags); |
127 | return -ENOMEM; |
128 | } |
129 | |
130 | api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off); |
131 | api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr; |
132 | api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq; |
133 | |
134 | amdgpu_ring_write_multiple(ring, src: pkt, count_dw: ndw); |
135 | amdgpu_ring_commit(ring); |
136 | spin_unlock_irqrestore(lock: &mes->ring_lock, flags); |
137 | |
138 | DRM_DEBUG("MES msg=%d was emitted\n" , x_pkt->header.opcode); |
139 | |
140 | r = amdgpu_fence_wait_polling(ring, wait_seq: ring->fence_drv.sync_seq, |
141 | timeout); |
142 | if (r < 1) { |
143 | DRM_ERROR("MES failed to response msg=%d\n" , |
144 | x_pkt->header.opcode); |
145 | |
146 | while (halt_if_hws_hang) |
147 | schedule(); |
148 | |
149 | return -ETIMEDOUT; |
150 | } |
151 | |
152 | return 0; |
153 | } |
154 | |
155 | static int convert_to_mes_queue_type(int queue_type) |
156 | { |
157 | if (queue_type == AMDGPU_RING_TYPE_GFX) |
158 | return MES_QUEUE_TYPE_GFX; |
159 | else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) |
160 | return MES_QUEUE_TYPE_COMPUTE; |
161 | else if (queue_type == AMDGPU_RING_TYPE_SDMA) |
162 | return MES_QUEUE_TYPE_SDMA; |
163 | else |
164 | BUG(); |
165 | return -1; |
166 | } |
167 | |
168 | static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes, |
169 | struct mes_add_queue_input *input) |
170 | { |
171 | struct amdgpu_device *adev = mes->adev; |
172 | union MESAPI__ADD_QUEUE mes_add_queue_pkt; |
173 | struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; |
174 | uint32_t vm_cntx_cntl = hub->vm_cntx_cntl; |
175 | |
176 | memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt)); |
177 | |
178 | mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER; |
179 | mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE; |
180 | mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; |
181 | |
182 | mes_add_queue_pkt.process_id = input->process_id; |
183 | mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr; |
184 | mes_add_queue_pkt.process_va_start = input->process_va_start; |
185 | mes_add_queue_pkt.process_va_end = input->process_va_end; |
186 | mes_add_queue_pkt.process_quantum = input->process_quantum; |
187 | mes_add_queue_pkt.process_context_addr = input->process_context_addr; |
188 | mes_add_queue_pkt.gang_quantum = input->gang_quantum; |
189 | mes_add_queue_pkt.gang_context_addr = input->gang_context_addr; |
190 | mes_add_queue_pkt.inprocess_gang_priority = |
191 | input->inprocess_gang_priority; |
192 | mes_add_queue_pkt.gang_global_priority_level = |
193 | input->gang_global_priority_level; |
194 | mes_add_queue_pkt.doorbell_offset = input->doorbell_offset; |
195 | mes_add_queue_pkt.mqd_addr = input->mqd_addr; |
196 | |
197 | if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> |
198 | AMDGPU_MES_API_VERSION_SHIFT) >= 2) |
199 | mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr; |
200 | else |
201 | mes_add_queue_pkt.wptr_addr = input->wptr_addr; |
202 | |
203 | mes_add_queue_pkt.queue_type = |
204 | convert_to_mes_queue_type(queue_type: input->queue_type); |
205 | mes_add_queue_pkt.paging = input->paging; |
206 | mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl; |
207 | mes_add_queue_pkt.gws_base = input->gws_base; |
208 | mes_add_queue_pkt.gws_size = input->gws_size; |
209 | mes_add_queue_pkt.trap_handler_addr = input->tba_addr; |
210 | mes_add_queue_pkt.tma_addr = input->tma_addr; |
211 | mes_add_queue_pkt.trap_en = input->trap_en; |
212 | mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear; |
213 | mes_add_queue_pkt.is_kfd_process = input->is_kfd_process; |
214 | |
215 | /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */ |
216 | mes_add_queue_pkt.is_aql_queue = input->is_aql_queue; |
217 | mes_add_queue_pkt.gds_size = input->queue_size; |
218 | |
219 | mes_add_queue_pkt.exclusively_scheduled = input->exclusively_scheduled; |
220 | |
221 | return mes_v11_0_submit_pkt_and_poll_completion(mes, |
222 | pkt: &mes_add_queue_pkt, size: sizeof(mes_add_queue_pkt), |
223 | offsetof(union MESAPI__ADD_QUEUE, api_status)); |
224 | } |
225 | |
226 | static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes, |
227 | struct mes_remove_queue_input *input) |
228 | { |
229 | union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt; |
230 | |
231 | memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt)); |
232 | |
233 | mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER; |
234 | mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE; |
235 | mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; |
236 | |
237 | mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; |
238 | mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr; |
239 | |
240 | return mes_v11_0_submit_pkt_and_poll_completion(mes, |
241 | pkt: &mes_remove_queue_pkt, size: sizeof(mes_remove_queue_pkt), |
242 | offsetof(union MESAPI__REMOVE_QUEUE, api_status)); |
243 | } |
244 | |
245 | static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes, |
246 | struct mes_unmap_legacy_queue_input *input) |
247 | { |
248 | union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt; |
249 | |
250 | memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt)); |
251 | |
252 | mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER; |
253 | mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE; |
254 | mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; |
255 | |
256 | mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; |
257 | mes_remove_queue_pkt.gang_context_addr = 0; |
258 | |
259 | mes_remove_queue_pkt.pipe_id = input->pipe_id; |
260 | mes_remove_queue_pkt.queue_id = input->queue_id; |
261 | |
262 | if (input->action == PREEMPT_QUEUES_NO_UNMAP) { |
263 | mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1; |
264 | mes_remove_queue_pkt.tf_addr = input->trail_fence_addr; |
265 | mes_remove_queue_pkt.tf_data = |
266 | lower_32_bits(input->trail_fence_data); |
267 | } else { |
268 | mes_remove_queue_pkt.unmap_legacy_queue = 1; |
269 | mes_remove_queue_pkt.queue_type = |
270 | convert_to_mes_queue_type(queue_type: input->queue_type); |
271 | } |
272 | |
273 | return mes_v11_0_submit_pkt_and_poll_completion(mes, |
274 | pkt: &mes_remove_queue_pkt, size: sizeof(mes_remove_queue_pkt), |
275 | offsetof(union MESAPI__REMOVE_QUEUE, api_status)); |
276 | } |
277 | |
278 | static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes, |
279 | struct mes_suspend_gang_input *input) |
280 | { |
281 | return 0; |
282 | } |
283 | |
284 | static int mes_v11_0_resume_gang(struct amdgpu_mes *mes, |
285 | struct mes_resume_gang_input *input) |
286 | { |
287 | return 0; |
288 | } |
289 | |
290 | static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes) |
291 | { |
292 | union MESAPI__QUERY_MES_STATUS mes_status_pkt; |
293 | |
294 | memset(&mes_status_pkt, 0, sizeof(mes_status_pkt)); |
295 | |
296 | mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER; |
297 | mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS; |
298 | mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; |
299 | |
300 | return mes_v11_0_submit_pkt_and_poll_completion(mes, |
301 | pkt: &mes_status_pkt, size: sizeof(mes_status_pkt), |
302 | offsetof(union MESAPI__QUERY_MES_STATUS, api_status)); |
303 | } |
304 | |
305 | static int mes_v11_0_misc_op(struct amdgpu_mes *mes, |
306 | struct mes_misc_op_input *input) |
307 | { |
308 | union MESAPI__MISC misc_pkt; |
309 | |
310 | memset(&misc_pkt, 0, sizeof(misc_pkt)); |
311 | |
312 | misc_pkt.header.type = MES_API_TYPE_SCHEDULER; |
313 | misc_pkt.header.opcode = MES_SCH_API_MISC; |
314 | misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; |
315 | |
316 | switch (input->op) { |
317 | case MES_MISC_OP_READ_REG: |
318 | misc_pkt.opcode = MESAPI_MISC__READ_REG; |
319 | misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset; |
320 | misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr; |
321 | break; |
322 | case MES_MISC_OP_WRITE_REG: |
323 | misc_pkt.opcode = MESAPI_MISC__WRITE_REG; |
324 | misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset; |
325 | misc_pkt.write_reg.reg_value = input->write_reg.reg_value; |
326 | break; |
327 | case MES_MISC_OP_WRM_REG_WAIT: |
328 | misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM; |
329 | misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM; |
330 | misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref; |
331 | misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask; |
332 | misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0; |
333 | misc_pkt.wait_reg_mem.reg_offset2 = 0; |
334 | break; |
335 | case MES_MISC_OP_WRM_REG_WR_WAIT: |
336 | misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM; |
337 | misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG; |
338 | misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref; |
339 | misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask; |
340 | misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0; |
341 | misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1; |
342 | break; |
343 | case MES_MISC_OP_SET_SHADER_DEBUGGER: |
344 | misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER; |
345 | misc_pkt.set_shader_debugger.process_context_addr = |
346 | input->set_shader_debugger.process_context_addr; |
347 | misc_pkt.set_shader_debugger.flags.u32all = |
348 | input->set_shader_debugger.flags.u32all; |
349 | misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl = |
350 | input->set_shader_debugger.spi_gdbg_per_vmid_cntl; |
351 | memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl, |
352 | input->set_shader_debugger.tcp_watch_cntl, |
353 | sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl)); |
354 | misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en; |
355 | break; |
356 | default: |
357 | DRM_ERROR("unsupported misc op (%d) \n" , input->op); |
358 | return -EINVAL; |
359 | } |
360 | |
361 | return mes_v11_0_submit_pkt_and_poll_completion(mes, |
362 | pkt: &misc_pkt, size: sizeof(misc_pkt), |
363 | offsetof(union MESAPI__MISC, api_status)); |
364 | } |
365 | |
366 | static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes) |
367 | { |
368 | int i; |
369 | struct amdgpu_device *adev = mes->adev; |
370 | union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt; |
371 | |
372 | memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt)); |
373 | |
374 | mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER; |
375 | mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC; |
376 | mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; |
377 | |
378 | mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub; |
379 | mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub; |
380 | mes_set_hw_res_pkt.gds_size = adev->gds.gds_size; |
381 | mes_set_hw_res_pkt.paging_vmid = 0; |
382 | mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr; |
383 | mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr = |
384 | mes->query_status_fence_gpu_addr; |
385 | |
386 | for (i = 0; i < MAX_COMPUTE_PIPES; i++) |
387 | mes_set_hw_res_pkt.compute_hqd_mask[i] = |
388 | mes->compute_hqd_mask[i]; |
389 | |
390 | for (i = 0; i < MAX_GFX_PIPES; i++) |
391 | mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i]; |
392 | |
393 | for (i = 0; i < MAX_SDMA_PIPES; i++) |
394 | mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i]; |
395 | |
396 | for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++) |
397 | mes_set_hw_res_pkt.aggregated_doorbells[i] = |
398 | mes->aggregated_doorbells[i]; |
399 | |
400 | for (i = 0; i < 5; i++) { |
401 | mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i]; |
402 | mes_set_hw_res_pkt.mmhub_base[i] = |
403 | adev->reg_offset[MMHUB_HWIP][0][i]; |
404 | mes_set_hw_res_pkt.osssys_base[i] = |
405 | adev->reg_offset[OSSSYS_HWIP][0][i]; |
406 | } |
407 | |
408 | mes_set_hw_res_pkt.disable_reset = 1; |
409 | mes_set_hw_res_pkt.disable_mes_log = 1; |
410 | mes_set_hw_res_pkt.use_different_vmid_compute = 1; |
411 | mes_set_hw_res_pkt.enable_reg_active_poll = 1; |
412 | mes_set_hw_res_pkt.enable_level_process_quantum_check = 1; |
413 | mes_set_hw_res_pkt.oversubscription_timer = 50; |
414 | if (amdgpu_mes_log_enable) { |
415 | mes_set_hw_res_pkt.enable_mes_event_int_logging = 1; |
416 | mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = |
417 | mes->event_log_gpu_addr; |
418 | } |
419 | |
420 | return mes_v11_0_submit_pkt_and_poll_completion(mes, |
421 | pkt: &mes_set_hw_res_pkt, size: sizeof(mes_set_hw_res_pkt), |
422 | offsetof(union MESAPI_SET_HW_RESOURCES, api_status)); |
423 | } |
424 | |
425 | static const struct amdgpu_mes_funcs mes_v11_0_funcs = { |
426 | .add_hw_queue = mes_v11_0_add_hw_queue, |
427 | .remove_hw_queue = mes_v11_0_remove_hw_queue, |
428 | .unmap_legacy_queue = mes_v11_0_unmap_legacy_queue, |
429 | .suspend_gang = mes_v11_0_suspend_gang, |
430 | .resume_gang = mes_v11_0_resume_gang, |
431 | .misc_op = mes_v11_0_misc_op, |
432 | }; |
433 | |
434 | static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev, |
435 | enum admgpu_mes_pipe pipe) |
436 | { |
437 | int r; |
438 | const struct mes_firmware_header_v1_0 *mes_hdr; |
439 | const __le32 *fw_data; |
440 | unsigned fw_size; |
441 | |
442 | mes_hdr = (const struct mes_firmware_header_v1_0 *) |
443 | adev->mes.fw[pipe]->data; |
444 | |
445 | fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + |
446 | le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); |
447 | fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); |
448 | |
449 | r = amdgpu_bo_create_reserved(adev, size: fw_size, |
450 | PAGE_SIZE, |
451 | AMDGPU_GEM_DOMAIN_VRAM | |
452 | AMDGPU_GEM_DOMAIN_GTT, |
453 | bo_ptr: &adev->mes.ucode_fw_obj[pipe], |
454 | gpu_addr: &adev->mes.ucode_fw_gpu_addr[pipe], |
455 | cpu_addr: (void **)&adev->mes.ucode_fw_ptr[pipe]); |
456 | if (r) { |
457 | dev_err(adev->dev, "(%d) failed to create mes fw bo\n" , r); |
458 | return r; |
459 | } |
460 | |
461 | memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); |
462 | |
463 | amdgpu_bo_kunmap(bo: adev->mes.ucode_fw_obj[pipe]); |
464 | amdgpu_bo_unreserve(bo: adev->mes.ucode_fw_obj[pipe]); |
465 | |
466 | return 0; |
467 | } |
468 | |
469 | static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev, |
470 | enum admgpu_mes_pipe pipe) |
471 | { |
472 | int r; |
473 | const struct mes_firmware_header_v1_0 *mes_hdr; |
474 | const __le32 *fw_data; |
475 | unsigned fw_size; |
476 | |
477 | mes_hdr = (const struct mes_firmware_header_v1_0 *) |
478 | adev->mes.fw[pipe]->data; |
479 | |
480 | fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + |
481 | le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); |
482 | fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); |
483 | |
484 | if (fw_size > GFX_MES_DRAM_SIZE) { |
485 | dev_err(adev->dev, "PIPE%d ucode data fw size (%d) is greater than dram size (%d)\n" , |
486 | pipe, fw_size, GFX_MES_DRAM_SIZE); |
487 | return -EINVAL; |
488 | } |
489 | |
490 | r = amdgpu_bo_create_reserved(adev, GFX_MES_DRAM_SIZE, |
491 | align: 64 * 1024, |
492 | AMDGPU_GEM_DOMAIN_VRAM | |
493 | AMDGPU_GEM_DOMAIN_GTT, |
494 | bo_ptr: &adev->mes.data_fw_obj[pipe], |
495 | gpu_addr: &adev->mes.data_fw_gpu_addr[pipe], |
496 | cpu_addr: (void **)&adev->mes.data_fw_ptr[pipe]); |
497 | if (r) { |
498 | dev_err(adev->dev, "(%d) failed to create mes data fw bo\n" , r); |
499 | return r; |
500 | } |
501 | |
502 | memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size); |
503 | |
504 | amdgpu_bo_kunmap(bo: adev->mes.data_fw_obj[pipe]); |
505 | amdgpu_bo_unreserve(bo: adev->mes.data_fw_obj[pipe]); |
506 | |
507 | return 0; |
508 | } |
509 | |
510 | static void mes_v11_0_free_ucode_buffers(struct amdgpu_device *adev, |
511 | enum admgpu_mes_pipe pipe) |
512 | { |
513 | amdgpu_bo_free_kernel(bo: &adev->mes.data_fw_obj[pipe], |
514 | gpu_addr: &adev->mes.data_fw_gpu_addr[pipe], |
515 | cpu_addr: (void **)&adev->mes.data_fw_ptr[pipe]); |
516 | |
517 | amdgpu_bo_free_kernel(bo: &adev->mes.ucode_fw_obj[pipe], |
518 | gpu_addr: &adev->mes.ucode_fw_gpu_addr[pipe], |
519 | cpu_addr: (void **)&adev->mes.ucode_fw_ptr[pipe]); |
520 | } |
521 | |
522 | static void mes_v11_0_enable(struct amdgpu_device *adev, bool enable) |
523 | { |
524 | uint64_t ucode_addr; |
525 | uint32_t pipe, data = 0; |
526 | |
527 | if (enable) { |
528 | data = RREG32_SOC15(GC, 0, regCP_MES_CNTL); |
529 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1); |
530 | data = REG_SET_FIELD(data, CP_MES_CNTL, |
531 | MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0); |
532 | WREG32_SOC15(GC, 0, regCP_MES_CNTL, data); |
533 | |
534 | mutex_lock(&adev->srbm_mutex); |
535 | for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { |
536 | if (!adev->enable_mes_kiq && |
537 | pipe == AMDGPU_MES_KIQ_PIPE) |
538 | continue; |
539 | |
540 | soc21_grbm_select(adev, me: 3, pipe, queue: 0, vmid: 0); |
541 | |
542 | ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; |
543 | WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START, |
544 | lower_32_bits(ucode_addr)); |
545 | WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI, |
546 | upper_32_bits(ucode_addr)); |
547 | } |
548 | soc21_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0); |
549 | mutex_unlock(lock: &adev->srbm_mutex); |
550 | |
551 | /* unhalt MES and activate pipe0 */ |
552 | data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1); |
553 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, |
554 | adev->enable_mes_kiq ? 1 : 0); |
555 | WREG32_SOC15(GC, 0, regCP_MES_CNTL, data); |
556 | |
557 | if (amdgpu_emu_mode) |
558 | msleep(msecs: 100); |
559 | else |
560 | udelay(50); |
561 | } else { |
562 | data = RREG32_SOC15(GC, 0, regCP_MES_CNTL); |
563 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0); |
564 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0); |
565 | data = REG_SET_FIELD(data, CP_MES_CNTL, |
566 | MES_INVALIDATE_ICACHE, 1); |
567 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1); |
568 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, |
569 | adev->enable_mes_kiq ? 1 : 0); |
570 | data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1); |
571 | WREG32_SOC15(GC, 0, regCP_MES_CNTL, data); |
572 | } |
573 | } |
574 | |
575 | /* This function is for backdoor MES firmware */ |
576 | static int mes_v11_0_load_microcode(struct amdgpu_device *adev, |
577 | enum admgpu_mes_pipe pipe, bool prime_icache) |
578 | { |
579 | int r; |
580 | uint32_t data; |
581 | uint64_t ucode_addr; |
582 | |
583 | mes_v11_0_enable(adev, enable: false); |
584 | |
585 | if (!adev->mes.fw[pipe]) |
586 | return -EINVAL; |
587 | |
588 | r = mes_v11_0_allocate_ucode_buffer(adev, pipe); |
589 | if (r) |
590 | return r; |
591 | |
592 | r = mes_v11_0_allocate_ucode_data_buffer(adev, pipe); |
593 | if (r) { |
594 | mes_v11_0_free_ucode_buffers(adev, pipe); |
595 | return r; |
596 | } |
597 | |
598 | mutex_lock(&adev->srbm_mutex); |
599 | /* me=3, pipe=0, queue=0 */ |
600 | soc21_grbm_select(adev, me: 3, pipe, queue: 0, vmid: 0); |
601 | |
602 | WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0); |
603 | |
604 | /* set ucode start address */ |
605 | ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; |
606 | WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START, |
607 | lower_32_bits(ucode_addr)); |
608 | WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI, |
609 | upper_32_bits(ucode_addr)); |
610 | |
611 | /* set ucode fimrware address */ |
612 | WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO, |
613 | lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe])); |
614 | WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI, |
615 | upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe])); |
616 | |
617 | /* set ucode instruction cache boundary to 2M-1 */ |
618 | WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF); |
619 | |
620 | /* set ucode data firmware address */ |
621 | WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO, |
622 | lower_32_bits(adev->mes.data_fw_gpu_addr[pipe])); |
623 | WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI, |
624 | upper_32_bits(adev->mes.data_fw_gpu_addr[pipe])); |
625 | |
626 | /* Set 0x7FFFF (512K-1) to CP_MES_MDBOUND_LO */ |
627 | WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF); |
628 | |
629 | if (prime_icache) { |
630 | /* invalidate ICACHE */ |
631 | data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL); |
632 | data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0); |
633 | data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1); |
634 | WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data); |
635 | |
636 | /* prime the ICACHE. */ |
637 | data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL); |
638 | data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1); |
639 | WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data); |
640 | } |
641 | |
642 | soc21_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0); |
643 | mutex_unlock(lock: &adev->srbm_mutex); |
644 | |
645 | return 0; |
646 | } |
647 | |
648 | static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev, |
649 | enum admgpu_mes_pipe pipe) |
650 | { |
651 | int r; |
652 | u32 *eop; |
653 | |
654 | r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE, |
655 | AMDGPU_GEM_DOMAIN_GTT, |
656 | bo_ptr: &adev->mes.eop_gpu_obj[pipe], |
657 | gpu_addr: &adev->mes.eop_gpu_addr[pipe], |
658 | cpu_addr: (void **)&eop); |
659 | if (r) { |
660 | dev_warn(adev->dev, "(%d) create EOP bo failed\n" , r); |
661 | return r; |
662 | } |
663 | |
664 | memset(eop, 0, |
665 | adev->mes.eop_gpu_obj[pipe]->tbo.base.size); |
666 | |
667 | amdgpu_bo_kunmap(bo: adev->mes.eop_gpu_obj[pipe]); |
668 | amdgpu_bo_unreserve(bo: adev->mes.eop_gpu_obj[pipe]); |
669 | |
670 | return 0; |
671 | } |
672 | |
673 | static int mes_v11_0_mqd_init(struct amdgpu_ring *ring) |
674 | { |
675 | struct v11_compute_mqd *mqd = ring->mqd_ptr; |
676 | uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; |
677 | uint32_t tmp; |
678 | |
679 | memset(mqd, 0, sizeof(*mqd)); |
680 | |
681 | mqd->header = 0xC0310800; |
682 | mqd->compute_pipelinestat_enable = 0x00000001; |
683 | mqd->compute_static_thread_mgmt_se0 = 0xffffffff; |
684 | mqd->compute_static_thread_mgmt_se1 = 0xffffffff; |
685 | mqd->compute_static_thread_mgmt_se2 = 0xffffffff; |
686 | mqd->compute_static_thread_mgmt_se3 = 0xffffffff; |
687 | mqd->compute_misc_reserved = 0x00000007; |
688 | |
689 | eop_base_addr = ring->eop_gpu_addr >> 8; |
690 | |
691 | /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ |
692 | tmp = regCP_HQD_EOP_CONTROL_DEFAULT; |
693 | tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, |
694 | (order_base_2(MES_EOP_SIZE / 4) - 1)); |
695 | |
696 | mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr); |
697 | mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); |
698 | mqd->cp_hqd_eop_control = tmp; |
699 | |
700 | /* disable the queue if it's active */ |
701 | ring->wptr = 0; |
702 | mqd->cp_hqd_pq_rptr = 0; |
703 | mqd->cp_hqd_pq_wptr_lo = 0; |
704 | mqd->cp_hqd_pq_wptr_hi = 0; |
705 | |
706 | /* set the pointer to the MQD */ |
707 | mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; |
708 | mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); |
709 | |
710 | /* set MQD vmid to 0 */ |
711 | tmp = regCP_MQD_CONTROL_DEFAULT; |
712 | tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); |
713 | mqd->cp_mqd_control = tmp; |
714 | |
715 | /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ |
716 | hqd_gpu_addr = ring->gpu_addr >> 8; |
717 | mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr); |
718 | mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); |
719 | |
720 | /* set the wb address whether it's enabled or not */ |
721 | wb_gpu_addr = ring->rptr_gpu_addr; |
722 | mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; |
723 | mqd->cp_hqd_pq_rptr_report_addr_hi = |
724 | upper_32_bits(wb_gpu_addr) & 0xffff; |
725 | |
726 | /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ |
727 | wb_gpu_addr = ring->wptr_gpu_addr; |
728 | mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8; |
729 | mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; |
730 | |
731 | /* set up the HQD, this is similar to CP_RB0_CNTL */ |
732 | tmp = regCP_HQD_PQ_CONTROL_DEFAULT; |
733 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, |
734 | (order_base_2(ring->ring_size / 4) - 1)); |
735 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, |
736 | ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); |
737 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1); |
738 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0); |
739 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); |
740 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); |
741 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1); |
742 | mqd->cp_hqd_pq_control = tmp; |
743 | |
744 | /* enable doorbell */ |
745 | tmp = 0; |
746 | if (ring->use_doorbell) { |
747 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, |
748 | DOORBELL_OFFSET, ring->doorbell_index); |
749 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, |
750 | DOORBELL_EN, 1); |
751 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, |
752 | DOORBELL_SOURCE, 0); |
753 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, |
754 | DOORBELL_HIT, 0); |
755 | } else |
756 | tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, |
757 | DOORBELL_EN, 0); |
758 | mqd->cp_hqd_pq_doorbell_control = tmp; |
759 | |
760 | mqd->cp_hqd_vmid = 0; |
761 | /* activate the queue */ |
762 | mqd->cp_hqd_active = 1; |
763 | |
764 | tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; |
765 | tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, |
766 | PRELOAD_SIZE, 0x55); |
767 | mqd->cp_hqd_persistent_state = tmp; |
768 | |
769 | mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT; |
770 | mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT; |
771 | mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT; |
772 | |
773 | amdgpu_device_flush_hdp(adev: ring->adev, NULL); |
774 | return 0; |
775 | } |
776 | |
777 | static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring) |
778 | { |
779 | struct v11_compute_mqd *mqd = ring->mqd_ptr; |
780 | struct amdgpu_device *adev = ring->adev; |
781 | uint32_t data = 0; |
782 | |
783 | mutex_lock(&adev->srbm_mutex); |
784 | soc21_grbm_select(adev, me: 3, pipe: ring->pipe, queue: 0, vmid: 0); |
785 | |
786 | /* set CP_HQD_VMID.VMID = 0. */ |
787 | data = RREG32_SOC15(GC, 0, regCP_HQD_VMID); |
788 | data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0); |
789 | WREG32_SOC15(GC, 0, regCP_HQD_VMID, data); |
790 | |
791 | /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */ |
792 | data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL); |
793 | data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL, |
794 | DOORBELL_EN, 0); |
795 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data); |
796 | |
797 | /* set CP_MQD_BASE_ADDR/HI with the MQD base address */ |
798 | WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo); |
799 | WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi); |
800 | |
801 | /* set CP_MQD_CONTROL.VMID=0 */ |
802 | data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL); |
803 | data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0); |
804 | WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0); |
805 | |
806 | /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */ |
807 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo); |
808 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi); |
809 | |
810 | /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */ |
811 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, |
812 | mqd->cp_hqd_pq_rptr_report_addr_lo); |
813 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, |
814 | mqd->cp_hqd_pq_rptr_report_addr_hi); |
815 | |
816 | /* set CP_HQD_PQ_CONTROL */ |
817 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control); |
818 | |
819 | /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */ |
820 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, |
821 | mqd->cp_hqd_pq_wptr_poll_addr_lo); |
822 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, |
823 | mqd->cp_hqd_pq_wptr_poll_addr_hi); |
824 | |
825 | /* set CP_HQD_PQ_DOORBELL_CONTROL */ |
826 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, |
827 | mqd->cp_hqd_pq_doorbell_control); |
828 | |
829 | /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */ |
830 | WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state); |
831 | |
832 | /* set CP_HQD_ACTIVE.ACTIVE=1 */ |
833 | WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active); |
834 | |
835 | soc21_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0); |
836 | mutex_unlock(lock: &adev->srbm_mutex); |
837 | } |
838 | |
839 | static int mes_v11_0_kiq_enable_queue(struct amdgpu_device *adev) |
840 | { |
841 | struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; |
842 | struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; |
843 | int r; |
844 | |
845 | if (!kiq->pmf || !kiq->pmf->kiq_map_queues) |
846 | return -EINVAL; |
847 | |
848 | r = amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->map_queues_size); |
849 | if (r) { |
850 | DRM_ERROR("Failed to lock KIQ (%d).\n" , r); |
851 | return r; |
852 | } |
853 | |
854 | kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring); |
855 | |
856 | return amdgpu_ring_test_helper(ring: kiq_ring); |
857 | } |
858 | |
859 | static int mes_v11_0_queue_init(struct amdgpu_device *adev, |
860 | enum admgpu_mes_pipe pipe) |
861 | { |
862 | struct amdgpu_ring *ring; |
863 | int r; |
864 | |
865 | if (pipe == AMDGPU_MES_KIQ_PIPE) |
866 | ring = &adev->gfx.kiq[0].ring; |
867 | else if (pipe == AMDGPU_MES_SCHED_PIPE) |
868 | ring = &adev->mes.ring; |
869 | else |
870 | BUG(); |
871 | |
872 | if ((pipe == AMDGPU_MES_SCHED_PIPE) && |
873 | (amdgpu_in_reset(adev) || adev->in_suspend)) { |
874 | *(ring->wptr_cpu_addr) = 0; |
875 | *(ring->rptr_cpu_addr) = 0; |
876 | amdgpu_ring_clear_ring(ring); |
877 | } |
878 | |
879 | r = mes_v11_0_mqd_init(ring); |
880 | if (r) |
881 | return r; |
882 | |
883 | if (pipe == AMDGPU_MES_SCHED_PIPE) { |
884 | r = mes_v11_0_kiq_enable_queue(adev); |
885 | if (r) |
886 | return r; |
887 | } else { |
888 | mes_v11_0_queue_init_register(ring); |
889 | } |
890 | |
891 | /* get MES scheduler/KIQ versions */ |
892 | mutex_lock(&adev->srbm_mutex); |
893 | soc21_grbm_select(adev, me: 3, pipe, queue: 0, vmid: 0); |
894 | |
895 | if (pipe == AMDGPU_MES_SCHED_PIPE) |
896 | adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO); |
897 | else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq) |
898 | adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO); |
899 | |
900 | soc21_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0); |
901 | mutex_unlock(lock: &adev->srbm_mutex); |
902 | |
903 | return 0; |
904 | } |
905 | |
906 | static int mes_v11_0_ring_init(struct amdgpu_device *adev) |
907 | { |
908 | struct amdgpu_ring *ring; |
909 | |
910 | ring = &adev->mes.ring; |
911 | |
912 | ring->funcs = &mes_v11_0_ring_funcs; |
913 | |
914 | ring->me = 3; |
915 | ring->pipe = 0; |
916 | ring->queue = 0; |
917 | |
918 | ring->ring_obj = NULL; |
919 | ring->use_doorbell = true; |
920 | ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1; |
921 | ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE]; |
922 | ring->no_scheduler = true; |
923 | sprintf(buf: ring->name, fmt: "mes_%d.%d.%d" , ring->me, ring->pipe, ring->queue); |
924 | |
925 | return amdgpu_ring_init(adev, ring, max_dw: 1024, NULL, irq_type: 0, |
926 | hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL); |
927 | } |
928 | |
929 | static int mes_v11_0_kiq_ring_init(struct amdgpu_device *adev) |
930 | { |
931 | struct amdgpu_ring *ring; |
932 | |
933 | spin_lock_init(&adev->gfx.kiq[0].ring_lock); |
934 | |
935 | ring = &adev->gfx.kiq[0].ring; |
936 | |
937 | ring->me = 3; |
938 | ring->pipe = 1; |
939 | ring->queue = 0; |
940 | |
941 | ring->adev = NULL; |
942 | ring->ring_obj = NULL; |
943 | ring->use_doorbell = true; |
944 | ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1; |
945 | ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE]; |
946 | ring->no_scheduler = true; |
947 | sprintf(buf: ring->name, fmt: "mes_kiq_%d.%d.%d" , |
948 | ring->me, ring->pipe, ring->queue); |
949 | |
950 | return amdgpu_ring_init(adev, ring, max_dw: 1024, NULL, irq_type: 0, |
951 | hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL); |
952 | } |
953 | |
954 | static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev, |
955 | enum admgpu_mes_pipe pipe) |
956 | { |
957 | int r, mqd_size = sizeof(struct v11_compute_mqd); |
958 | struct amdgpu_ring *ring; |
959 | |
960 | if (pipe == AMDGPU_MES_KIQ_PIPE) |
961 | ring = &adev->gfx.kiq[0].ring; |
962 | else if (pipe == AMDGPU_MES_SCHED_PIPE) |
963 | ring = &adev->mes.ring; |
964 | else |
965 | BUG(); |
966 | |
967 | if (ring->mqd_obj) |
968 | return 0; |
969 | |
970 | r = amdgpu_bo_create_kernel(adev, size: mqd_size, PAGE_SIZE, |
971 | AMDGPU_GEM_DOMAIN_VRAM | |
972 | AMDGPU_GEM_DOMAIN_GTT, bo_ptr: &ring->mqd_obj, |
973 | gpu_addr: &ring->mqd_gpu_addr, cpu_addr: &ring->mqd_ptr); |
974 | if (r) { |
975 | dev_warn(adev->dev, "failed to create ring mqd bo (%d)" , r); |
976 | return r; |
977 | } |
978 | |
979 | memset(ring->mqd_ptr, 0, mqd_size); |
980 | |
981 | /* prepare MQD backup */ |
982 | adev->mes.mqd_backup[pipe] = kmalloc(size: mqd_size, GFP_KERNEL); |
983 | if (!adev->mes.mqd_backup[pipe]) { |
984 | dev_warn(adev->dev, |
985 | "no memory to create MQD backup for ring %s\n" , |
986 | ring->name); |
987 | return -ENOMEM; |
988 | } |
989 | |
990 | return 0; |
991 | } |
992 | |
993 | static int mes_v11_0_sw_init(void *handle) |
994 | { |
995 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
996 | int pipe, r; |
997 | |
998 | adev->mes.funcs = &mes_v11_0_funcs; |
999 | adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init; |
1000 | adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini; |
1001 | |
1002 | r = amdgpu_mes_init(adev); |
1003 | if (r) |
1004 | return r; |
1005 | |
1006 | for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { |
1007 | if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE) |
1008 | continue; |
1009 | |
1010 | r = mes_v11_0_allocate_eop_buf(adev, pipe); |
1011 | if (r) |
1012 | return r; |
1013 | |
1014 | r = mes_v11_0_mqd_sw_init(adev, pipe); |
1015 | if (r) |
1016 | return r; |
1017 | } |
1018 | |
1019 | if (adev->enable_mes_kiq) { |
1020 | r = mes_v11_0_kiq_ring_init(adev); |
1021 | if (r) |
1022 | return r; |
1023 | } |
1024 | |
1025 | r = mes_v11_0_ring_init(adev); |
1026 | if (r) |
1027 | return r; |
1028 | |
1029 | return 0; |
1030 | } |
1031 | |
1032 | static int mes_v11_0_sw_fini(void *handle) |
1033 | { |
1034 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1035 | int pipe; |
1036 | |
1037 | amdgpu_device_wb_free(adev, wb: adev->mes.sch_ctx_offs); |
1038 | amdgpu_device_wb_free(adev, wb: adev->mes.query_status_fence_offs); |
1039 | |
1040 | for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { |
1041 | kfree(objp: adev->mes.mqd_backup[pipe]); |
1042 | |
1043 | amdgpu_bo_free_kernel(bo: &adev->mes.eop_gpu_obj[pipe], |
1044 | gpu_addr: &adev->mes.eop_gpu_addr[pipe], |
1045 | NULL); |
1046 | amdgpu_ucode_release(fw: &adev->mes.fw[pipe]); |
1047 | } |
1048 | |
1049 | amdgpu_bo_free_kernel(bo: &adev->gfx.kiq[0].ring.mqd_obj, |
1050 | gpu_addr: &adev->gfx.kiq[0].ring.mqd_gpu_addr, |
1051 | cpu_addr: &adev->gfx.kiq[0].ring.mqd_ptr); |
1052 | |
1053 | amdgpu_bo_free_kernel(bo: &adev->mes.ring.mqd_obj, |
1054 | gpu_addr: &adev->mes.ring.mqd_gpu_addr, |
1055 | cpu_addr: &adev->mes.ring.mqd_ptr); |
1056 | |
1057 | amdgpu_ring_fini(ring: &adev->gfx.kiq[0].ring); |
1058 | amdgpu_ring_fini(ring: &adev->mes.ring); |
1059 | |
1060 | if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { |
1061 | mes_v11_0_free_ucode_buffers(adev, pipe: AMDGPU_MES_KIQ_PIPE); |
1062 | mes_v11_0_free_ucode_buffers(adev, pipe: AMDGPU_MES_SCHED_PIPE); |
1063 | } |
1064 | |
1065 | amdgpu_mes_fini(adev); |
1066 | return 0; |
1067 | } |
1068 | |
1069 | static void mes_v11_0_kiq_dequeue(struct amdgpu_ring *ring) |
1070 | { |
1071 | uint32_t data; |
1072 | int i; |
1073 | struct amdgpu_device *adev = ring->adev; |
1074 | |
1075 | mutex_lock(&adev->srbm_mutex); |
1076 | soc21_grbm_select(adev, me: 3, pipe: ring->pipe, queue: 0, vmid: 0); |
1077 | |
1078 | /* disable the queue if it's active */ |
1079 | if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { |
1080 | WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); |
1081 | for (i = 0; i < adev->usec_timeout; i++) { |
1082 | if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) |
1083 | break; |
1084 | udelay(1); |
1085 | } |
1086 | } |
1087 | data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL); |
1088 | data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL, |
1089 | DOORBELL_EN, 0); |
1090 | data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL, |
1091 | DOORBELL_HIT, 1); |
1092 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data); |
1093 | |
1094 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0); |
1095 | |
1096 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0); |
1097 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0); |
1098 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0); |
1099 | |
1100 | soc21_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0); |
1101 | mutex_unlock(lock: &adev->srbm_mutex); |
1102 | } |
1103 | |
1104 | static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring) |
1105 | { |
1106 | uint32_t tmp; |
1107 | struct amdgpu_device *adev = ring->adev; |
1108 | |
1109 | /* tell RLC which is KIQ queue */ |
1110 | tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); |
1111 | tmp &= 0xffffff00; |
1112 | tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); |
1113 | WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp); |
1114 | tmp |= 0x80; |
1115 | WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp); |
1116 | } |
1117 | |
1118 | static void mes_v11_0_kiq_clear(struct amdgpu_device *adev) |
1119 | { |
1120 | uint32_t tmp; |
1121 | |
1122 | /* tell RLC which is KIQ dequeue */ |
1123 | tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); |
1124 | tmp &= ~RLC_CP_SCHEDULERS__scheduler0_MASK; |
1125 | WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp); |
1126 | } |
1127 | |
1128 | static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev) |
1129 | { |
1130 | int r = 0; |
1131 | |
1132 | if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { |
1133 | |
1134 | r = mes_v11_0_load_microcode(adev, pipe: AMDGPU_MES_SCHED_PIPE, prime_icache: false); |
1135 | if (r) { |
1136 | DRM_ERROR("failed to load MES fw, r=%d\n" , r); |
1137 | return r; |
1138 | } |
1139 | |
1140 | r = mes_v11_0_load_microcode(adev, pipe: AMDGPU_MES_KIQ_PIPE, prime_icache: true); |
1141 | if (r) { |
1142 | DRM_ERROR("failed to load MES kiq fw, r=%d\n" , r); |
1143 | return r; |
1144 | } |
1145 | |
1146 | } |
1147 | |
1148 | mes_v11_0_enable(adev, enable: true); |
1149 | |
1150 | mes_v11_0_kiq_setting(ring: &adev->gfx.kiq[0].ring); |
1151 | |
1152 | r = mes_v11_0_queue_init(adev, pipe: AMDGPU_MES_KIQ_PIPE); |
1153 | if (r) |
1154 | goto failure; |
1155 | |
1156 | return r; |
1157 | |
1158 | failure: |
1159 | mes_v11_0_hw_fini(handle: adev); |
1160 | return r; |
1161 | } |
1162 | |
1163 | static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev) |
1164 | { |
1165 | if (adev->mes.ring.sched.ready) { |
1166 | mes_v11_0_kiq_dequeue(ring: &adev->mes.ring); |
1167 | adev->mes.ring.sched.ready = false; |
1168 | } |
1169 | |
1170 | if (amdgpu_sriov_vf(adev)) { |
1171 | mes_v11_0_kiq_dequeue(ring: &adev->gfx.kiq[0].ring); |
1172 | mes_v11_0_kiq_clear(adev); |
1173 | } |
1174 | |
1175 | mes_v11_0_enable(adev, enable: false); |
1176 | |
1177 | return 0; |
1178 | } |
1179 | |
1180 | static int mes_v11_0_hw_init(void *handle) |
1181 | { |
1182 | int r; |
1183 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1184 | |
1185 | if (!adev->enable_mes_kiq) { |
1186 | if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { |
1187 | r = mes_v11_0_load_microcode(adev, |
1188 | pipe: AMDGPU_MES_SCHED_PIPE, prime_icache: true); |
1189 | if (r) { |
1190 | DRM_ERROR("failed to MES fw, r=%d\n" , r); |
1191 | return r; |
1192 | } |
1193 | } |
1194 | |
1195 | mes_v11_0_enable(adev, enable: true); |
1196 | } |
1197 | |
1198 | r = mes_v11_0_queue_init(adev, pipe: AMDGPU_MES_SCHED_PIPE); |
1199 | if (r) |
1200 | goto failure; |
1201 | |
1202 | r = mes_v11_0_set_hw_resources(mes: &adev->mes); |
1203 | if (r) |
1204 | goto failure; |
1205 | |
1206 | r = mes_v11_0_query_sched_status(mes: &adev->mes); |
1207 | if (r) { |
1208 | DRM_ERROR("MES is busy\n" ); |
1209 | goto failure; |
1210 | } |
1211 | |
1212 | /* |
1213 | * Disable KIQ ring usage from the driver once MES is enabled. |
1214 | * MES uses KIQ ring exclusively so driver cannot access KIQ ring |
1215 | * with MES enabled. |
1216 | */ |
1217 | adev->gfx.kiq[0].ring.sched.ready = false; |
1218 | adev->mes.ring.sched.ready = true; |
1219 | |
1220 | return 0; |
1221 | |
1222 | failure: |
1223 | mes_v11_0_hw_fini(handle: adev); |
1224 | return r; |
1225 | } |
1226 | |
1227 | static int mes_v11_0_hw_fini(void *handle) |
1228 | { |
1229 | return 0; |
1230 | } |
1231 | |
1232 | static int mes_v11_0_suspend(void *handle) |
1233 | { |
1234 | int r; |
1235 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1236 | |
1237 | r = amdgpu_mes_suspend(adev); |
1238 | if (r) |
1239 | return r; |
1240 | |
1241 | return mes_v11_0_hw_fini(handle: adev); |
1242 | } |
1243 | |
1244 | static int mes_v11_0_resume(void *handle) |
1245 | { |
1246 | int r; |
1247 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1248 | |
1249 | r = mes_v11_0_hw_init(handle: adev); |
1250 | if (r) |
1251 | return r; |
1252 | |
1253 | return amdgpu_mes_resume(adev); |
1254 | } |
1255 | |
1256 | static int mes_v11_0_early_init(void *handle) |
1257 | { |
1258 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1259 | int pipe, r; |
1260 | |
1261 | for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { |
1262 | if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE) |
1263 | continue; |
1264 | r = amdgpu_mes_init_microcode(adev, pipe); |
1265 | if (r) |
1266 | return r; |
1267 | } |
1268 | |
1269 | return 0; |
1270 | } |
1271 | |
1272 | static int mes_v11_0_late_init(void *handle) |
1273 | { |
1274 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1275 | |
1276 | /* it's only intended for use in mes_self_test case, not for s0ix and reset */ |
1277 | if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend && |
1278 | (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(11, 0, 3))) |
1279 | amdgpu_mes_self_test(adev); |
1280 | |
1281 | return 0; |
1282 | } |
1283 | |
1284 | static const struct amd_ip_funcs mes_v11_0_ip_funcs = { |
1285 | .name = "mes_v11_0" , |
1286 | .early_init = mes_v11_0_early_init, |
1287 | .late_init = mes_v11_0_late_init, |
1288 | .sw_init = mes_v11_0_sw_init, |
1289 | .sw_fini = mes_v11_0_sw_fini, |
1290 | .hw_init = mes_v11_0_hw_init, |
1291 | .hw_fini = mes_v11_0_hw_fini, |
1292 | .suspend = mes_v11_0_suspend, |
1293 | .resume = mes_v11_0_resume, |
1294 | }; |
1295 | |
1296 | const struct amdgpu_ip_block_version mes_v11_0_ip_block = { |
1297 | .type = AMD_IP_BLOCK_TYPE_MES, |
1298 | .major = 11, |
1299 | .minor = 0, |
1300 | .rev = 0, |
1301 | .funcs = &mes_v11_0_ip_funcs, |
1302 | }; |
1303 | |