1 | /* |
2 | * Copyright 2019 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | */ |
22 | #include "amdgpu.h" |
23 | #include "amdgpu_amdkfd.h" |
24 | #include "amdgpu_amdkfd_gfx_v10.h" |
25 | #include "gc/gc_10_1_0_offset.h" |
26 | #include "gc/gc_10_1_0_sh_mask.h" |
27 | #include "athub/athub_2_0_0_offset.h" |
28 | #include "athub/athub_2_0_0_sh_mask.h" |
29 | #include "oss/osssys_5_0_0_offset.h" |
30 | #include "oss/osssys_5_0_0_sh_mask.h" |
31 | #include "soc15_common.h" |
32 | #include "v10_structs.h" |
33 | #include "nv.h" |
34 | #include "nvd.h" |
35 | #include <uapi/linux/kfd_ioctl.h> |
36 | |
37 | enum hqd_dequeue_request_type { |
38 | NO_ACTION = 0, |
39 | DRAIN_PIPE, |
40 | RESET_WAVES, |
41 | SAVE_WAVES |
42 | }; |
43 | |
44 | static void lock_srbm(struct amdgpu_device *adev, uint32_t mec, uint32_t pipe, |
45 | uint32_t queue, uint32_t vmid) |
46 | { |
47 | mutex_lock(&adev->srbm_mutex); |
48 | nv_grbm_select(adev, me: mec, pipe, queue, vmid); |
49 | } |
50 | |
51 | static void unlock_srbm(struct amdgpu_device *adev) |
52 | { |
53 | nv_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0); |
54 | mutex_unlock(lock: &adev->srbm_mutex); |
55 | } |
56 | |
57 | static void acquire_queue(struct amdgpu_device *adev, uint32_t pipe_id, |
58 | uint32_t queue_id) |
59 | { |
60 | uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; |
61 | uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); |
62 | |
63 | lock_srbm(adev, mec, pipe, queue: queue_id, vmid: 0); |
64 | } |
65 | |
66 | static uint64_t get_queue_mask(struct amdgpu_device *adev, |
67 | uint32_t pipe_id, uint32_t queue_id) |
68 | { |
69 | unsigned int bit = pipe_id * adev->gfx.mec.num_queue_per_pipe + |
70 | queue_id; |
71 | |
72 | return 1ull << bit; |
73 | } |
74 | |
75 | static void release_queue(struct amdgpu_device *adev) |
76 | { |
77 | unlock_srbm(adev); |
78 | } |
79 | |
80 | static void kgd_program_sh_mem_settings(struct amdgpu_device *adev, uint32_t vmid, |
81 | uint32_t sh_mem_config, |
82 | uint32_t sh_mem_ape1_base, |
83 | uint32_t sh_mem_ape1_limit, |
84 | uint32_t sh_mem_bases, uint32_t inst) |
85 | { |
86 | lock_srbm(adev, mec: 0, pipe: 0, queue: 0, vmid); |
87 | |
88 | WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); |
89 | WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases); |
90 | /* APE1 no longer exists on GFX9 */ |
91 | |
92 | unlock_srbm(adev); |
93 | } |
94 | |
95 | static int kgd_set_pasid_vmid_mapping(struct amdgpu_device *adev, u32 pasid, |
96 | unsigned int vmid, uint32_t inst) |
97 | { |
98 | /* |
99 | * We have to assume that there is no outstanding mapping. |
100 | * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because |
101 | * a mapping is in progress or because a mapping finished |
102 | * and the SW cleared it. |
103 | * So the protocol is to always wait & clear. |
104 | */ |
105 | uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | |
106 | ATC_VMID0_PASID_MAPPING__VALID_MASK; |
107 | |
108 | pr_debug("pasid 0x%x vmid %d, reg value %x\n" , pasid, vmid, pasid_mapping); |
109 | |
110 | pr_debug("ATHUB, reg %x\n" , SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid); |
111 | WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid, |
112 | pasid_mapping); |
113 | |
114 | #if 0 |
115 | /* TODO: uncomment this code when the hardware support is ready. */ |
116 | while (!(RREG32(SOC15_REG_OFFSET( |
117 | ATHUB, 0, |
118 | mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) & |
119 | (1U << vmid))) |
120 | cpu_relax(); |
121 | |
122 | pr_debug("ATHUB mapping update finished\n" ); |
123 | WREG32(SOC15_REG_OFFSET(ATHUB, 0, |
124 | mmATC_VMID_PASID_MAPPING_UPDATE_STATUS), |
125 | 1U << vmid); |
126 | #endif |
127 | |
128 | /* Mapping vmid to pasid also for IH block */ |
129 | pr_debug("update mapping for IH block and mmhub" ); |
130 | WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid, |
131 | pasid_mapping); |
132 | |
133 | return 0; |
134 | } |
135 | |
136 | /* TODO - RING0 form of field is obsolete, seems to date back to SI |
137 | * but still works |
138 | */ |
139 | |
140 | static int kgd_init_interrupts(struct amdgpu_device *adev, uint32_t pipe_id, |
141 | uint32_t inst) |
142 | { |
143 | uint32_t mec; |
144 | uint32_t pipe; |
145 | |
146 | mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; |
147 | pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); |
148 | |
149 | lock_srbm(adev, mec, pipe, queue: 0, vmid: 0); |
150 | |
151 | WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, |
152 | CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | |
153 | CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); |
154 | |
155 | unlock_srbm(adev); |
156 | |
157 | return 0; |
158 | } |
159 | |
160 | static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, |
161 | unsigned int engine_id, |
162 | unsigned int queue_id) |
163 | { |
164 | uint32_t sdma_engine_reg_base[2] = { |
165 | SOC15_REG_OFFSET(SDMA0, 0, |
166 | mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL, |
167 | /* On gfx10, mmSDMA1_xxx registers are defined NOT based |
168 | * on SDMA1 base address (dw 0x1860) but based on SDMA0 |
169 | * base address (dw 0x1260). Therefore use mmSDMA0_RLC0_RB_CNTL |
170 | * instead of mmSDMA1_RLC0_RB_CNTL for the base address calc |
171 | * below |
172 | */ |
173 | SOC15_REG_OFFSET(SDMA1, 0, |
174 | mmSDMA1_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL |
175 | }; |
176 | |
177 | uint32_t retval = sdma_engine_reg_base[engine_id] |
178 | + queue_id * (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL); |
179 | |
180 | pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n" , engine_id, |
181 | queue_id, retval); |
182 | |
183 | return retval; |
184 | } |
185 | |
186 | #if 0 |
187 | static uint32_t get_watch_base_addr(struct amdgpu_device *adev) |
188 | { |
189 | uint32_t retval = SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) - |
190 | mmTCP_WATCH0_ADDR_H; |
191 | |
192 | pr_debug("kfd: reg watch base address: 0x%x\n" , retval); |
193 | |
194 | return retval; |
195 | } |
196 | #endif |
197 | |
198 | static inline struct v10_compute_mqd *get_mqd(void *mqd) |
199 | { |
200 | return (struct v10_compute_mqd *)mqd; |
201 | } |
202 | |
203 | static inline struct v10_sdma_mqd *get_sdma_mqd(void *mqd) |
204 | { |
205 | return (struct v10_sdma_mqd *)mqd; |
206 | } |
207 | |
208 | static int kgd_hqd_load(struct amdgpu_device *adev, void *mqd, |
209 | uint32_t pipe_id, uint32_t queue_id, |
210 | uint32_t __user *wptr, uint32_t wptr_shift, |
211 | uint32_t wptr_mask, struct mm_struct *mm, uint32_t inst) |
212 | { |
213 | struct v10_compute_mqd *m; |
214 | uint32_t *mqd_hqd; |
215 | uint32_t reg, hqd_base, data; |
216 | |
217 | m = get_mqd(mqd); |
218 | |
219 | pr_debug("Load hqd of pipe %d queue %d\n" , pipe_id, queue_id); |
220 | acquire_queue(adev, pipe_id, queue_id); |
221 | |
222 | /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ |
223 | mqd_hqd = &m->cp_mqd_base_addr_lo; |
224 | hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); |
225 | |
226 | for (reg = hqd_base; |
227 | reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) |
228 | WREG32_SOC15_IP(GC, reg, mqd_hqd[reg - hqd_base]); |
229 | |
230 | |
231 | /* Activate doorbell logic before triggering WPTR poll. */ |
232 | data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, |
233 | CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); |
234 | WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, data); |
235 | |
236 | if (wptr) { |
237 | /* Don't read wptr with get_user because the user |
238 | * context may not be accessible (if this function |
239 | * runs in a work queue). Instead trigger a one-shot |
240 | * polling read from memory in the CP. This assumes |
241 | * that wptr is GPU-accessible in the queue's VMID via |
242 | * ATC or SVM. WPTR==RPTR before starting the poll so |
243 | * the CP starts fetching new commands from the right |
244 | * place. |
245 | * |
246 | * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit |
247 | * tricky. Assume that the queue didn't overflow. The |
248 | * number of valid bits in the 32-bit RPTR depends on |
249 | * the queue size. The remaining bits are taken from |
250 | * the saved 64-bit WPTR. If the WPTR wrapped, add the |
251 | * queue size. |
252 | */ |
253 | uint32_t queue_size = |
254 | 2 << REG_GET_FIELD(m->cp_hqd_pq_control, |
255 | CP_HQD_PQ_CONTROL, QUEUE_SIZE); |
256 | uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1); |
257 | |
258 | if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr) |
259 | guessed_wptr += queue_size; |
260 | guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1); |
261 | guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32; |
262 | |
263 | WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO, |
264 | lower_32_bits(guessed_wptr)); |
265 | WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI, |
266 | upper_32_bits(guessed_wptr)); |
267 | WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, |
268 | lower_32_bits((uint64_t)wptr)); |
269 | WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, |
270 | upper_32_bits((uint64_t)wptr)); |
271 | pr_debug("%s setting CP_PQ_WPTR_POLL_CNTL1 to %x\n" , __func__, |
272 | (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); |
273 | WREG32_SOC15(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1, |
274 | (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); |
275 | } |
276 | |
277 | /* Start the EOP fetcher */ |
278 | WREG32_SOC15(GC, 0, mmCP_HQD_EOP_RPTR, |
279 | REG_SET_FIELD(m->cp_hqd_eop_rptr, |
280 | CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); |
281 | |
282 | data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); |
283 | WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, data); |
284 | |
285 | release_queue(adev); |
286 | |
287 | return 0; |
288 | } |
289 | |
290 | static int kgd_hiq_mqd_load(struct amdgpu_device *adev, void *mqd, |
291 | uint32_t pipe_id, uint32_t queue_id, |
292 | uint32_t doorbell_off, uint32_t inst) |
293 | { |
294 | struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; |
295 | struct v10_compute_mqd *m; |
296 | uint32_t mec, pipe; |
297 | int r; |
298 | |
299 | m = get_mqd(mqd); |
300 | |
301 | acquire_queue(adev, pipe_id, queue_id); |
302 | |
303 | mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; |
304 | pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); |
305 | |
306 | pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n" , |
307 | mec, pipe, queue_id); |
308 | |
309 | spin_lock(lock: &adev->gfx.kiq[0].ring_lock); |
310 | r = amdgpu_ring_alloc(ring: kiq_ring, ndw: 7); |
311 | if (r) { |
312 | pr_err("Failed to alloc KIQ (%d).\n" , r); |
313 | goto out_unlock; |
314 | } |
315 | |
316 | amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); |
317 | amdgpu_ring_write(ring: kiq_ring, |
318 | PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ |
319 | PACKET3_MAP_QUEUES_VMID(m->cp_hqd_vmid) | /* VMID */ |
320 | PACKET3_MAP_QUEUES_QUEUE(queue_id) | |
321 | PACKET3_MAP_QUEUES_PIPE(pipe) | |
322 | PACKET3_MAP_QUEUES_ME((mec - 1)) | |
323 | PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ |
324 | PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ |
325 | PACKET3_MAP_QUEUES_ENGINE_SEL(1) | /* engine_sel: hiq */ |
326 | PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ |
327 | amdgpu_ring_write(ring: kiq_ring, |
328 | PACKET3_MAP_QUEUES_DOORBELL_OFFSET(doorbell_off)); |
329 | amdgpu_ring_write(ring: kiq_ring, v: m->cp_mqd_base_addr_lo); |
330 | amdgpu_ring_write(ring: kiq_ring, v: m->cp_mqd_base_addr_hi); |
331 | amdgpu_ring_write(ring: kiq_ring, v: m->cp_hqd_pq_wptr_poll_addr_lo); |
332 | amdgpu_ring_write(ring: kiq_ring, v: m->cp_hqd_pq_wptr_poll_addr_hi); |
333 | amdgpu_ring_commit(ring: kiq_ring); |
334 | |
335 | out_unlock: |
336 | spin_unlock(lock: &adev->gfx.kiq[0].ring_lock); |
337 | release_queue(adev); |
338 | |
339 | return r; |
340 | } |
341 | |
342 | static int kgd_hqd_dump(struct amdgpu_device *adev, |
343 | uint32_t pipe_id, uint32_t queue_id, |
344 | uint32_t (**dump)[2], uint32_t *n_regs, uint32_t inst) |
345 | { |
346 | uint32_t i = 0, reg; |
347 | #define HQD_N_REGS 56 |
348 | #define DUMP_REG(addr) do { \ |
349 | if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ |
350 | break; \ |
351 | (*dump)[i][0] = (addr) << 2; \ |
352 | (*dump)[i++][1] = RREG32_SOC15_IP(GC, addr); \ |
353 | } while (0) |
354 | |
355 | *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); |
356 | if (*dump == NULL) |
357 | return -ENOMEM; |
358 | |
359 | acquire_queue(adev, pipe_id, queue_id); |
360 | |
361 | for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); |
362 | reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) |
363 | DUMP_REG(reg); |
364 | |
365 | release_queue(adev); |
366 | |
367 | WARN_ON_ONCE(i != HQD_N_REGS); |
368 | *n_regs = i; |
369 | |
370 | return 0; |
371 | } |
372 | |
373 | static int kgd_hqd_sdma_load(struct amdgpu_device *adev, void *mqd, |
374 | uint32_t __user *wptr, struct mm_struct *mm) |
375 | { |
376 | struct v10_sdma_mqd *m; |
377 | uint32_t sdma_rlc_reg_offset; |
378 | unsigned long end_jiffies; |
379 | uint32_t data; |
380 | uint64_t data64; |
381 | uint64_t __user *wptr64 = (uint64_t __user *)wptr; |
382 | |
383 | m = get_sdma_mqd(mqd); |
384 | sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, engine_id: m->sdma_engine_id, |
385 | queue_id: m->sdma_queue_id); |
386 | |
387 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, |
388 | m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); |
389 | |
390 | end_jiffies = msecs_to_jiffies(m: 2000) + jiffies; |
391 | while (true) { |
392 | data = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS); |
393 | if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) |
394 | break; |
395 | if (time_after(jiffies, end_jiffies)) { |
396 | pr_err("SDMA RLC not idle in %s\n" , __func__); |
397 | return -ETIME; |
398 | } |
399 | usleep_range(min: 500, max: 1000); |
400 | } |
401 | |
402 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL_OFFSET, |
403 | m->sdmax_rlcx_doorbell_offset); |
404 | |
405 | data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, |
406 | ENABLE, 1); |
407 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, data); |
408 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR, |
409 | m->sdmax_rlcx_rb_rptr); |
410 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI, |
411 | m->sdmax_rlcx_rb_rptr_hi); |
412 | |
413 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1); |
414 | if (read_user_wptr(mm, wptr64, data64)) { |
415 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, |
416 | lower_32_bits(data64)); |
417 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI, |
418 | upper_32_bits(data64)); |
419 | } else { |
420 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, |
421 | m->sdmax_rlcx_rb_rptr); |
422 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI, |
423 | m->sdmax_rlcx_rb_rptr_hi); |
424 | } |
425 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0); |
426 | |
427 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base); |
428 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE_HI, |
429 | m->sdmax_rlcx_rb_base_hi); |
430 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, |
431 | m->sdmax_rlcx_rb_rptr_addr_lo); |
432 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, |
433 | m->sdmax_rlcx_rb_rptr_addr_hi); |
434 | |
435 | data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, |
436 | RB_ENABLE, 1); |
437 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, data); |
438 | |
439 | return 0; |
440 | } |
441 | |
442 | static int kgd_hqd_sdma_dump(struct amdgpu_device *adev, |
443 | uint32_t engine_id, uint32_t queue_id, |
444 | uint32_t (**dump)[2], uint32_t *n_regs) |
445 | { |
446 | uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, |
447 | engine_id, queue_id); |
448 | uint32_t i = 0, reg; |
449 | #undef HQD_N_REGS |
450 | #define HQD_N_REGS (19+6+7+10) |
451 | |
452 | *dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL); |
453 | if (*dump == NULL) |
454 | return -ENOMEM; |
455 | |
456 | for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) |
457 | DUMP_REG(sdma_rlc_reg_offset + reg); |
458 | for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++) |
459 | DUMP_REG(sdma_rlc_reg_offset + reg); |
460 | for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; |
461 | reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++) |
462 | DUMP_REG(sdma_rlc_reg_offset + reg); |
463 | for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; |
464 | reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++) |
465 | DUMP_REG(sdma_rlc_reg_offset + reg); |
466 | |
467 | WARN_ON_ONCE(i != HQD_N_REGS); |
468 | *n_regs = i; |
469 | |
470 | return 0; |
471 | } |
472 | |
473 | static bool kgd_hqd_is_occupied(struct amdgpu_device *adev, |
474 | uint64_t queue_address, uint32_t pipe_id, |
475 | uint32_t queue_id, uint32_t inst) |
476 | { |
477 | uint32_t act; |
478 | bool retval = false; |
479 | uint32_t low, high; |
480 | |
481 | acquire_queue(adev, pipe_id, queue_id); |
482 | act = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); |
483 | if (act) { |
484 | low = lower_32_bits(queue_address >> 8); |
485 | high = upper_32_bits(queue_address >> 8); |
486 | |
487 | if (low == RREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE) && |
488 | high == RREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI)) |
489 | retval = true; |
490 | } |
491 | release_queue(adev); |
492 | return retval; |
493 | } |
494 | |
495 | static bool kgd_hqd_sdma_is_occupied(struct amdgpu_device *adev, void *mqd) |
496 | { |
497 | struct v10_sdma_mqd *m; |
498 | uint32_t sdma_rlc_reg_offset; |
499 | uint32_t sdma_rlc_rb_cntl; |
500 | |
501 | m = get_sdma_mqd(mqd); |
502 | sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, engine_id: m->sdma_engine_id, |
503 | queue_id: m->sdma_queue_id); |
504 | |
505 | sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL); |
506 | |
507 | if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) |
508 | return true; |
509 | |
510 | return false; |
511 | } |
512 | |
513 | static int kgd_hqd_destroy(struct amdgpu_device *adev, void *mqd, |
514 | enum kfd_preempt_type reset_type, |
515 | unsigned int utimeout, uint32_t pipe_id, |
516 | uint32_t queue_id, uint32_t inst) |
517 | { |
518 | enum hqd_dequeue_request_type type; |
519 | unsigned long end_jiffies; |
520 | uint32_t temp; |
521 | struct v10_compute_mqd *m = get_mqd(mqd); |
522 | |
523 | if (amdgpu_in_reset(adev)) |
524 | return -EIO; |
525 | |
526 | #if 0 |
527 | unsigned long flags; |
528 | int retry; |
529 | #endif |
530 | |
531 | acquire_queue(adev, pipe_id, queue_id); |
532 | |
533 | if (m->cp_hqd_vmid == 0) |
534 | WREG32_FIELD15(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0); |
535 | |
536 | switch (reset_type) { |
537 | case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: |
538 | type = DRAIN_PIPE; |
539 | break; |
540 | case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: |
541 | type = RESET_WAVES; |
542 | break; |
543 | case KFD_PREEMPT_TYPE_WAVEFRONT_SAVE: |
544 | type = SAVE_WAVES; |
545 | break; |
546 | default: |
547 | type = DRAIN_PIPE; |
548 | break; |
549 | } |
550 | |
551 | #if 0 /* Is this still needed? */ |
552 | /* Workaround: If IQ timer is active and the wait time is close to or |
553 | * equal to 0, dequeueing is not safe. Wait until either the wait time |
554 | * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is |
555 | * cleared before continuing. Also, ensure wait times are set to at |
556 | * least 0x3. |
557 | */ |
558 | local_irq_save(flags); |
559 | preempt_disable(); |
560 | retry = 5000; /* wait for 500 usecs at maximum */ |
561 | while (true) { |
562 | temp = RREG32(mmCP_HQD_IQ_TIMER); |
563 | if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) { |
564 | pr_debug("HW is processing IQ\n" ); |
565 | goto loop; |
566 | } |
567 | if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) { |
568 | if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE) |
569 | == 3) /* SEM-rearm is safe */ |
570 | break; |
571 | /* Wait time 3 is safe for CP, but our MMIO read/write |
572 | * time is close to 1 microsecond, so check for 10 to |
573 | * leave more buffer room |
574 | */ |
575 | if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME) |
576 | >= 10) |
577 | break; |
578 | pr_debug("IQ timer is active\n" ); |
579 | } else |
580 | break; |
581 | loop: |
582 | if (!retry) { |
583 | pr_err("CP HQD IQ timer status time out\n" ); |
584 | break; |
585 | } |
586 | ndelay(100); |
587 | --retry; |
588 | } |
589 | retry = 1000; |
590 | while (true) { |
591 | temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST); |
592 | if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK)) |
593 | break; |
594 | pr_debug("Dequeue request is pending\n" ); |
595 | |
596 | if (!retry) { |
597 | pr_err("CP HQD dequeue request time out\n" ); |
598 | break; |
599 | } |
600 | ndelay(100); |
601 | --retry; |
602 | } |
603 | local_irq_restore(flags); |
604 | preempt_enable(); |
605 | #endif |
606 | |
607 | WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, type); |
608 | |
609 | end_jiffies = (utimeout * HZ / 1000) + jiffies; |
610 | while (true) { |
611 | temp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); |
612 | if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) |
613 | break; |
614 | if (time_after(jiffies, end_jiffies)) { |
615 | pr_err("cp queue preemption time out.\n" ); |
616 | release_queue(adev); |
617 | return -ETIME; |
618 | } |
619 | usleep_range(min: 500, max: 1000); |
620 | } |
621 | |
622 | release_queue(adev); |
623 | return 0; |
624 | } |
625 | |
626 | static int kgd_hqd_sdma_destroy(struct amdgpu_device *adev, void *mqd, |
627 | unsigned int utimeout) |
628 | { |
629 | struct v10_sdma_mqd *m; |
630 | uint32_t sdma_rlc_reg_offset; |
631 | uint32_t temp; |
632 | unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; |
633 | |
634 | m = get_sdma_mqd(mqd); |
635 | sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, engine_id: m->sdma_engine_id, |
636 | queue_id: m->sdma_queue_id); |
637 | |
638 | temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL); |
639 | temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; |
640 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, temp); |
641 | |
642 | while (true) { |
643 | temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS); |
644 | if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) |
645 | break; |
646 | if (time_after(jiffies, end_jiffies)) { |
647 | pr_err("SDMA RLC not idle in %s\n" , __func__); |
648 | return -ETIME; |
649 | } |
650 | usleep_range(min: 500, max: 1000); |
651 | } |
652 | |
653 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, 0); |
654 | WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, |
655 | RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL) | |
656 | SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); |
657 | |
658 | m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR); |
659 | m->sdmax_rlcx_rb_rptr_hi = |
660 | RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI); |
661 | |
662 | return 0; |
663 | } |
664 | |
665 | static bool get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, |
666 | uint8_t vmid, uint16_t *p_pasid) |
667 | { |
668 | uint32_t value; |
669 | |
670 | value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) |
671 | + vmid); |
672 | *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; |
673 | |
674 | return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); |
675 | } |
676 | |
677 | static int kgd_wave_control_execute(struct amdgpu_device *adev, |
678 | uint32_t gfx_index_val, |
679 | uint32_t sq_cmd, uint32_t inst) |
680 | { |
681 | uint32_t data = 0; |
682 | |
683 | mutex_lock(&adev->grbm_idx_mutex); |
684 | |
685 | WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, gfx_index_val); |
686 | WREG32_SOC15(GC, 0, mmSQ_CMD, sq_cmd); |
687 | |
688 | data = REG_SET_FIELD(data, GRBM_GFX_INDEX, |
689 | INSTANCE_BROADCAST_WRITES, 1); |
690 | data = REG_SET_FIELD(data, GRBM_GFX_INDEX, |
691 | SA_BROADCAST_WRITES, 1); |
692 | data = REG_SET_FIELD(data, GRBM_GFX_INDEX, |
693 | SE_BROADCAST_WRITES, 1); |
694 | |
695 | WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data); |
696 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
697 | |
698 | return 0; |
699 | } |
700 | |
701 | static void set_vm_context_page_table_base(struct amdgpu_device *adev, |
702 | uint32_t vmid, uint64_t page_table_base) |
703 | { |
704 | if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { |
705 | pr_err("trying to set page table base for wrong VMID %u\n" , |
706 | vmid); |
707 | return; |
708 | } |
709 | |
710 | /* SDMA is on gfxhub as well for Navi1* series */ |
711 | adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, page_table_base); |
712 | } |
713 | |
714 | /* |
715 | * GFX10 helper for wave launch stall requirements on debug trap setting. |
716 | * |
717 | * vmid: |
718 | * Target VMID to stall/unstall. |
719 | * |
720 | * stall: |
721 | * 0-unstall wave launch (enable), 1-stall wave launch (disable). |
722 | * After wavefront launch has been stalled, allocated waves must drain from |
723 | * SPI in order for debug trap settings to take effect on those waves. |
724 | * This is roughly a ~3500 clock cycle wait on SPI where a read on |
725 | * SPI_GDBG_WAVE_CNTL translates to ~32 clock cycles. |
726 | * KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY indicates the number of reads required. |
727 | * |
728 | * NOTE: We can afford to clear the entire STALL_VMID field on unstall |
729 | * because current GFX10 chips cannot support multi-process debugging due to |
730 | * trap configuration and masking being limited to global scope. Always |
731 | * assume single process conditions. |
732 | * |
733 | */ |
734 | |
735 | #define KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY 110 |
736 | static void kgd_gfx_v10_set_wave_launch_stall(struct amdgpu_device *adev, uint32_t vmid, bool stall) |
737 | { |
738 | uint32_t data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); |
739 | int i; |
740 | |
741 | data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL, STALL_VMID, |
742 | stall ? 1 << vmid : 0); |
743 | |
744 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), data); |
745 | |
746 | if (!stall) |
747 | return; |
748 | |
749 | for (i = 0; i < KGD_GFX_V10_WAVE_LAUNCH_SPI_DRAIN_LATENCY; i++) |
750 | RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); |
751 | } |
752 | |
753 | uint32_t kgd_gfx_v10_enable_debug_trap(struct amdgpu_device *adev, |
754 | bool restore_dbg_registers, |
755 | uint32_t vmid) |
756 | { |
757 | |
758 | mutex_lock(&adev->grbm_idx_mutex); |
759 | |
760 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: true); |
761 | |
762 | /* assume gfx off is disabled for the debug session if rlc restore not supported. */ |
763 | if (restore_dbg_registers) { |
764 | uint32_t data = 0; |
765 | |
766 | data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, |
767 | VMID_SEL, 1 << vmid); |
768 | data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, |
769 | TRAP_EN, 1); |
770 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); |
771 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0); |
772 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0); |
773 | |
774 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: false); |
775 | |
776 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
777 | |
778 | return 0; |
779 | } |
780 | |
781 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); |
782 | |
783 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: false); |
784 | |
785 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
786 | |
787 | return 0; |
788 | } |
789 | |
790 | uint32_t kgd_gfx_v10_disable_debug_trap(struct amdgpu_device *adev, |
791 | bool keep_trap_enabled, |
792 | uint32_t vmid) |
793 | { |
794 | mutex_lock(&adev->grbm_idx_mutex); |
795 | |
796 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: true); |
797 | |
798 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); |
799 | |
800 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: false); |
801 | |
802 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
803 | |
804 | return 0; |
805 | } |
806 | |
807 | int kgd_gfx_v10_validate_trap_override_request(struct amdgpu_device *adev, |
808 | uint32_t trap_override, |
809 | uint32_t *trap_mask_supported) |
810 | { |
811 | *trap_mask_supported &= KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH; |
812 | |
813 | /* The SPI_GDBG_TRAP_MASK register is global and affects all |
814 | * processes. Only allow OR-ing the address-watch bit, since |
815 | * this only affects processes under the debugger. Other bits |
816 | * should stay 0 to avoid the debugger interfering with other |
817 | * processes. |
818 | */ |
819 | if (trap_override != KFD_DBG_TRAP_OVERRIDE_OR) |
820 | return -EINVAL; |
821 | |
822 | return 0; |
823 | } |
824 | |
825 | uint32_t kgd_gfx_v10_set_wave_launch_trap_override(struct amdgpu_device *adev, |
826 | uint32_t vmid, |
827 | uint32_t trap_override, |
828 | uint32_t trap_mask_bits, |
829 | uint32_t trap_mask_request, |
830 | uint32_t *trap_mask_prev, |
831 | uint32_t kfd_dbg_trap_cntl_prev) |
832 | { |
833 | uint32_t data, wave_cntl_prev; |
834 | |
835 | mutex_lock(&adev->grbm_idx_mutex); |
836 | |
837 | wave_cntl_prev = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL)); |
838 | |
839 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: true); |
840 | |
841 | data = RREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK)); |
842 | *trap_mask_prev = REG_GET_FIELD(data, SPI_GDBG_TRAP_MASK, EXCP_EN); |
843 | |
844 | trap_mask_bits = (trap_mask_bits & trap_mask_request) | |
845 | (*trap_mask_prev & ~trap_mask_request); |
846 | |
847 | data = REG_SET_FIELD(data, SPI_GDBG_TRAP_MASK, EXCP_EN, trap_mask_bits); |
848 | data = REG_SET_FIELD(data, SPI_GDBG_TRAP_MASK, REPLACE, trap_override); |
849 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), data); |
850 | |
851 | /* We need to preserve wave launch mode stall settings. */ |
852 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL), wave_cntl_prev); |
853 | |
854 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
855 | |
856 | return 0; |
857 | } |
858 | |
859 | uint32_t kgd_gfx_v10_set_wave_launch_mode(struct amdgpu_device *adev, |
860 | uint8_t wave_launch_mode, |
861 | uint32_t vmid) |
862 | { |
863 | uint32_t data = 0; |
864 | bool is_mode_set = !!wave_launch_mode; |
865 | |
866 | mutex_lock(&adev->grbm_idx_mutex); |
867 | |
868 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: true); |
869 | |
870 | data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL2, |
871 | VMID_MASK, is_mode_set ? 1 << vmid : 0); |
872 | data = REG_SET_FIELD(data, SPI_GDBG_WAVE_CNTL2, |
873 | MODE, is_mode_set ? wave_launch_mode : 0); |
874 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_WAVE_CNTL2), data); |
875 | |
876 | kgd_gfx_v10_set_wave_launch_stall(adev, vmid, stall: false); |
877 | |
878 | mutex_unlock(lock: &adev->grbm_idx_mutex); |
879 | |
880 | return 0; |
881 | } |
882 | |
883 | #define TCP_WATCH_STRIDE (mmTCP_WATCH1_ADDR_H - mmTCP_WATCH0_ADDR_H) |
884 | uint32_t kgd_gfx_v10_set_address_watch(struct amdgpu_device *adev, |
885 | uint64_t watch_address, |
886 | uint32_t watch_address_mask, |
887 | uint32_t watch_id, |
888 | uint32_t watch_mode, |
889 | uint32_t debug_vmid, |
890 | uint32_t inst) |
891 | { |
892 | uint32_t watch_address_high; |
893 | uint32_t watch_address_low; |
894 | uint32_t watch_address_cntl; |
895 | |
896 | watch_address_cntl = 0; |
897 | |
898 | watch_address_low = lower_32_bits(watch_address); |
899 | watch_address_high = upper_32_bits(watch_address) & 0xffff; |
900 | |
901 | watch_address_cntl = REG_SET_FIELD(watch_address_cntl, |
902 | TCP_WATCH0_CNTL, |
903 | VMID, |
904 | debug_vmid); |
905 | watch_address_cntl = REG_SET_FIELD(watch_address_cntl, |
906 | TCP_WATCH0_CNTL, |
907 | MODE, |
908 | watch_mode); |
909 | watch_address_cntl = REG_SET_FIELD(watch_address_cntl, |
910 | TCP_WATCH0_CNTL, |
911 | MASK, |
912 | watch_address_mask >> 7); |
913 | |
914 | /* Turning off this watch point until we set all the registers */ |
915 | watch_address_cntl = REG_SET_FIELD(watch_address_cntl, |
916 | TCP_WATCH0_CNTL, |
917 | VALID, |
918 | 0); |
919 | |
920 | WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) + |
921 | (watch_id * TCP_WATCH_STRIDE)), |
922 | watch_address_cntl); |
923 | |
924 | WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) + |
925 | (watch_id * TCP_WATCH_STRIDE)), |
926 | watch_address_high); |
927 | |
928 | WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_L) + |
929 | (watch_id * TCP_WATCH_STRIDE)), |
930 | watch_address_low); |
931 | |
932 | /* Enable the watch point */ |
933 | watch_address_cntl = REG_SET_FIELD(watch_address_cntl, |
934 | TCP_WATCH0_CNTL, |
935 | VALID, |
936 | 1); |
937 | |
938 | WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) + |
939 | (watch_id * TCP_WATCH_STRIDE)), |
940 | watch_address_cntl); |
941 | |
942 | return 0; |
943 | } |
944 | |
945 | uint32_t kgd_gfx_v10_clear_address_watch(struct amdgpu_device *adev, |
946 | uint32_t watch_id) |
947 | { |
948 | uint32_t watch_address_cntl; |
949 | |
950 | watch_address_cntl = 0; |
951 | |
952 | WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) + |
953 | (watch_id * TCP_WATCH_STRIDE)), |
954 | watch_address_cntl); |
955 | |
956 | return 0; |
957 | } |
958 | |
959 | |
960 | /* kgd_gfx_v10_get_iq_wait_times: Returns the mmCP_IQ_WAIT_TIME1/2 values |
961 | * The values read are: |
962 | * ib_offload_wait_time -- Wait Count for Indirect Buffer Offloads. |
963 | * atomic_offload_wait_time -- Wait Count for L2 and GDS Atomics Offloads. |
964 | * wrm_offload_wait_time -- Wait Count for WAIT_REG_MEM Offloads. |
965 | * gws_wait_time -- Wait Count for Global Wave Syncs. |
966 | * que_sleep_wait_time -- Wait Count for Dequeue Retry. |
967 | * sch_wave_wait_time -- Wait Count for Scheduling Wave Message. |
968 | * sem_rearm_wait_time -- Wait Count for Semaphore re-arm. |
969 | * deq_retry_wait_time -- Wait Count for Global Wave Syncs. |
970 | */ |
971 | void kgd_gfx_v10_get_iq_wait_times(struct amdgpu_device *adev, |
972 | uint32_t *wait_times, |
973 | uint32_t inst) |
974 | |
975 | { |
976 | *wait_times = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2)); |
977 | } |
978 | |
979 | void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev, |
980 | uint32_t wait_times, |
981 | uint32_t grace_period, |
982 | uint32_t *reg_offset, |
983 | uint32_t *reg_data) |
984 | { |
985 | *reg_data = wait_times; |
986 | |
987 | /* |
988 | * The CP cannont handle a 0 grace period input and will result in |
989 | * an infinite grace period being set so set to 1 to prevent this. |
990 | */ |
991 | if (grace_period == 0) |
992 | grace_period = 1; |
993 | |
994 | *reg_data = REG_SET_FIELD(*reg_data, |
995 | CP_IQ_WAIT_TIME2, |
996 | SCH_WAVE, |
997 | grace_period); |
998 | |
999 | *reg_offset = SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2); |
1000 | } |
1001 | |
1002 | static void program_trap_handler_settings(struct amdgpu_device *adev, |
1003 | uint32_t vmid, uint64_t tba_addr, uint64_t tma_addr, |
1004 | uint32_t inst) |
1005 | { |
1006 | lock_srbm(adev, mec: 0, pipe: 0, queue: 0, vmid); |
1007 | |
1008 | /* |
1009 | * Program TBA registers |
1010 | */ |
1011 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_LO), |
1012 | lower_32_bits(tba_addr >> 8)); |
1013 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TBA_HI), |
1014 | upper_32_bits(tba_addr >> 8) | |
1015 | (1 << SQ_SHADER_TBA_HI__TRAP_EN__SHIFT)); |
1016 | |
1017 | /* |
1018 | * Program TMA registers |
1019 | */ |
1020 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_LO), |
1021 | lower_32_bits(tma_addr >> 8)); |
1022 | WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_SHADER_TMA_HI), |
1023 | upper_32_bits(tma_addr >> 8)); |
1024 | |
1025 | unlock_srbm(adev); |
1026 | } |
1027 | |
1028 | const struct kfd2kgd_calls gfx_v10_kfd2kgd = { |
1029 | .program_sh_mem_settings = kgd_program_sh_mem_settings, |
1030 | .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, |
1031 | .init_interrupts = kgd_init_interrupts, |
1032 | .hqd_load = kgd_hqd_load, |
1033 | .hiq_mqd_load = kgd_hiq_mqd_load, |
1034 | .hqd_sdma_load = kgd_hqd_sdma_load, |
1035 | .hqd_dump = kgd_hqd_dump, |
1036 | .hqd_sdma_dump = kgd_hqd_sdma_dump, |
1037 | .hqd_is_occupied = kgd_hqd_is_occupied, |
1038 | .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, |
1039 | .hqd_destroy = kgd_hqd_destroy, |
1040 | .hqd_sdma_destroy = kgd_hqd_sdma_destroy, |
1041 | .wave_control_execute = kgd_wave_control_execute, |
1042 | .get_atc_vmid_pasid_mapping_info = |
1043 | get_atc_vmid_pasid_mapping_info, |
1044 | .set_vm_context_page_table_base = set_vm_context_page_table_base, |
1045 | .enable_debug_trap = kgd_gfx_v10_enable_debug_trap, |
1046 | .disable_debug_trap = kgd_gfx_v10_disable_debug_trap, |
1047 | .validate_trap_override_request = kgd_gfx_v10_validate_trap_override_request, |
1048 | .set_wave_launch_trap_override = kgd_gfx_v10_set_wave_launch_trap_override, |
1049 | .set_wave_launch_mode = kgd_gfx_v10_set_wave_launch_mode, |
1050 | .set_address_watch = kgd_gfx_v10_set_address_watch, |
1051 | .clear_address_watch = kgd_gfx_v10_clear_address_watch, |
1052 | .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times, |
1053 | .build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info, |
1054 | .program_trap_handler_settings = program_trap_handler_settings, |
1055 | }; |
1056 | |