1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2016-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | */ |
23 | |
24 | #include "kfd_priv.h" |
25 | #include "kfd_events.h" |
26 | #include "kfd_debug.h" |
27 | #include "soc15_int.h" |
28 | #include "kfd_device_queue_manager.h" |
29 | #include "kfd_smi_events.h" |
30 | |
31 | /* |
32 | * GFX9 SQ Interrupts |
33 | * |
34 | * There are 3 encoding types of interrupts sourced from SQ sent as a 44-bit |
35 | * packet to the Interrupt Handler: |
36 | * Auto - Generated by the SQG (various cmd overflows, timestamps etc) |
37 | * Wave - Generated by S_SENDMSG through a shader program |
38 | * Error - HW generated errors (Illegal instructions, Memviols, EDC etc) |
39 | * |
40 | * The 44-bit packet is mapped as {context_id1[7:0],context_id0[31:0]} plus |
41 | * 4-bits for VMID (SOC15_VMID_FROM_IH_ENTRY) as such: |
42 | * |
43 | * - context_id0[27:26] |
44 | * Encoding type (0 = Auto, 1 = Wave, 2 = Error) |
45 | * |
46 | * - context_id0[13] |
47 | * PRIV bit indicates that Wave S_SEND or error occurred within trap |
48 | * |
49 | * - {context_id1[7:0],context_id0[31:28],context_id0[11:0]} |
50 | * 24-bit data with the following layout per encoding type: |
51 | * Auto - only context_id0[8:0] is used, which reports various interrupts |
52 | * generated by SQG. The rest is 0. |
53 | * Wave - user data sent from m0 via S_SENDMSG |
54 | * Error - Error type (context_id1[7:4]), Error Details (rest of bits) |
55 | * |
56 | * The other context_id bits show coordinates (SE/SH/CU/SIMD/WAVE) for wave |
57 | * S_SENDMSG and Errors. These are 0 for Auto. |
58 | */ |
59 | |
60 | enum SQ_INTERRUPT_WORD_ENCODING { |
61 | SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0, |
62 | SQ_INTERRUPT_WORD_ENCODING_INST, |
63 | SQ_INTERRUPT_WORD_ENCODING_ERROR, |
64 | }; |
65 | |
66 | enum SQ_INTERRUPT_ERROR_TYPE { |
67 | SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0, |
68 | SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST, |
69 | SQ_INTERRUPT_ERROR_TYPE_MEMVIOL, |
70 | SQ_INTERRUPT_ERROR_TYPE_EDC_FED, |
71 | }; |
72 | |
73 | /* SQ_INTERRUPT_WORD_AUTO_CTXID */ |
74 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT 0 |
75 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT 1 |
76 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT 2 |
77 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT 3 |
78 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT 4 |
79 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT 5 |
80 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT 6 |
81 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT 7 |
82 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT 8 |
83 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT 24 |
84 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT 26 |
85 | |
86 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK 0x00000001 |
87 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK 0x00000002 |
88 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK 0x00000004 |
89 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK 0x00000008 |
90 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK 0x00000010 |
91 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK 0x00000020 |
92 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK 0x00000040 |
93 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK 0x00000080 |
94 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK 0x00000100 |
95 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK 0x03000000 |
96 | #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK 0x0c000000 |
97 | |
98 | /* SQ_INTERRUPT_WORD_WAVE_CTXID */ |
99 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT 0 |
100 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT 12 |
101 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT 13 |
102 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT 14 |
103 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT 18 |
104 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT 20 |
105 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT 24 |
106 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT 26 |
107 | |
108 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK 0x00000fff |
109 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK 0x00001000 |
110 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK 0x00002000 |
111 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK 0x0003c000 |
112 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK 0x000c0000 |
113 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK 0x00f00000 |
114 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK 0x03000000 |
115 | #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK 0x0c000000 |
116 | |
117 | /* GFX9 SQ interrupt 24-bit data from context_id<0,1> */ |
118 | #define KFD_CONTEXT_ID_GET_SQ_INT_DATA(ctx0, ctx1) \ |
119 | ((ctx0 & 0xfff) | ((ctx0 >> 16) & 0xf000) | ((ctx1 << 16) & 0xff0000)) |
120 | |
121 | #define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000 |
122 | #define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20 |
123 | |
124 | /* |
125 | * The debugger will send user data(m0) with PRIV=1 to indicate it requires |
126 | * notification from the KFD with the following queue id (DOORBELL_ID) and |
127 | * trap code (TRAP_CODE). |
128 | */ |
129 | #define KFD_INT_DATA_DEBUG_DOORBELL_MASK 0x0003ff |
130 | #define KFD_INT_DATA_DEBUG_TRAP_CODE_SHIFT 10 |
131 | #define KFD_INT_DATA_DEBUG_TRAP_CODE_MASK 0x07fc00 |
132 | #define KFD_DEBUG_DOORBELL_ID(sq_int_data) ((sq_int_data) & \ |
133 | KFD_INT_DATA_DEBUG_DOORBELL_MASK) |
134 | #define KFD_DEBUG_TRAP_CODE(sq_int_data) (((sq_int_data) & \ |
135 | KFD_INT_DATA_DEBUG_TRAP_CODE_MASK) \ |
136 | >> KFD_INT_DATA_DEBUG_TRAP_CODE_SHIFT) |
137 | #define KFD_DEBUG_CP_BAD_OP_ECODE_MASK 0x3fffc00 |
138 | #define KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT 10 |
139 | #define KFD_DEBUG_CP_BAD_OP_ECODE(ctxid0) (((ctxid0) & \ |
140 | KFD_DEBUG_CP_BAD_OP_ECODE_MASK) \ |
141 | >> KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT) |
142 | |
143 | static void event_interrupt_poison_consumption_v9(struct kfd_node *dev, |
144 | uint16_t pasid, uint16_t client_id) |
145 | { |
146 | int old_poison, ret = -EINVAL; |
147 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); |
148 | |
149 | if (!p) |
150 | return; |
151 | |
152 | /* all queues of a process will be unmapped in one time */ |
153 | old_poison = atomic_cmpxchg(v: &p->poison, old: 0, new: 1); |
154 | kfd_unref_process(p); |
155 | if (old_poison) |
156 | return; |
157 | |
158 | switch (client_id) { |
159 | case SOC15_IH_CLIENTID_SE0SH: |
160 | case SOC15_IH_CLIENTID_SE1SH: |
161 | case SOC15_IH_CLIENTID_SE2SH: |
162 | case SOC15_IH_CLIENTID_SE3SH: |
163 | case SOC15_IH_CLIENTID_UTCL2: |
164 | ret = kfd_dqm_evict_pasid(dqm: dev->dqm, pasid); |
165 | break; |
166 | case SOC15_IH_CLIENTID_SDMA0: |
167 | case SOC15_IH_CLIENTID_SDMA1: |
168 | case SOC15_IH_CLIENTID_SDMA2: |
169 | case SOC15_IH_CLIENTID_SDMA3: |
170 | case SOC15_IH_CLIENTID_SDMA4: |
171 | break; |
172 | default: |
173 | break; |
174 | } |
175 | |
176 | kfd_signal_poison_consumed_event(dev, pasid); |
177 | |
178 | /* resetting queue passes, do page retirement without gpu reset |
179 | * resetting queue fails, fallback to gpu reset solution |
180 | */ |
181 | if (!ret) { |
182 | dev_warn(dev->adev->dev, |
183 | "RAS poison consumption, unmap queue flow succeeded: client id %d\n" , |
184 | client_id); |
185 | amdgpu_amdkfd_ras_poison_consumption_handler(adev: dev->adev, reset: false); |
186 | } else { |
187 | dev_warn(dev->adev->dev, |
188 | "RAS poison consumption, fall back to gpu reset flow: client id %d\n" , |
189 | client_id); |
190 | amdgpu_amdkfd_ras_poison_consumption_handler(adev: dev->adev, reset: true); |
191 | } |
192 | } |
193 | |
194 | static bool context_id_expected(struct kfd_dev *dev) |
195 | { |
196 | switch (KFD_GC_VERSION(dev)) { |
197 | case IP_VERSION(9, 0, 1): |
198 | return dev->mec_fw_version >= 0x817a; |
199 | case IP_VERSION(9, 1, 0): |
200 | case IP_VERSION(9, 2, 1): |
201 | case IP_VERSION(9, 2, 2): |
202 | case IP_VERSION(9, 3, 0): |
203 | case IP_VERSION(9, 4, 0): |
204 | return dev->mec_fw_version >= 0x17a; |
205 | default: |
206 | /* Other GFXv9 and later GPUs always sent valid context IDs |
207 | * on legitimate events |
208 | */ |
209 | return KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 1); |
210 | } |
211 | } |
212 | |
213 | static bool event_interrupt_isr_v9(struct kfd_node *dev, |
214 | const uint32_t *ih_ring_entry, |
215 | uint32_t *patched_ihre, |
216 | bool *patched_flag) |
217 | { |
218 | uint16_t source_id, client_id, pasid, vmid; |
219 | const uint32_t *data = ih_ring_entry; |
220 | |
221 | source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry); |
222 | client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); |
223 | |
224 | /* Only handle interrupts from KFD VMIDs */ |
225 | vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); |
226 | if (!KFD_IRQ_IS_FENCE(client_id, source_id) && |
227 | (vmid < dev->vm_info.first_vmid_kfd || |
228 | vmid > dev->vm_info.last_vmid_kfd)) |
229 | return false; |
230 | |
231 | pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); |
232 | |
233 | /* Only handle clients we care about */ |
234 | if (client_id != SOC15_IH_CLIENTID_GRBM_CP && |
235 | client_id != SOC15_IH_CLIENTID_SDMA0 && |
236 | client_id != SOC15_IH_CLIENTID_SDMA1 && |
237 | client_id != SOC15_IH_CLIENTID_SDMA2 && |
238 | client_id != SOC15_IH_CLIENTID_SDMA3 && |
239 | client_id != SOC15_IH_CLIENTID_SDMA4 && |
240 | client_id != SOC15_IH_CLIENTID_SDMA5 && |
241 | client_id != SOC15_IH_CLIENTID_SDMA6 && |
242 | client_id != SOC15_IH_CLIENTID_SDMA7 && |
243 | client_id != SOC15_IH_CLIENTID_VMC && |
244 | client_id != SOC15_IH_CLIENTID_VMC1 && |
245 | client_id != SOC15_IH_CLIENTID_UTCL2 && |
246 | client_id != SOC15_IH_CLIENTID_SE0SH && |
247 | client_id != SOC15_IH_CLIENTID_SE1SH && |
248 | client_id != SOC15_IH_CLIENTID_SE2SH && |
249 | client_id != SOC15_IH_CLIENTID_SE3SH && |
250 | !KFD_IRQ_IS_FENCE(client_id, source_id)) |
251 | return false; |
252 | |
253 | /* This is a known issue for gfx9. Under non HWS, pasid is not set |
254 | * in the interrupt payload, so we need to find out the pasid on our |
255 | * own. |
256 | */ |
257 | if (!pasid && dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { |
258 | const uint32_t pasid_mask = 0xffff; |
259 | |
260 | *patched_flag = true; |
261 | memcpy(patched_ihre, ih_ring_entry, |
262 | dev->kfd->device_info.ih_ring_entry_size); |
263 | |
264 | pasid = dev->dqm->vmid_pasid[vmid]; |
265 | |
266 | /* Patch the pasid field */ |
267 | patched_ihre[3] = cpu_to_le32((le32_to_cpu(patched_ihre[3]) |
268 | & ~pasid_mask) | pasid); |
269 | } |
270 | |
271 | pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n" , |
272 | client_id, source_id, vmid, pasid); |
273 | pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n" , |
274 | data[0], data[1], data[2], data[3], |
275 | data[4], data[5], data[6], data[7]); |
276 | |
277 | /* If there is no valid PASID, it's likely a bug */ |
278 | if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt" )) |
279 | return false; |
280 | |
281 | /* Workaround CP firmware sending bogus signals with 0 context_id. |
282 | * Those can be safely ignored on hardware and firmware versions that |
283 | * include a valid context_id on legitimate signals. This avoids the |
284 | * slow path in kfd_signal_event_interrupt that scans all event slots |
285 | * for signaled events. |
286 | */ |
287 | if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) { |
288 | uint32_t context_id = |
289 | SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry); |
290 | |
291 | if (context_id == 0 && context_id_expected(dev: dev->kfd)) |
292 | return false; |
293 | } |
294 | |
295 | /* Interrupt types we care about: various signals and faults. |
296 | * They will be forwarded to a work queue (see below). |
297 | */ |
298 | return source_id == SOC15_INTSRC_CP_END_OF_PIPE || |
299 | source_id == SOC15_INTSRC_SDMA_TRAP || |
300 | source_id == SOC15_INTSRC_SDMA_ECC || |
301 | source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG || |
302 | source_id == SOC15_INTSRC_CP_BAD_OPCODE || |
303 | KFD_IRQ_IS_FENCE(client_id, source_id) || |
304 | ((client_id == SOC15_IH_CLIENTID_VMC || |
305 | client_id == SOC15_IH_CLIENTID_VMC1 || |
306 | client_id == SOC15_IH_CLIENTID_UTCL2) && |
307 | !amdgpu_no_queue_eviction_on_vm_fault); |
308 | } |
309 | |
310 | static void event_interrupt_wq_v9(struct kfd_node *dev, |
311 | const uint32_t *ih_ring_entry) |
312 | { |
313 | uint16_t source_id, client_id, pasid, vmid; |
314 | uint32_t context_id0, context_id1; |
315 | uint32_t sq_intr_err, sq_int_data, encoding; |
316 | |
317 | source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry); |
318 | client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); |
319 | pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); |
320 | vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); |
321 | context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry); |
322 | context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry); |
323 | |
324 | if (client_id == SOC15_IH_CLIENTID_GRBM_CP || |
325 | client_id == SOC15_IH_CLIENTID_SE0SH || |
326 | client_id == SOC15_IH_CLIENTID_SE1SH || |
327 | client_id == SOC15_IH_CLIENTID_SE2SH || |
328 | client_id == SOC15_IH_CLIENTID_SE3SH) { |
329 | if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) |
330 | kfd_signal_event_interrupt(pasid, partial_id: context_id0, valid_id_bits: 32); |
331 | else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) { |
332 | sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1); |
333 | encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING); |
334 | switch (encoding) { |
335 | case SQ_INTERRUPT_WORD_ENCODING_AUTO: |
336 | pr_debug_ratelimited( |
337 | "sq_intr: auto, se %d, ttrace %d, wlt %d, ttrac_buf_full %d, reg_tms %d, cmd_tms %d, host_cmd_ovf %d, host_reg_ovf %d, immed_ovf %d, ttrace_utc_err %d\n" , |
338 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, SE_ID), |
339 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE), |
340 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, WLT), |
341 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_BUF_FULL), |
342 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, REG_TIMESTAMP), |
343 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, CMD_TIMESTAMP), |
344 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_CMD_OVERFLOW), |
345 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_REG_OVERFLOW), |
346 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, IMMED_OVERFLOW), |
347 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_UTC_ERROR)); |
348 | break; |
349 | case SQ_INTERRUPT_WORD_ENCODING_INST: |
350 | pr_debug_ratelimited("sq_intr: inst, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, intr_data 0x%x\n" , |
351 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID), |
352 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA), |
353 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID), |
354 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV), |
355 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID), |
356 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID), |
357 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID), |
358 | sq_int_data); |
359 | if (context_id0 & SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK) { |
360 | if (kfd_set_dbg_ev_from_interrupt(dev, pasid, |
361 | KFD_DEBUG_DOORBELL_ID(sq_int_data), |
362 | KFD_DEBUG_TRAP_CODE(sq_int_data), |
363 | NULL, exception_data_size: 0)) |
364 | return; |
365 | } |
366 | break; |
367 | case SQ_INTERRUPT_WORD_ENCODING_ERROR: |
368 | sq_intr_err = REG_GET_FIELD(sq_int_data, KFD_SQ_INT_DATA, ERR_TYPE); |
369 | pr_warn_ratelimited("sq_intr: error, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, err_type %d\n" , |
370 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID), |
371 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA), |
372 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID), |
373 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV), |
374 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID), |
375 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID), |
376 | REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID), |
377 | sq_intr_err); |
378 | if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST && |
379 | sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) { |
380 | event_interrupt_poison_consumption_v9(dev, pasid, client_id); |
381 | return; |
382 | } |
383 | break; |
384 | default: |
385 | break; |
386 | } |
387 | kfd_signal_event_interrupt(pasid, partial_id: sq_int_data, valid_id_bits: 24); |
388 | } else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) { |
389 | kfd_set_dbg_ev_from_interrupt(dev, pasid, |
390 | KFD_DEBUG_DOORBELL_ID(context_id0), |
391 | KFD_EC_MASK(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0)), |
392 | NULL, exception_data_size: 0); |
393 | } |
394 | } else if (client_id == SOC15_IH_CLIENTID_SDMA0 || |
395 | client_id == SOC15_IH_CLIENTID_SDMA1 || |
396 | client_id == SOC15_IH_CLIENTID_SDMA2 || |
397 | client_id == SOC15_IH_CLIENTID_SDMA3 || |
398 | client_id == SOC15_IH_CLIENTID_SDMA4 || |
399 | client_id == SOC15_IH_CLIENTID_SDMA5 || |
400 | client_id == SOC15_IH_CLIENTID_SDMA6 || |
401 | client_id == SOC15_IH_CLIENTID_SDMA7) { |
402 | if (source_id == SOC15_INTSRC_SDMA_TRAP) { |
403 | kfd_signal_event_interrupt(pasid, partial_id: context_id0 & 0xfffffff, valid_id_bits: 28); |
404 | } else if (source_id == SOC15_INTSRC_SDMA_ECC) { |
405 | event_interrupt_poison_consumption_v9(dev, pasid, client_id); |
406 | return; |
407 | } |
408 | } else if (client_id == SOC15_IH_CLIENTID_VMC || |
409 | client_id == SOC15_IH_CLIENTID_VMC1 || |
410 | client_id == SOC15_IH_CLIENTID_UTCL2) { |
411 | struct kfd_vm_fault_info info = {0}; |
412 | uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry); |
413 | struct kfd_hsa_memory_exception_data exception_data; |
414 | |
415 | if (client_id == SOC15_IH_CLIENTID_UTCL2 && |
416 | amdgpu_amdkfd_ras_query_utcl2_poison_status(adev: dev->adev)) { |
417 | event_interrupt_poison_consumption_v9(dev, pasid, client_id); |
418 | return; |
419 | } |
420 | |
421 | info.vmid = vmid; |
422 | info.mc_id = client_id; |
423 | info.page_addr = ih_ring_entry[4] | |
424 | (uint64_t)(ih_ring_entry[5] & 0xf) << 32; |
425 | info.prot_valid = ring_id & 0x08; |
426 | info.prot_read = ring_id & 0x10; |
427 | info.prot_write = ring_id & 0x20; |
428 | |
429 | memset(&exception_data, 0, sizeof(exception_data)); |
430 | exception_data.gpu_id = dev->id; |
431 | exception_data.va = (info.page_addr) << PAGE_SHIFT; |
432 | exception_data.failure.NotPresent = info.prot_valid ? 1 : 0; |
433 | exception_data.failure.NoExecute = info.prot_exec ? 1 : 0; |
434 | exception_data.failure.ReadOnly = info.prot_write ? 1 : 0; |
435 | exception_data.failure.imprecise = 0; |
436 | |
437 | kfd_set_dbg_ev_from_interrupt(dev, |
438 | pasid, |
439 | doorbell_id: -1, |
440 | KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION), |
441 | exception_data: &exception_data, |
442 | exception_data_size: sizeof(exception_data)); |
443 | kfd_smi_event_update_vmfault(dev, pasid); |
444 | } else if (KFD_IRQ_IS_FENCE(client_id, source_id)) { |
445 | kfd_process_close_interrupt_drain(pasid); |
446 | } |
447 | } |
448 | |
449 | static bool event_interrupt_isr_v9_4_3(struct kfd_node *node, |
450 | const uint32_t *ih_ring_entry, |
451 | uint32_t *patched_ihre, |
452 | bool *patched_flag) |
453 | { |
454 | uint16_t node_id, vmid; |
455 | |
456 | /* |
457 | * For GFX 9.4.3, process the interrupt if: |
458 | * - NodeID field in IH entry matches the corresponding bit |
459 | * set in interrupt_bitmap Bits 0-15. |
460 | * OR |
461 | * - If partition mode is CPX and interrupt came from |
462 | * Node_id 0,4,8,12, then check if the Bit (16 + client id) |
463 | * is set in interrupt bitmap Bits 16-31. |
464 | */ |
465 | node_id = SOC15_NODEID_FROM_IH_ENTRY(ih_ring_entry); |
466 | vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); |
467 | if (kfd_irq_is_from_node(node, node_id, vmid)) |
468 | return event_interrupt_isr_v9(dev: node, ih_ring_entry, |
469 | patched_ihre, patched_flag); |
470 | return false; |
471 | } |
472 | |
473 | const struct kfd_event_interrupt_class event_interrupt_class_v9 = { |
474 | .interrupt_isr = event_interrupt_isr_v9, |
475 | .interrupt_wq = event_interrupt_wq_v9, |
476 | }; |
477 | |
478 | const struct kfd_event_interrupt_class event_interrupt_class_v9_4_3 = { |
479 | .interrupt_isr = event_interrupt_isr_v9_4_3, |
480 | .interrupt_wq = event_interrupt_wq_v9, |
481 | }; |
482 | |