1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/dma-mapping.h> |
25 | |
26 | #include "amdgpu.h" |
27 | #include "amdgpu_ih.h" |
28 | |
29 | /** |
30 | * amdgpu_ih_ring_init - initialize the IH state |
31 | * |
32 | * @adev: amdgpu_device pointer |
33 | * @ih: ih ring to initialize |
34 | * @ring_size: ring size to allocate |
35 | * @use_bus_addr: true when we can use dma_alloc_coherent |
36 | * |
37 | * Initializes the IH state and allocates a buffer |
38 | * for the IH ring buffer. |
39 | * Returns 0 for success, errors for failure. |
40 | */ |
41 | int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, |
42 | unsigned ring_size, bool use_bus_addr) |
43 | { |
44 | u32 rb_bufsz; |
45 | int r; |
46 | |
47 | /* Align ring size */ |
48 | rb_bufsz = order_base_2(ring_size / 4); |
49 | ring_size = (1 << rb_bufsz) * 4; |
50 | ih->ring_size = ring_size; |
51 | ih->ptr_mask = ih->ring_size - 1; |
52 | ih->rptr = 0; |
53 | ih->use_bus_addr = use_bus_addr; |
54 | |
55 | if (use_bus_addr) { |
56 | dma_addr_t dma_addr; |
57 | |
58 | if (ih->ring) |
59 | return 0; |
60 | |
61 | /* add 8 bytes for the rptr/wptr shadows and |
62 | * add them to the end of the ring allocation. |
63 | */ |
64 | ih->ring = dma_alloc_coherent(dev: adev->dev, size: ih->ring_size + 8, |
65 | dma_handle: &dma_addr, GFP_KERNEL); |
66 | if (ih->ring == NULL) |
67 | return -ENOMEM; |
68 | |
69 | ih->gpu_addr = dma_addr; |
70 | ih->wptr_addr = dma_addr + ih->ring_size; |
71 | ih->wptr_cpu = &ih->ring[ih->ring_size / 4]; |
72 | ih->rptr_addr = dma_addr + ih->ring_size + 4; |
73 | ih->rptr_cpu = &ih->ring[(ih->ring_size / 4) + 1]; |
74 | } else { |
75 | unsigned wptr_offs, rptr_offs; |
76 | |
77 | r = amdgpu_device_wb_get(adev, wb: &wptr_offs); |
78 | if (r) |
79 | return r; |
80 | |
81 | r = amdgpu_device_wb_get(adev, wb: &rptr_offs); |
82 | if (r) { |
83 | amdgpu_device_wb_free(adev, wb: wptr_offs); |
84 | return r; |
85 | } |
86 | |
87 | r = amdgpu_bo_create_kernel(adev, size: ih->ring_size, PAGE_SIZE, |
88 | AMDGPU_GEM_DOMAIN_GTT, |
89 | bo_ptr: &ih->ring_obj, gpu_addr: &ih->gpu_addr, |
90 | cpu_addr: (void **)&ih->ring); |
91 | if (r) { |
92 | amdgpu_device_wb_free(adev, wb: rptr_offs); |
93 | amdgpu_device_wb_free(adev, wb: wptr_offs); |
94 | return r; |
95 | } |
96 | |
97 | ih->wptr_addr = adev->wb.gpu_addr + wptr_offs * 4; |
98 | ih->wptr_cpu = &adev->wb.wb[wptr_offs]; |
99 | ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4; |
100 | ih->rptr_cpu = &adev->wb.wb[rptr_offs]; |
101 | } |
102 | |
103 | init_waitqueue_head(&ih->wait_process); |
104 | return 0; |
105 | } |
106 | |
107 | /** |
108 | * amdgpu_ih_ring_fini - tear down the IH state |
109 | * |
110 | * @adev: amdgpu_device pointer |
111 | * @ih: ih ring to tear down |
112 | * |
113 | * Tears down the IH state and frees buffer |
114 | * used for the IH ring buffer. |
115 | */ |
116 | void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) |
117 | { |
118 | |
119 | if (!ih->ring) |
120 | return; |
121 | |
122 | if (ih->use_bus_addr) { |
123 | |
124 | /* add 8 bytes for the rptr/wptr shadows and |
125 | * add them to the end of the ring allocation. |
126 | */ |
127 | dma_free_coherent(dev: adev->dev, size: ih->ring_size + 8, |
128 | cpu_addr: (void *)ih->ring, dma_handle: ih->gpu_addr); |
129 | ih->ring = NULL; |
130 | } else { |
131 | amdgpu_bo_free_kernel(bo: &ih->ring_obj, gpu_addr: &ih->gpu_addr, |
132 | cpu_addr: (void **)&ih->ring); |
133 | amdgpu_device_wb_free(adev, wb: (ih->wptr_addr - ih->gpu_addr) / 4); |
134 | amdgpu_device_wb_free(adev, wb: (ih->rptr_addr - ih->gpu_addr) / 4); |
135 | } |
136 | } |
137 | |
138 | /** |
139 | * amdgpu_ih_ring_write - write IV to the ring buffer |
140 | * |
141 | * @adev: amdgpu_device pointer |
142 | * @ih: ih ring to write to |
143 | * @iv: the iv to write |
144 | * @num_dw: size of the iv in dw |
145 | * |
146 | * Writes an IV to the ring buffer using the CPU and increment the wptr. |
147 | * Used for testing and delegating IVs to a software ring. |
148 | */ |
149 | void amdgpu_ih_ring_write(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, |
150 | const uint32_t *iv, unsigned int num_dw) |
151 | { |
152 | uint32_t wptr = le32_to_cpu(*ih->wptr_cpu) >> 2; |
153 | unsigned int i; |
154 | |
155 | for (i = 0; i < num_dw; ++i) |
156 | ih->ring[wptr++] = cpu_to_le32(iv[i]); |
157 | |
158 | wptr <<= 2; |
159 | wptr &= ih->ptr_mask; |
160 | |
161 | /* Only commit the new wptr if we don't overflow */ |
162 | if (wptr != READ_ONCE(ih->rptr)) { |
163 | wmb(); |
164 | WRITE_ONCE(*ih->wptr_cpu, cpu_to_le32(wptr)); |
165 | } else if (adev->irq.retry_cam_enabled) { |
166 | dev_warn_once(adev->dev, "IH soft ring buffer overflow 0x%X, 0x%X\n" , |
167 | wptr, ih->rptr); |
168 | } |
169 | } |
170 | |
171 | /** |
172 | * amdgpu_ih_wait_on_checkpoint_process_ts - wait to process IVs up to checkpoint |
173 | * |
174 | * @adev: amdgpu_device pointer |
175 | * @ih: ih ring to process |
176 | * |
177 | * Used to ensure ring has processed IVs up to the checkpoint write pointer. |
178 | */ |
179 | int amdgpu_ih_wait_on_checkpoint_process_ts(struct amdgpu_device *adev, |
180 | struct amdgpu_ih_ring *ih) |
181 | { |
182 | uint32_t checkpoint_wptr; |
183 | uint64_t checkpoint_ts; |
184 | long timeout = HZ; |
185 | |
186 | if (!ih->enabled || adev->shutdown) |
187 | return -ENODEV; |
188 | |
189 | checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); |
190 | /* Order wptr with ring data. */ |
191 | rmb(); |
192 | checkpoint_ts = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); |
193 | |
194 | return wait_event_interruptible_timeout(ih->wait_process, |
195 | amdgpu_ih_ts_after(checkpoint_ts, ih->processed_timestamp) || |
196 | ih->rptr == amdgpu_ih_get_wptr(adev, ih), timeout); |
197 | } |
198 | |
199 | /** |
200 | * amdgpu_ih_process - interrupt handler |
201 | * |
202 | * @adev: amdgpu_device pointer |
203 | * @ih: ih ring to process |
204 | * |
205 | * Interrupt hander (VI), walk the IH ring. |
206 | * Returns irq process return code. |
207 | */ |
208 | int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) |
209 | { |
210 | unsigned int count; |
211 | u32 wptr; |
212 | |
213 | if (!ih->enabled || adev->shutdown) |
214 | return IRQ_NONE; |
215 | |
216 | wptr = amdgpu_ih_get_wptr(adev, ih); |
217 | |
218 | restart_ih: |
219 | count = AMDGPU_IH_MAX_NUM_IVS; |
220 | DRM_DEBUG("%s: rptr %d, wptr %d\n" , __func__, ih->rptr, wptr); |
221 | |
222 | /* Order reading of wptr vs. reading of IH ring data */ |
223 | rmb(); |
224 | |
225 | while (ih->rptr != wptr && --count) { |
226 | amdgpu_irq_dispatch(adev, ih); |
227 | ih->rptr &= ih->ptr_mask; |
228 | } |
229 | |
230 | amdgpu_ih_set_rptr(adev, ih); |
231 | wake_up_all(&ih->wait_process); |
232 | |
233 | /* make sure wptr hasn't changed while processing */ |
234 | wptr = amdgpu_ih_get_wptr(adev, ih); |
235 | if (wptr != ih->rptr) |
236 | goto restart_ih; |
237 | |
238 | return IRQ_HANDLED; |
239 | } |
240 | |
241 | /** |
242 | * amdgpu_ih_decode_iv_helper - decode an interrupt vector |
243 | * |
244 | * @adev: amdgpu_device pointer |
245 | * @ih: ih ring to process |
246 | * @entry: IV entry |
247 | * |
248 | * Decodes the interrupt vector at the current rptr |
249 | * position and also advance the position for Vega10 |
250 | * and later GPUs. |
251 | */ |
252 | void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, |
253 | struct amdgpu_ih_ring *ih, |
254 | struct amdgpu_iv_entry *entry) |
255 | { |
256 | /* wptr/rptr are in bytes! */ |
257 | u32 ring_index = ih->rptr >> 2; |
258 | uint32_t dw[8]; |
259 | |
260 | dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); |
261 | dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); |
262 | dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); |
263 | dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); |
264 | dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); |
265 | dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); |
266 | dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); |
267 | dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); |
268 | |
269 | entry->client_id = dw[0] & 0xff; |
270 | entry->src_id = (dw[0] >> 8) & 0xff; |
271 | entry->ring_id = (dw[0] >> 16) & 0xff; |
272 | entry->vmid = (dw[0] >> 24) & 0xf; |
273 | entry->vmid_src = (dw[0] >> 31); |
274 | entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); |
275 | entry->timestamp_src = dw[2] >> 31; |
276 | entry->pasid = dw[3] & 0xffff; |
277 | entry->node_id = (dw[3] >> 16) & 0xff; |
278 | entry->src_data[0] = dw[4]; |
279 | entry->src_data[1] = dw[5]; |
280 | entry->src_data[2] = dw[6]; |
281 | entry->src_data[3] = dw[7]; |
282 | |
283 | /* wptr/rptr are in bytes! */ |
284 | ih->rptr += 32; |
285 | } |
286 | |
287 | uint64_t amdgpu_ih_decode_iv_ts_helper(struct amdgpu_ih_ring *ih, u32 rptr, |
288 | signed int offset) |
289 | { |
290 | uint32_t iv_size = 32; |
291 | uint32_t ring_index; |
292 | uint32_t dw1, dw2; |
293 | |
294 | rptr += iv_size * offset; |
295 | ring_index = (rptr & ih->ptr_mask) >> 2; |
296 | |
297 | dw1 = le32_to_cpu(ih->ring[ring_index + 1]); |
298 | dw2 = le32_to_cpu(ih->ring[ring_index + 2]); |
299 | return dw1 | ((u64)(dw2 & 0xffff) << 32); |
300 | } |
301 | |