1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright 2018-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24#include <linux/kconfig.h>
25
26#if IS_REACHABLE(CONFIG_AMD_IOMMU_V2)
27
28#include <linux/printk.h>
29#include <linux/device.h>
30#include <linux/slab.h>
31#include <linux/pci.h>
32#include <linux/amd-iommu.h>
33#include "kfd_priv.h"
34#include "kfd_topology.h"
35#include "kfd_iommu.h"
36
37static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
38 AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
39 AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
40
41/** kfd_iommu_check_device - Check whether IOMMU is available for device
42 */
43int kfd_iommu_check_device(struct kfd_dev *kfd)
44{
45 struct amd_iommu_device_info iommu_info;
46 int err;
47
48 if (!kfd->use_iommu_v2)
49 return -ENODEV;
50
51 iommu_info.flags = 0;
52 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
53 if (err)
54 return err;
55
56 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
57 return -ENODEV;
58
59 return 0;
60}
61
62/** kfd_iommu_device_init - Initialize IOMMU for device
63 */
64int kfd_iommu_device_init(struct kfd_dev *kfd)
65{
66 struct amd_iommu_device_info iommu_info;
67 unsigned int pasid_limit;
68 int err;
69
70 if (!kfd->use_iommu_v2)
71 return 0;
72
73 iommu_info.flags = 0;
74 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
75 if (err < 0) {
76 dev_err(kfd_device,
77 "error getting iommu info. is the iommu enabled?\n");
78 return -ENODEV;
79 }
80
81 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
82 dev_err(kfd_device,
83 "error required iommu flags ats %i, pri %i, pasid %i\n",
84 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
85 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
86 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
87 != 0);
88 return -ENODEV;
89 }
90
91 pasid_limit = min_t(unsigned int,
92 (unsigned int)(1 << kfd->device_info.max_pasid_bits),
93 iommu_info.max_pasids);
94
95 if (!kfd_set_pasid_limit(pasid_limit)) {
96 dev_err(kfd_device, "error setting pasid limit\n");
97 return -EBUSY;
98 }
99
100 return 0;
101}
102
103/** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
104 *
105 * Binds the given process to the given device using its PASID. This
106 * enables IOMMUv2 address translation for the process on the device.
107 *
108 * This function assumes that the process mutex is held.
109 */
110int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
111{
112 struct kfd_dev *dev = pdd->dev;
113 struct kfd_process *p = pdd->process;
114 int err;
115
116 if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND)
117 return 0;
118
119 if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
120 pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
121 return -EINVAL;
122 }
123
124 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
125 if (!err)
126 pdd->bound = PDD_BOUND;
127
128 return err;
129}
130
131/** kfd_iommu_unbind_process - Unbind process from all devices
132 *
133 * This removes all IOMMU device bindings of the process. To be used
134 * before process termination.
135 */
136void kfd_iommu_unbind_process(struct kfd_process *p)
137{
138 int i;
139
140 for (i = 0; i < p->n_pdds; i++)
141 if (p->pdds[i]->bound == PDD_BOUND)
142 amd_iommu_unbind_pasid(p->pdds[i]->dev->pdev, p->pasid);
143}
144
145/* Callback for process shutdown invoked by the IOMMU driver */
146static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
147{
148 struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
149 struct kfd_process *p;
150 struct kfd_process_device *pdd;
151
152 if (!dev)
153 return;
154
155 /*
156 * Look for the process that matches the pasid. If there is no such
157 * process, we either released it in amdkfd's own notifier, or there
158 * is a bug. Unfortunately, there is no way to tell...
159 */
160 p = kfd_lookup_process_by_pasid(pasid);
161 if (!p)
162 return;
163
164 pr_debug("Unbinding process 0x%x from IOMMU\n", pasid);
165
166 mutex_lock(&p->mutex);
167
168 pdd = kfd_get_process_device_data(dev, p);
169 if (pdd)
170 /* For GPU relying on IOMMU, we need to dequeue here
171 * when PASID is still bound.
172 */
173 kfd_process_dequeue_from_device(pdd);
174
175 mutex_unlock(&p->mutex);
176
177 kfd_unref_process(p);
178}
179
180/* This function called by IOMMU driver on PPR failure */
181static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
182 unsigned long address, u16 flags)
183{
184 struct kfd_dev *dev;
185
186 dev_warn_ratelimited(kfd_device,
187 "Invalid PPR device %x:%x.%x pasid 0x%x address 0x%lX flags 0x%X",
188 pdev->bus->number,
189 PCI_SLOT(pdev->devfn),
190 PCI_FUNC(pdev->devfn),
191 pasid,
192 address,
193 flags);
194
195 dev = kfd_device_by_pci_dev(pdev);
196 if (!WARN_ON(!dev))
197 kfd_signal_iommu_event(dev, pasid, address,
198 flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
199
200 return AMD_IOMMU_INV_PRI_RSP_INVALID;
201}
202
203/*
204 * Bind processes do the device that have been temporarily unbound
205 * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
206 */
207static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
208{
209 struct kfd_process_device *pdd;
210 struct kfd_process *p;
211 unsigned int temp;
212 int err = 0;
213
214 int idx = srcu_read_lock(&kfd_processes_srcu);
215
216 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
217 mutex_lock(&p->mutex);
218 pdd = kfd_get_process_device_data(kfd, p);
219
220 if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
221 mutex_unlock(&p->mutex);
222 continue;
223 }
224
225 err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
226 p->lead_thread);
227 if (err < 0) {
228 pr_err("Unexpected pasid 0x%x binding failure\n",
229 p->pasid);
230 mutex_unlock(&p->mutex);
231 break;
232 }
233
234 pdd->bound = PDD_BOUND;
235 mutex_unlock(&p->mutex);
236 }
237
238 srcu_read_unlock(&kfd_processes_srcu, idx);
239
240 return err;
241}
242
243/*
244 * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
245 * processes will be restored to PDD_BOUND state in
246 * kfd_bind_processes_to_device.
247 */
248static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
249{
250 struct kfd_process_device *pdd;
251 struct kfd_process *p;
252 unsigned int temp;
253
254 int idx = srcu_read_lock(&kfd_processes_srcu);
255
256 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
257 mutex_lock(&p->mutex);
258 pdd = kfd_get_process_device_data(kfd, p);
259
260 if (WARN_ON(!pdd)) {
261 mutex_unlock(&p->mutex);
262 continue;
263 }
264
265 if (pdd->bound == PDD_BOUND)
266 pdd->bound = PDD_BOUND_SUSPENDED;
267 mutex_unlock(&p->mutex);
268 }
269
270 srcu_read_unlock(&kfd_processes_srcu, idx);
271}
272
273/** kfd_iommu_suspend - Prepare IOMMU for suspend
274 *
275 * This unbinds processes from the device and disables the IOMMU for
276 * the device.
277 */
278void kfd_iommu_suspend(struct kfd_dev *kfd)
279{
280 if (!kfd->use_iommu_v2)
281 return;
282
283 kfd_unbind_processes_from_device(kfd);
284
285 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
286 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
287 amd_iommu_free_device(kfd->pdev);
288}
289
290/** kfd_iommu_resume - Restore IOMMU after resume
291 *
292 * This reinitializes the IOMMU for the device and re-binds previously
293 * suspended processes to the device.
294 */
295int kfd_iommu_resume(struct kfd_dev *kfd)
296{
297 unsigned int pasid_limit;
298 int err;
299
300 if (!kfd->use_iommu_v2)
301 return 0;
302
303 pasid_limit = kfd_get_pasid_limit();
304
305 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
306 if (err)
307 return -ENXIO;
308
309 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
310 iommu_pasid_shutdown_callback);
311 amd_iommu_set_invalid_ppr_cb(kfd->pdev,
312 iommu_invalid_ppr_cb);
313
314 err = kfd_bind_processes_to_device(kfd);
315 if (err) {
316 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
317 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
318 amd_iommu_free_device(kfd->pdev);
319 return err;
320 }
321
322 return 0;
323}
324
325/** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
326 */
327int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
328{
329 struct kfd_perf_properties *props;
330
331 if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
332 return 0;
333
334 if (!amd_iommu_pc_supported())
335 return 0;
336
337 props = kfd_alloc_struct(props);
338 if (!props)
339 return -ENOMEM;
340 strcpy(props->block_name, "iommu");
341 props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
342 amd_iommu_pc_get_max_counters(0); /* assume one iommu */
343 list_add_tail(&props->list, &kdev->perf_props);
344
345 return 0;
346}
347
348#endif
349

source code of linux/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c