1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. |
4 | * Author: Joerg Roedel <joerg.roedel@amd.com> |
5 | */ |
6 | |
7 | #ifndef __LINUX_IOMMU_H |
8 | #define __LINUX_IOMMU_H |
9 | |
10 | #include <linux/scatterlist.h> |
11 | #include <linux/device.h> |
12 | #include <linux/types.h> |
13 | #include <linux/errno.h> |
14 | #include <linux/err.h> |
15 | #include <linux/of.h> |
16 | #include <linux/iova_bitmap.h> |
17 | #include <uapi/linux/iommu.h> |
18 | |
19 | #define IOMMU_READ (1 << 0) |
20 | #define IOMMU_WRITE (1 << 1) |
21 | #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */ |
22 | #define IOMMU_NOEXEC (1 << 3) |
23 | #define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */ |
24 | /* |
25 | * Where the bus hardware includes a privilege level as part of its access type |
26 | * markings, and certain devices are capable of issuing transactions marked as |
27 | * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other |
28 | * given permission flags only apply to accesses at the higher privilege level, |
29 | * and that unprivileged transactions should have as little access as possible. |
30 | * This would usually imply the same permissions as kernel mappings on the CPU, |
31 | * if the IOMMU page table format is equivalent. |
32 | */ |
33 | #define IOMMU_PRIV (1 << 5) |
34 | |
35 | struct iommu_ops; |
36 | struct iommu_group; |
37 | struct bus_type; |
38 | struct device; |
39 | struct iommu_domain; |
40 | struct iommu_domain_ops; |
41 | struct iommu_dirty_ops; |
42 | struct notifier_block; |
43 | struct iommu_sva; |
44 | struct iommu_fault_event; |
45 | struct iommu_dma_cookie; |
46 | |
47 | /* iommu fault flags */ |
48 | #define IOMMU_FAULT_READ 0x0 |
49 | #define IOMMU_FAULT_WRITE 0x1 |
50 | |
51 | typedef int (*iommu_fault_handler_t)(struct iommu_domain *, |
52 | struct device *, unsigned long, int, void *); |
53 | typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); |
54 | |
55 | struct iommu_domain_geometry { |
56 | dma_addr_t aperture_start; /* First address that can be mapped */ |
57 | dma_addr_t aperture_end; /* Last address that can be mapped */ |
58 | bool force_aperture; /* DMA only allowed in mappable range? */ |
59 | }; |
60 | |
61 | /* Domain feature flags */ |
62 | #define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */ |
63 | #define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API |
64 | implementation */ |
65 | #define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */ |
66 | #define __IOMMU_DOMAIN_DMA_FQ (1U << 3) /* DMA-API uses flush queue */ |
67 | |
68 | #define __IOMMU_DOMAIN_SVA (1U << 4) /* Shared process address space */ |
69 | |
70 | #define __IOMMU_DOMAIN_NESTED (1U << 6) /* User-managed address space nested |
71 | on a stage-2 translation */ |
72 | |
73 | #define IOMMU_DOMAIN_ALLOC_FLAGS ~__IOMMU_DOMAIN_DMA_FQ |
74 | /* |
75 | * This are the possible domain-types |
76 | * |
77 | * IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate |
78 | * devices |
79 | * IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses |
80 | * IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used |
81 | * for VMs |
82 | * IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations. |
83 | * This flag allows IOMMU drivers to implement |
84 | * certain optimizations for these domains |
85 | * IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB |
86 | * invalidation. |
87 | * IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses |
88 | * represented by mm_struct's. |
89 | */ |
90 | #define IOMMU_DOMAIN_BLOCKED (0U) |
91 | #define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT) |
92 | #define IOMMU_DOMAIN_UNMANAGED (__IOMMU_DOMAIN_PAGING) |
93 | #define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \ |
94 | __IOMMU_DOMAIN_DMA_API) |
95 | #define IOMMU_DOMAIN_DMA_FQ (__IOMMU_DOMAIN_PAGING | \ |
96 | __IOMMU_DOMAIN_DMA_API | \ |
97 | __IOMMU_DOMAIN_DMA_FQ) |
98 | #define IOMMU_DOMAIN_SVA (__IOMMU_DOMAIN_SVA) |
99 | #define IOMMU_DOMAIN_NESTED (__IOMMU_DOMAIN_NESTED) |
100 | |
101 | struct iommu_domain { |
102 | unsigned type; |
103 | const struct iommu_domain_ops *ops; |
104 | const struct iommu_dirty_ops *dirty_ops; |
105 | |
106 | unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ |
107 | struct iommu_domain_geometry geometry; |
108 | struct iommu_dma_cookie *iova_cookie; |
109 | enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault, |
110 | void *data); |
111 | void *fault_data; |
112 | union { |
113 | struct { |
114 | iommu_fault_handler_t handler; |
115 | void *handler_token; |
116 | }; |
117 | struct { /* IOMMU_DOMAIN_SVA */ |
118 | struct mm_struct *mm; |
119 | int users; |
120 | }; |
121 | }; |
122 | }; |
123 | |
124 | static inline bool iommu_is_dma_domain(struct iommu_domain *domain) |
125 | { |
126 | return domain->type & __IOMMU_DOMAIN_DMA_API; |
127 | } |
128 | |
129 | enum iommu_cap { |
130 | IOMMU_CAP_CACHE_COHERENCY, /* IOMMU_CACHE is supported */ |
131 | IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ |
132 | IOMMU_CAP_PRE_BOOT_PROTECTION, /* Firmware says it used the IOMMU for |
133 | DMA protection and we should too */ |
134 | /* |
135 | * Per-device flag indicating if enforce_cache_coherency() will work on |
136 | * this device. |
137 | */ |
138 | IOMMU_CAP_ENFORCE_CACHE_COHERENCY, |
139 | /* |
140 | * IOMMU driver does not issue TLB maintenance during .unmap, so can |
141 | * usefully support the non-strict DMA flush queue. |
142 | */ |
143 | IOMMU_CAP_DEFERRED_FLUSH, |
144 | IOMMU_CAP_DIRTY_TRACKING, /* IOMMU supports dirty tracking */ |
145 | }; |
146 | |
147 | /* These are the possible reserved region types */ |
148 | enum iommu_resv_type { |
149 | /* Memory regions which must be mapped 1:1 at all times */ |
150 | IOMMU_RESV_DIRECT, |
151 | /* |
152 | * Memory regions which are advertised to be 1:1 but are |
153 | * commonly considered relaxable in some conditions, |
154 | * for instance in device assignment use case (USB, Graphics) |
155 | */ |
156 | IOMMU_RESV_DIRECT_RELAXABLE, |
157 | /* Arbitrary "never map this or give it to a device" address ranges */ |
158 | IOMMU_RESV_RESERVED, |
159 | /* Hardware MSI region (untranslated) */ |
160 | IOMMU_RESV_MSI, |
161 | /* Software-managed MSI translation window */ |
162 | IOMMU_RESV_SW_MSI, |
163 | }; |
164 | |
165 | /** |
166 | * struct iommu_resv_region - descriptor for a reserved memory region |
167 | * @list: Linked list pointers |
168 | * @start: System physical start address of the region |
169 | * @length: Length of the region in bytes |
170 | * @prot: IOMMU Protection flags (READ/WRITE/...) |
171 | * @type: Type of the reserved region |
172 | * @free: Callback to free associated memory allocations |
173 | */ |
174 | struct iommu_resv_region { |
175 | struct list_head list; |
176 | phys_addr_t start; |
177 | size_t length; |
178 | int prot; |
179 | enum iommu_resv_type type; |
180 | void (*free)(struct device *dev, struct iommu_resv_region *region); |
181 | }; |
182 | |
183 | struct iommu_iort_rmr_data { |
184 | struct iommu_resv_region rr; |
185 | |
186 | /* Stream IDs associated with IORT RMR entry */ |
187 | const u32 *sids; |
188 | u32 num_sids; |
189 | }; |
190 | |
191 | /** |
192 | * enum iommu_dev_features - Per device IOMMU features |
193 | * @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses |
194 | * @IOMMU_DEV_FEAT_IOPF: I/O Page Faults such as PRI or Stall. Generally |
195 | * enabling %IOMMU_DEV_FEAT_SVA requires |
196 | * %IOMMU_DEV_FEAT_IOPF, but some devices manage I/O Page |
197 | * Faults themselves instead of relying on the IOMMU. When |
198 | * supported, this feature must be enabled before and |
199 | * disabled after %IOMMU_DEV_FEAT_SVA. |
200 | * |
201 | * Device drivers enable a feature using iommu_dev_enable_feature(). |
202 | */ |
203 | enum iommu_dev_features { |
204 | IOMMU_DEV_FEAT_SVA, |
205 | IOMMU_DEV_FEAT_IOPF, |
206 | }; |
207 | |
208 | #define IOMMU_NO_PASID (0U) /* Reserved for DMA w/o PASID */ |
209 | #define IOMMU_FIRST_GLOBAL_PASID (1U) /*starting range for allocation */ |
210 | #define IOMMU_PASID_INVALID (-1U) |
211 | typedef unsigned int ioasid_t; |
212 | |
213 | #ifdef CONFIG_IOMMU_API |
214 | |
215 | /** |
216 | * struct iommu_iotlb_gather - Range information for a pending IOTLB flush |
217 | * |
218 | * @start: IOVA representing the start of the range to be flushed |
219 | * @end: IOVA representing the end of the range to be flushed (inclusive) |
220 | * @pgsize: The interval at which to perform the flush |
221 | * @freelist: Removed pages to free after sync |
222 | * @queued: Indicates that the flush will be queued |
223 | * |
224 | * This structure is intended to be updated by multiple calls to the |
225 | * ->unmap() function in struct iommu_ops before eventually being passed |
226 | * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after |
227 | * ->iotlb_sync() or ->iotlb_flush_all() have cleared all cached references to |
228 | * them. @queued is set to indicate when ->iotlb_flush_all() will be called |
229 | * later instead of ->iotlb_sync(), so drivers may optimise accordingly. |
230 | */ |
231 | struct iommu_iotlb_gather { |
232 | unsigned long start; |
233 | unsigned long end; |
234 | size_t pgsize; |
235 | struct list_head freelist; |
236 | bool queued; |
237 | }; |
238 | |
239 | /** |
240 | * struct iommu_dirty_bitmap - Dirty IOVA bitmap state |
241 | * @bitmap: IOVA bitmap |
242 | * @gather: Range information for a pending IOTLB flush |
243 | */ |
244 | struct iommu_dirty_bitmap { |
245 | struct iova_bitmap *bitmap; |
246 | struct iommu_iotlb_gather *gather; |
247 | }; |
248 | |
249 | /* Read but do not clear any dirty bits */ |
250 | #define IOMMU_DIRTY_NO_CLEAR (1 << 0) |
251 | |
252 | /** |
253 | * struct iommu_dirty_ops - domain specific dirty tracking operations |
254 | * @set_dirty_tracking: Enable or Disable dirty tracking on the iommu domain |
255 | * @read_and_clear_dirty: Walk IOMMU page tables for dirtied PTEs marshalled |
256 | * into a bitmap, with a bit represented as a page. |
257 | * Reads the dirty PTE bits and clears it from IO |
258 | * pagetables. |
259 | */ |
260 | struct iommu_dirty_ops { |
261 | int (*set_dirty_tracking)(struct iommu_domain *domain, bool enabled); |
262 | int (*read_and_clear_dirty)(struct iommu_domain *domain, |
263 | unsigned long iova, size_t size, |
264 | unsigned long flags, |
265 | struct iommu_dirty_bitmap *dirty); |
266 | }; |
267 | |
268 | /** |
269 | * struct iommu_user_data - iommu driver specific user space data info |
270 | * @type: The data type of the user buffer |
271 | * @uptr: Pointer to the user buffer for copy_from_user() |
272 | * @len: The length of the user buffer in bytes |
273 | * |
274 | * A user space data is an uAPI that is defined in include/uapi/linux/iommufd.h |
275 | * @type, @uptr and @len should be just copied from an iommufd core uAPI struct. |
276 | */ |
277 | struct iommu_user_data { |
278 | unsigned int type; |
279 | void __user *uptr; |
280 | size_t len; |
281 | }; |
282 | |
283 | /** |
284 | * __iommu_copy_struct_from_user - Copy iommu driver specific user space data |
285 | * @dst_data: Pointer to an iommu driver specific user data that is defined in |
286 | * include/uapi/linux/iommufd.h |
287 | * @src_data: Pointer to a struct iommu_user_data for user space data info |
288 | * @data_type: The data type of the @dst_data. Must match with @src_data.type |
289 | * @data_len: Length of current user data structure, i.e. sizeof(struct _dst) |
290 | * @min_len: Initial length of user data structure for backward compatibility. |
291 | * This should be offsetofend using the last member in the user data |
292 | * struct that was initially added to include/uapi/linux/iommufd.h |
293 | */ |
294 | static inline int __iommu_copy_struct_from_user( |
295 | void *dst_data, const struct iommu_user_data *src_data, |
296 | unsigned int data_type, size_t data_len, size_t min_len) |
297 | { |
298 | if (src_data->type != data_type) |
299 | return -EINVAL; |
300 | if (WARN_ON(!dst_data || !src_data)) |
301 | return -EINVAL; |
302 | if (src_data->len < min_len || data_len < src_data->len) |
303 | return -EINVAL; |
304 | return copy_struct_from_user(dst: dst_data, ksize: data_len, src: src_data->uptr, |
305 | usize: src_data->len); |
306 | } |
307 | |
308 | /** |
309 | * iommu_copy_struct_from_user - Copy iommu driver specific user space data |
310 | * @kdst: Pointer to an iommu driver specific user data that is defined in |
311 | * include/uapi/linux/iommufd.h |
312 | * @user_data: Pointer to a struct iommu_user_data for user space data info |
313 | * @data_type: The data type of the @kdst. Must match with @user_data->type |
314 | * @min_last: The last memember of the data structure @kdst points in the |
315 | * initial version. |
316 | * Return 0 for success, otherwise -error. |
317 | */ |
318 | #define iommu_copy_struct_from_user(kdst, user_data, data_type, min_last) \ |
319 | __iommu_copy_struct_from_user(kdst, user_data, data_type, \ |
320 | sizeof(*kdst), \ |
321 | offsetofend(typeof(*kdst), min_last)) |
322 | |
323 | /** |
324 | * struct iommu_ops - iommu ops and capabilities |
325 | * @capable: check capability |
326 | * @hw_info: report iommu hardware information. The data buffer returned by this |
327 | * op is allocated in the iommu driver and freed by the caller after |
328 | * use. The information type is one of enum iommu_hw_info_type defined |
329 | * in include/uapi/linux/iommufd.h. |
330 | * @domain_alloc: allocate and return an iommu domain if success. Otherwise |
331 | * NULL is returned. The domain is not fully initialized until |
332 | * the caller iommu_domain_alloc() returns. |
333 | * @domain_alloc_user: Allocate an iommu domain corresponding to the input |
334 | * parameters as defined in include/uapi/linux/iommufd.h. |
335 | * Unlike @domain_alloc, it is called only by IOMMUFD and |
336 | * must fully initialize the new domain before return. |
337 | * Upon success, if the @user_data is valid and the @parent |
338 | * points to a kernel-managed domain, the new domain must be |
339 | * IOMMU_DOMAIN_NESTED type; otherwise, the @parent must be |
340 | * NULL while the @user_data can be optionally provided, the |
341 | * new domain must support __IOMMU_DOMAIN_PAGING. |
342 | * Upon failure, ERR_PTR must be returned. |
343 | * @probe_device: Add device to iommu driver handling |
344 | * @release_device: Remove device from iommu driver handling |
345 | * @probe_finalize: Do final setup work after the device is added to an IOMMU |
346 | * group and attached to the groups domain |
347 | * @set_platform_dma_ops: Returning control back to the platform DMA ops. This op |
348 | * is to support old IOMMU drivers, new drivers should use |
349 | * default domains, and the common IOMMU DMA ops. |
350 | * @device_group: find iommu group for a particular device |
351 | * @get_resv_regions: Request list of reserved regions for a device |
352 | * @of_xlate: add OF master IDs to iommu grouping |
353 | * @is_attach_deferred: Check if domain attach should be deferred from iommu |
354 | * driver init to device driver init (default no) |
355 | * @dev_enable/disable_feat: per device entries to enable/disable |
356 | * iommu specific features. |
357 | * @page_response: handle page request response |
358 | * @def_domain_type: device default domain type, return value: |
359 | * - IOMMU_DOMAIN_IDENTITY: must use an identity domain |
360 | * - IOMMU_DOMAIN_DMA: must use a dma domain |
361 | * - 0: use the default setting |
362 | * @default_domain_ops: the default ops for domains |
363 | * @remove_dev_pasid: Remove any translation configurations of a specific |
364 | * pasid, so that any DMA transactions with this pasid |
365 | * will be blocked by the hardware. |
366 | * @pgsize_bitmap: bitmap of all possible supported page sizes |
367 | * @owner: Driver module providing these ops |
368 | */ |
369 | struct iommu_ops { |
370 | bool (*capable)(struct device *dev, enum iommu_cap); |
371 | void *(*hw_info)(struct device *dev, u32 *length, u32 *type); |
372 | |
373 | /* Domain allocation and freeing by the iommu driver */ |
374 | struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); |
375 | struct iommu_domain *(*domain_alloc_user)( |
376 | struct device *dev, u32 flags, struct iommu_domain *parent, |
377 | const struct iommu_user_data *user_data); |
378 | |
379 | struct iommu_device *(*probe_device)(struct device *dev); |
380 | void (*release_device)(struct device *dev); |
381 | void (*probe_finalize)(struct device *dev); |
382 | void (*set_platform_dma_ops)(struct device *dev); |
383 | struct iommu_group *(*device_group)(struct device *dev); |
384 | |
385 | /* Request/Free a list of reserved regions for a device */ |
386 | void (*get_resv_regions)(struct device *dev, struct list_head *list); |
387 | |
388 | int (*of_xlate)(struct device *dev, struct of_phandle_args *args); |
389 | bool (*is_attach_deferred)(struct device *dev); |
390 | |
391 | /* Per device IOMMU features */ |
392 | int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); |
393 | int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); |
394 | |
395 | int (*page_response)(struct device *dev, |
396 | struct iommu_fault_event *evt, |
397 | struct iommu_page_response *msg); |
398 | |
399 | int (*def_domain_type)(struct device *dev); |
400 | void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid); |
401 | |
402 | const struct iommu_domain_ops *default_domain_ops; |
403 | unsigned long pgsize_bitmap; |
404 | struct module *owner; |
405 | }; |
406 | |
407 | /** |
408 | * struct iommu_domain_ops - domain specific operations |
409 | * @attach_dev: attach an iommu domain to a device |
410 | * Return: |
411 | * * 0 - success |
412 | * * EINVAL - can indicate that device and domain are incompatible due to |
413 | * some previous configuration of the domain, in which case the |
414 | * driver shouldn't log an error, since it is legitimate for a |
415 | * caller to test reuse of existing domains. Otherwise, it may |
416 | * still represent some other fundamental problem |
417 | * * ENOMEM - out of memory |
418 | * * ENOSPC - non-ENOMEM type of resource allocation failures |
419 | * * EBUSY - device is attached to a domain and cannot be changed |
420 | * * ENODEV - device specific errors, not able to be attached |
421 | * * <others> - treated as ENODEV by the caller. Use is discouraged |
422 | * @set_dev_pasid: set an iommu domain to a pasid of device |
423 | * @map: map a physically contiguous memory region to an iommu domain |
424 | * @map_pages: map a physically contiguous set of pages of the same size to |
425 | * an iommu domain. |
426 | * @unmap: unmap a physically contiguous memory region from an iommu domain |
427 | * @unmap_pages: unmap a number of pages of the same size from an iommu domain |
428 | * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain |
429 | * @iotlb_sync_map: Sync mappings created recently using @map to the hardware |
430 | * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush |
431 | * queue |
432 | * @iova_to_phys: translate iova to physical address |
433 | * @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE, |
434 | * including no-snoop TLPs on PCIe or other platform |
435 | * specific mechanisms. |
436 | * @enable_nesting: Enable nesting |
437 | * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*) |
438 | * @free: Release the domain after use. |
439 | */ |
440 | struct iommu_domain_ops { |
441 | int (*attach_dev)(struct iommu_domain *domain, struct device *dev); |
442 | int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev, |
443 | ioasid_t pasid); |
444 | |
445 | int (*map)(struct iommu_domain *domain, unsigned long iova, |
446 | phys_addr_t paddr, size_t size, int prot, gfp_t gfp); |
447 | int (*map_pages)(struct iommu_domain *domain, unsigned long iova, |
448 | phys_addr_t paddr, size_t pgsize, size_t pgcount, |
449 | int prot, gfp_t gfp, size_t *mapped); |
450 | size_t (*unmap)(struct iommu_domain *domain, unsigned long iova, |
451 | size_t size, struct iommu_iotlb_gather *iotlb_gather); |
452 | size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova, |
453 | size_t pgsize, size_t pgcount, |
454 | struct iommu_iotlb_gather *iotlb_gather); |
455 | |
456 | void (*flush_iotlb_all)(struct iommu_domain *domain); |
457 | void (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova, |
458 | size_t size); |
459 | void (*iotlb_sync)(struct iommu_domain *domain, |
460 | struct iommu_iotlb_gather *iotlb_gather); |
461 | |
462 | phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, |
463 | dma_addr_t iova); |
464 | |
465 | bool (*enforce_cache_coherency)(struct iommu_domain *domain); |
466 | int (*enable_nesting)(struct iommu_domain *domain); |
467 | int (*set_pgtable_quirks)(struct iommu_domain *domain, |
468 | unsigned long quirks); |
469 | |
470 | void (*free)(struct iommu_domain *domain); |
471 | }; |
472 | |
473 | /** |
474 | * struct iommu_device - IOMMU core representation of one IOMMU hardware |
475 | * instance |
476 | * @list: Used by the iommu-core to keep a list of registered iommus |
477 | * @ops: iommu-ops for talking to this iommu |
478 | * @dev: struct device for sysfs handling |
479 | * @max_pasids: number of supported PASIDs |
480 | */ |
481 | struct iommu_device { |
482 | struct list_head list; |
483 | const struct iommu_ops *ops; |
484 | struct fwnode_handle *fwnode; |
485 | struct device *dev; |
486 | u32 max_pasids; |
487 | }; |
488 | |
489 | /** |
490 | * struct iommu_fault_event - Generic fault event |
491 | * |
492 | * Can represent recoverable faults such as a page requests or |
493 | * unrecoverable faults such as DMA or IRQ remapping faults. |
494 | * |
495 | * @fault: fault descriptor |
496 | * @list: pending fault event list, used for tracking responses |
497 | */ |
498 | struct iommu_fault_event { |
499 | struct iommu_fault fault; |
500 | struct list_head list; |
501 | }; |
502 | |
503 | /** |
504 | * struct iommu_fault_param - per-device IOMMU fault data |
505 | * @handler: Callback function to handle IOMMU faults at device level |
506 | * @data: handler private data |
507 | * @faults: holds the pending faults which needs response |
508 | * @lock: protect pending faults list |
509 | */ |
510 | struct iommu_fault_param { |
511 | iommu_dev_fault_handler_t handler; |
512 | void *data; |
513 | struct list_head faults; |
514 | struct mutex lock; |
515 | }; |
516 | |
517 | /** |
518 | * struct dev_iommu - Collection of per-device IOMMU data |
519 | * |
520 | * @fault_param: IOMMU detected device fault reporting data |
521 | * @iopf_param: I/O Page Fault queue and data |
522 | * @fwspec: IOMMU fwspec data |
523 | * @iommu_dev: IOMMU device this device is linked to |
524 | * @priv: IOMMU Driver private data |
525 | * @max_pasids: number of PASIDs this device can consume |
526 | * @attach_deferred: the dma domain attachment is deferred |
527 | * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs |
528 | * @require_direct: device requires IOMMU_RESV_DIRECT regions |
529 | * |
530 | * TODO: migrate other per device data pointers under iommu_dev_data, e.g. |
531 | * struct iommu_group *iommu_group; |
532 | */ |
533 | struct dev_iommu { |
534 | struct mutex lock; |
535 | struct iommu_fault_param *fault_param; |
536 | struct iopf_device_param *iopf_param; |
537 | struct iommu_fwspec *fwspec; |
538 | struct iommu_device *iommu_dev; |
539 | void *priv; |
540 | u32 max_pasids; |
541 | u32 attach_deferred:1; |
542 | u32 pci_32bit_workaround:1; |
543 | u32 require_direct:1; |
544 | }; |
545 | |
546 | int iommu_device_register(struct iommu_device *iommu, |
547 | const struct iommu_ops *ops, |
548 | struct device *hwdev); |
549 | void iommu_device_unregister(struct iommu_device *iommu); |
550 | int iommu_device_sysfs_add(struct iommu_device *iommu, |
551 | struct device *parent, |
552 | const struct attribute_group **groups, |
553 | const char *fmt, ...) __printf(4, 5); |
554 | void iommu_device_sysfs_remove(struct iommu_device *iommu); |
555 | int iommu_device_link(struct iommu_device *iommu, struct device *link); |
556 | void iommu_device_unlink(struct iommu_device *iommu, struct device *link); |
557 | int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); |
558 | |
559 | static inline struct iommu_device *dev_to_iommu_device(struct device *dev) |
560 | { |
561 | return (struct iommu_device *)dev_get_drvdata(dev); |
562 | } |
563 | |
564 | static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) |
565 | { |
566 | *gather = (struct iommu_iotlb_gather) { |
567 | .start = ULONG_MAX, |
568 | .freelist = LIST_HEAD_INIT(gather->freelist), |
569 | }; |
570 | } |
571 | |
572 | extern int bus_iommu_probe(const struct bus_type *bus); |
573 | extern bool iommu_present(const struct bus_type *bus); |
574 | extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); |
575 | extern bool iommu_group_has_isolated_msi(struct iommu_group *group); |
576 | extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus); |
577 | extern void iommu_domain_free(struct iommu_domain *domain); |
578 | extern int iommu_attach_device(struct iommu_domain *domain, |
579 | struct device *dev); |
580 | extern void iommu_detach_device(struct iommu_domain *domain, |
581 | struct device *dev); |
582 | extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain, |
583 | struct device *dev, ioasid_t pasid); |
584 | extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); |
585 | extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); |
586 | extern int iommu_map(struct iommu_domain *domain, unsigned long iova, |
587 | phys_addr_t paddr, size_t size, int prot, gfp_t gfp); |
588 | extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, |
589 | size_t size); |
590 | extern size_t iommu_unmap_fast(struct iommu_domain *domain, |
591 | unsigned long iova, size_t size, |
592 | struct iommu_iotlb_gather *iotlb_gather); |
593 | extern ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, |
594 | struct scatterlist *sg, unsigned int nents, |
595 | int prot, gfp_t gfp); |
596 | extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); |
597 | extern void iommu_set_fault_handler(struct iommu_domain *domain, |
598 | iommu_fault_handler_t handler, void *token); |
599 | |
600 | extern void iommu_get_resv_regions(struct device *dev, struct list_head *list); |
601 | extern void iommu_put_resv_regions(struct device *dev, struct list_head *list); |
602 | extern void iommu_set_default_passthrough(bool cmd_line); |
603 | extern void iommu_set_default_translated(bool cmd_line); |
604 | extern bool iommu_default_passthrough(void); |
605 | extern struct iommu_resv_region * |
606 | iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, |
607 | enum iommu_resv_type type, gfp_t gfp); |
608 | extern int iommu_get_group_resv_regions(struct iommu_group *group, |
609 | struct list_head *head); |
610 | |
611 | extern int iommu_attach_group(struct iommu_domain *domain, |
612 | struct iommu_group *group); |
613 | extern void iommu_detach_group(struct iommu_domain *domain, |
614 | struct iommu_group *group); |
615 | extern struct iommu_group *iommu_group_alloc(void); |
616 | extern void *iommu_group_get_iommudata(struct iommu_group *group); |
617 | extern void iommu_group_set_iommudata(struct iommu_group *group, |
618 | void *iommu_data, |
619 | void (*release)(void *iommu_data)); |
620 | extern int iommu_group_set_name(struct iommu_group *group, const char *name); |
621 | extern int iommu_group_add_device(struct iommu_group *group, |
622 | struct device *dev); |
623 | extern void iommu_group_remove_device(struct device *dev); |
624 | extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, |
625 | int (*fn)(struct device *, void *)); |
626 | extern struct iommu_group *iommu_group_get(struct device *dev); |
627 | extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); |
628 | extern void iommu_group_put(struct iommu_group *group); |
629 | extern int iommu_register_device_fault_handler(struct device *dev, |
630 | iommu_dev_fault_handler_t handler, |
631 | void *data); |
632 | |
633 | extern int iommu_unregister_device_fault_handler(struct device *dev); |
634 | |
635 | extern int iommu_report_device_fault(struct device *dev, |
636 | struct iommu_fault_event *evt); |
637 | extern int iommu_page_response(struct device *dev, |
638 | struct iommu_page_response *msg); |
639 | |
640 | extern int iommu_group_id(struct iommu_group *group); |
641 | extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); |
642 | |
643 | int iommu_enable_nesting(struct iommu_domain *domain); |
644 | int iommu_set_pgtable_quirks(struct iommu_domain *domain, |
645 | unsigned long quirks); |
646 | |
647 | void iommu_set_dma_strict(void); |
648 | |
649 | extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, |
650 | unsigned long iova, int flags); |
651 | |
652 | static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) |
653 | { |
654 | if (domain->ops->flush_iotlb_all) |
655 | domain->ops->flush_iotlb_all(domain); |
656 | } |
657 | |
658 | static inline void iommu_iotlb_sync(struct iommu_domain *domain, |
659 | struct iommu_iotlb_gather *iotlb_gather) |
660 | { |
661 | if (domain->ops->iotlb_sync) |
662 | domain->ops->iotlb_sync(domain, iotlb_gather); |
663 | |
664 | iommu_iotlb_gather_init(gather: iotlb_gather); |
665 | } |
666 | |
667 | /** |
668 | * iommu_iotlb_gather_is_disjoint - Checks whether a new range is disjoint |
669 | * |
670 | * @gather: TLB gather data |
671 | * @iova: start of page to invalidate |
672 | * @size: size of page to invalidate |
673 | * |
674 | * Helper for IOMMU drivers to check whether a new range and the gathered range |
675 | * are disjoint. For many IOMMUs, flushing the IOMMU in this case is better |
676 | * than merging the two, which might lead to unnecessary invalidations. |
677 | */ |
678 | static inline |
679 | bool iommu_iotlb_gather_is_disjoint(struct iommu_iotlb_gather *gather, |
680 | unsigned long iova, size_t size) |
681 | { |
682 | unsigned long start = iova, end = start + size - 1; |
683 | |
684 | return gather->end != 0 && |
685 | (end + 1 < gather->start || start > gather->end + 1); |
686 | } |
687 | |
688 | |
689 | /** |
690 | * iommu_iotlb_gather_add_range - Gather for address-based TLB invalidation |
691 | * @gather: TLB gather data |
692 | * @iova: start of page to invalidate |
693 | * @size: size of page to invalidate |
694 | * |
695 | * Helper for IOMMU drivers to build arbitrarily-sized invalidation commands |
696 | * where only the address range matters, and simply minimising intermediate |
697 | * syncs is preferred. |
698 | */ |
699 | static inline void iommu_iotlb_gather_add_range(struct iommu_iotlb_gather *gather, |
700 | unsigned long iova, size_t size) |
701 | { |
702 | unsigned long end = iova + size - 1; |
703 | |
704 | if (gather->start > iova) |
705 | gather->start = iova; |
706 | if (gather->end < end) |
707 | gather->end = end; |
708 | } |
709 | |
710 | /** |
711 | * iommu_iotlb_gather_add_page - Gather for page-based TLB invalidation |
712 | * @domain: IOMMU domain to be invalidated |
713 | * @gather: TLB gather data |
714 | * @iova: start of page to invalidate |
715 | * @size: size of page to invalidate |
716 | * |
717 | * Helper for IOMMU drivers to build invalidation commands based on individual |
718 | * pages, or with page size/table level hints which cannot be gathered if they |
719 | * differ. |
720 | */ |
721 | static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, |
722 | struct iommu_iotlb_gather *gather, |
723 | unsigned long iova, size_t size) |
724 | { |
725 | /* |
726 | * If the new page is disjoint from the current range or is mapped at |
727 | * a different granularity, then sync the TLB so that the gather |
728 | * structure can be rewritten. |
729 | */ |
730 | if ((gather->pgsize && gather->pgsize != size) || |
731 | iommu_iotlb_gather_is_disjoint(gather, iova, size)) |
732 | iommu_iotlb_sync(domain, iotlb_gather: gather); |
733 | |
734 | gather->pgsize = size; |
735 | iommu_iotlb_gather_add_range(gather, iova, size); |
736 | } |
737 | |
738 | static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather) |
739 | { |
740 | return gather && gather->queued; |
741 | } |
742 | |
743 | static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty, |
744 | struct iova_bitmap *bitmap, |
745 | struct iommu_iotlb_gather *gather) |
746 | { |
747 | if (gather) |
748 | iommu_iotlb_gather_init(gather); |
749 | |
750 | dirty->bitmap = bitmap; |
751 | dirty->gather = gather; |
752 | } |
753 | |
754 | static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty, |
755 | unsigned long iova, |
756 | unsigned long length) |
757 | { |
758 | if (dirty->bitmap) |
759 | iova_bitmap_set(bitmap: dirty->bitmap, iova, length); |
760 | |
761 | if (dirty->gather) |
762 | iommu_iotlb_gather_add_range(gather: dirty->gather, iova, size: length); |
763 | } |
764 | |
765 | /* PCI device grouping function */ |
766 | extern struct iommu_group *pci_device_group(struct device *dev); |
767 | /* Generic device grouping function */ |
768 | extern struct iommu_group *generic_device_group(struct device *dev); |
769 | /* FSL-MC device grouping function */ |
770 | struct iommu_group *fsl_mc_device_group(struct device *dev); |
771 | |
772 | /** |
773 | * struct iommu_fwspec - per-device IOMMU instance data |
774 | * @ops: ops for this device's IOMMU |
775 | * @iommu_fwnode: firmware handle for this device's IOMMU |
776 | * @flags: IOMMU_FWSPEC_* flags |
777 | * @num_ids: number of associated device IDs |
778 | * @ids: IDs which this device may present to the IOMMU |
779 | * |
780 | * Note that the IDs (and any other information, really) stored in this structure should be |
781 | * considered private to the IOMMU device driver and are not to be used directly by IOMMU |
782 | * consumers. |
783 | */ |
784 | struct iommu_fwspec { |
785 | const struct iommu_ops *ops; |
786 | struct fwnode_handle *iommu_fwnode; |
787 | u32 flags; |
788 | unsigned int num_ids; |
789 | u32 ids[]; |
790 | }; |
791 | |
792 | /* ATS is supported */ |
793 | #define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0) |
794 | |
795 | /** |
796 | * struct iommu_sva - handle to a device-mm bond |
797 | */ |
798 | struct iommu_sva { |
799 | struct device *dev; |
800 | struct iommu_domain *domain; |
801 | }; |
802 | |
803 | int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, |
804 | const struct iommu_ops *ops); |
805 | void iommu_fwspec_free(struct device *dev); |
806 | int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); |
807 | const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); |
808 | |
809 | static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) |
810 | { |
811 | if (dev->iommu) |
812 | return dev->iommu->fwspec; |
813 | else |
814 | return NULL; |
815 | } |
816 | |
817 | static inline void dev_iommu_fwspec_set(struct device *dev, |
818 | struct iommu_fwspec *fwspec) |
819 | { |
820 | dev->iommu->fwspec = fwspec; |
821 | } |
822 | |
823 | static inline void *dev_iommu_priv_get(struct device *dev) |
824 | { |
825 | if (dev->iommu) |
826 | return dev->iommu->priv; |
827 | else |
828 | return NULL; |
829 | } |
830 | |
831 | static inline void dev_iommu_priv_set(struct device *dev, void *priv) |
832 | { |
833 | dev->iommu->priv = priv; |
834 | } |
835 | |
836 | int iommu_probe_device(struct device *dev); |
837 | |
838 | int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); |
839 | int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); |
840 | |
841 | int iommu_device_use_default_domain(struct device *dev); |
842 | void iommu_device_unuse_default_domain(struct device *dev); |
843 | |
844 | int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner); |
845 | void iommu_group_release_dma_owner(struct iommu_group *group); |
846 | bool iommu_group_dma_owner_claimed(struct iommu_group *group); |
847 | |
848 | int iommu_device_claim_dma_owner(struct device *dev, void *owner); |
849 | void iommu_device_release_dma_owner(struct device *dev); |
850 | |
851 | struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, |
852 | struct mm_struct *mm); |
853 | int iommu_attach_device_pasid(struct iommu_domain *domain, |
854 | struct device *dev, ioasid_t pasid); |
855 | void iommu_detach_device_pasid(struct iommu_domain *domain, |
856 | struct device *dev, ioasid_t pasid); |
857 | struct iommu_domain * |
858 | iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid, |
859 | unsigned int type); |
860 | ioasid_t iommu_alloc_global_pasid(struct device *dev); |
861 | void iommu_free_global_pasid(ioasid_t pasid); |
862 | #else /* CONFIG_IOMMU_API */ |
863 | |
864 | struct iommu_ops {}; |
865 | struct iommu_group {}; |
866 | struct iommu_fwspec {}; |
867 | struct iommu_device {}; |
868 | struct iommu_fault_param {}; |
869 | struct iommu_iotlb_gather {}; |
870 | struct iommu_dirty_bitmap {}; |
871 | struct iommu_dirty_ops {}; |
872 | |
873 | static inline bool iommu_present(const struct bus_type *bus) |
874 | { |
875 | return false; |
876 | } |
877 | |
878 | static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap) |
879 | { |
880 | return false; |
881 | } |
882 | |
883 | static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) |
884 | { |
885 | return NULL; |
886 | } |
887 | |
888 | static inline void iommu_domain_free(struct iommu_domain *domain) |
889 | { |
890 | } |
891 | |
892 | static inline int iommu_attach_device(struct iommu_domain *domain, |
893 | struct device *dev) |
894 | { |
895 | return -ENODEV; |
896 | } |
897 | |
898 | static inline void iommu_detach_device(struct iommu_domain *domain, |
899 | struct device *dev) |
900 | { |
901 | } |
902 | |
903 | static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) |
904 | { |
905 | return NULL; |
906 | } |
907 | |
908 | static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, |
909 | phys_addr_t paddr, size_t size, int prot, gfp_t gfp) |
910 | { |
911 | return -ENODEV; |
912 | } |
913 | |
914 | static inline size_t iommu_unmap(struct iommu_domain *domain, |
915 | unsigned long iova, size_t size) |
916 | { |
917 | return 0; |
918 | } |
919 | |
920 | static inline size_t iommu_unmap_fast(struct iommu_domain *domain, |
921 | unsigned long iova, int gfp_order, |
922 | struct iommu_iotlb_gather *iotlb_gather) |
923 | { |
924 | return 0; |
925 | } |
926 | |
927 | static inline ssize_t iommu_map_sg(struct iommu_domain *domain, |
928 | unsigned long iova, struct scatterlist *sg, |
929 | unsigned int nents, int prot, gfp_t gfp) |
930 | { |
931 | return -ENODEV; |
932 | } |
933 | |
934 | static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) |
935 | { |
936 | } |
937 | |
938 | static inline void iommu_iotlb_sync(struct iommu_domain *domain, |
939 | struct iommu_iotlb_gather *iotlb_gather) |
940 | { |
941 | } |
942 | |
943 | static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) |
944 | { |
945 | return 0; |
946 | } |
947 | |
948 | static inline void iommu_set_fault_handler(struct iommu_domain *domain, |
949 | iommu_fault_handler_t handler, void *token) |
950 | { |
951 | } |
952 | |
953 | static inline void iommu_get_resv_regions(struct device *dev, |
954 | struct list_head *list) |
955 | { |
956 | } |
957 | |
958 | static inline void iommu_put_resv_regions(struct device *dev, |
959 | struct list_head *list) |
960 | { |
961 | } |
962 | |
963 | static inline int iommu_get_group_resv_regions(struct iommu_group *group, |
964 | struct list_head *head) |
965 | { |
966 | return -ENODEV; |
967 | } |
968 | |
969 | static inline void iommu_set_default_passthrough(bool cmd_line) |
970 | { |
971 | } |
972 | |
973 | static inline void iommu_set_default_translated(bool cmd_line) |
974 | { |
975 | } |
976 | |
977 | static inline bool iommu_default_passthrough(void) |
978 | { |
979 | return true; |
980 | } |
981 | |
982 | static inline int iommu_attach_group(struct iommu_domain *domain, |
983 | struct iommu_group *group) |
984 | { |
985 | return -ENODEV; |
986 | } |
987 | |
988 | static inline void iommu_detach_group(struct iommu_domain *domain, |
989 | struct iommu_group *group) |
990 | { |
991 | } |
992 | |
993 | static inline struct iommu_group *iommu_group_alloc(void) |
994 | { |
995 | return ERR_PTR(-ENODEV); |
996 | } |
997 | |
998 | static inline void *iommu_group_get_iommudata(struct iommu_group *group) |
999 | { |
1000 | return NULL; |
1001 | } |
1002 | |
1003 | static inline void iommu_group_set_iommudata(struct iommu_group *group, |
1004 | void *iommu_data, |
1005 | void (*release)(void *iommu_data)) |
1006 | { |
1007 | } |
1008 | |
1009 | static inline int iommu_group_set_name(struct iommu_group *group, |
1010 | const char *name) |
1011 | { |
1012 | return -ENODEV; |
1013 | } |
1014 | |
1015 | static inline int iommu_group_add_device(struct iommu_group *group, |
1016 | struct device *dev) |
1017 | { |
1018 | return -ENODEV; |
1019 | } |
1020 | |
1021 | static inline void iommu_group_remove_device(struct device *dev) |
1022 | { |
1023 | } |
1024 | |
1025 | static inline int iommu_group_for_each_dev(struct iommu_group *group, |
1026 | void *data, |
1027 | int (*fn)(struct device *, void *)) |
1028 | { |
1029 | return -ENODEV; |
1030 | } |
1031 | |
1032 | static inline struct iommu_group *iommu_group_get(struct device *dev) |
1033 | { |
1034 | return NULL; |
1035 | } |
1036 | |
1037 | static inline void iommu_group_put(struct iommu_group *group) |
1038 | { |
1039 | } |
1040 | |
1041 | static inline |
1042 | int iommu_register_device_fault_handler(struct device *dev, |
1043 | iommu_dev_fault_handler_t handler, |
1044 | void *data) |
1045 | { |
1046 | return -ENODEV; |
1047 | } |
1048 | |
1049 | static inline int iommu_unregister_device_fault_handler(struct device *dev) |
1050 | { |
1051 | return 0; |
1052 | } |
1053 | |
1054 | static inline |
1055 | int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) |
1056 | { |
1057 | return -ENODEV; |
1058 | } |
1059 | |
1060 | static inline int iommu_page_response(struct device *dev, |
1061 | struct iommu_page_response *msg) |
1062 | { |
1063 | return -ENODEV; |
1064 | } |
1065 | |
1066 | static inline int iommu_group_id(struct iommu_group *group) |
1067 | { |
1068 | return -ENODEV; |
1069 | } |
1070 | |
1071 | static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain, |
1072 | unsigned long quirks) |
1073 | { |
1074 | return 0; |
1075 | } |
1076 | |
1077 | static inline int iommu_device_register(struct iommu_device *iommu, |
1078 | const struct iommu_ops *ops, |
1079 | struct device *hwdev) |
1080 | { |
1081 | return -ENODEV; |
1082 | } |
1083 | |
1084 | static inline struct iommu_device *dev_to_iommu_device(struct device *dev) |
1085 | { |
1086 | return NULL; |
1087 | } |
1088 | |
1089 | static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) |
1090 | { |
1091 | } |
1092 | |
1093 | static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, |
1094 | struct iommu_iotlb_gather *gather, |
1095 | unsigned long iova, size_t size) |
1096 | { |
1097 | } |
1098 | |
1099 | static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather) |
1100 | { |
1101 | return false; |
1102 | } |
1103 | |
1104 | static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty, |
1105 | struct iova_bitmap *bitmap, |
1106 | struct iommu_iotlb_gather *gather) |
1107 | { |
1108 | } |
1109 | |
1110 | static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty, |
1111 | unsigned long iova, |
1112 | unsigned long length) |
1113 | { |
1114 | } |
1115 | |
1116 | static inline void iommu_device_unregister(struct iommu_device *iommu) |
1117 | { |
1118 | } |
1119 | |
1120 | static inline int iommu_device_sysfs_add(struct iommu_device *iommu, |
1121 | struct device *parent, |
1122 | const struct attribute_group **groups, |
1123 | const char *fmt, ...) |
1124 | { |
1125 | return -ENODEV; |
1126 | } |
1127 | |
1128 | static inline void iommu_device_sysfs_remove(struct iommu_device *iommu) |
1129 | { |
1130 | } |
1131 | |
1132 | static inline int iommu_device_link(struct device *dev, struct device *link) |
1133 | { |
1134 | return -EINVAL; |
1135 | } |
1136 | |
1137 | static inline void iommu_device_unlink(struct device *dev, struct device *link) |
1138 | { |
1139 | } |
1140 | |
1141 | static inline int iommu_fwspec_init(struct device *dev, |
1142 | struct fwnode_handle *iommu_fwnode, |
1143 | const struct iommu_ops *ops) |
1144 | { |
1145 | return -ENODEV; |
1146 | } |
1147 | |
1148 | static inline void iommu_fwspec_free(struct device *dev) |
1149 | { |
1150 | } |
1151 | |
1152 | static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, |
1153 | int num_ids) |
1154 | { |
1155 | return -ENODEV; |
1156 | } |
1157 | |
1158 | static inline |
1159 | const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) |
1160 | { |
1161 | return NULL; |
1162 | } |
1163 | |
1164 | static inline int |
1165 | iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) |
1166 | { |
1167 | return -ENODEV; |
1168 | } |
1169 | |
1170 | static inline int |
1171 | iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) |
1172 | { |
1173 | return -ENODEV; |
1174 | } |
1175 | |
1176 | static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) |
1177 | { |
1178 | return NULL; |
1179 | } |
1180 | |
1181 | static inline int iommu_device_use_default_domain(struct device *dev) |
1182 | { |
1183 | return 0; |
1184 | } |
1185 | |
1186 | static inline void iommu_device_unuse_default_domain(struct device *dev) |
1187 | { |
1188 | } |
1189 | |
1190 | static inline int |
1191 | iommu_group_claim_dma_owner(struct iommu_group *group, void *owner) |
1192 | { |
1193 | return -ENODEV; |
1194 | } |
1195 | |
1196 | static inline void iommu_group_release_dma_owner(struct iommu_group *group) |
1197 | { |
1198 | } |
1199 | |
1200 | static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group) |
1201 | { |
1202 | return false; |
1203 | } |
1204 | |
1205 | static inline void iommu_device_release_dma_owner(struct device *dev) |
1206 | { |
1207 | } |
1208 | |
1209 | static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner) |
1210 | { |
1211 | return -ENODEV; |
1212 | } |
1213 | |
1214 | static inline struct iommu_domain * |
1215 | iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) |
1216 | { |
1217 | return NULL; |
1218 | } |
1219 | |
1220 | static inline int iommu_attach_device_pasid(struct iommu_domain *domain, |
1221 | struct device *dev, ioasid_t pasid) |
1222 | { |
1223 | return -ENODEV; |
1224 | } |
1225 | |
1226 | static inline void iommu_detach_device_pasid(struct iommu_domain *domain, |
1227 | struct device *dev, ioasid_t pasid) |
1228 | { |
1229 | } |
1230 | |
1231 | static inline struct iommu_domain * |
1232 | iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid, |
1233 | unsigned int type) |
1234 | { |
1235 | return NULL; |
1236 | } |
1237 | |
1238 | static inline ioasid_t iommu_alloc_global_pasid(struct device *dev) |
1239 | { |
1240 | return IOMMU_PASID_INVALID; |
1241 | } |
1242 | |
1243 | static inline void iommu_free_global_pasid(ioasid_t pasid) {} |
1244 | #endif /* CONFIG_IOMMU_API */ |
1245 | |
1246 | /** |
1247 | * iommu_map_sgtable - Map the given buffer to the IOMMU domain |
1248 | * @domain: The IOMMU domain to perform the mapping |
1249 | * @iova: The start address to map the buffer |
1250 | * @sgt: The sg_table object describing the buffer |
1251 | * @prot: IOMMU protection bits |
1252 | * |
1253 | * Creates a mapping at @iova for the buffer described by a scatterlist |
1254 | * stored in the given sg_table object in the provided IOMMU domain. |
1255 | */ |
1256 | static inline size_t iommu_map_sgtable(struct iommu_domain *domain, |
1257 | unsigned long iova, struct sg_table *sgt, int prot) |
1258 | { |
1259 | return iommu_map_sg(domain, iova, sg: sgt->sgl, nents: sgt->orig_nents, prot, |
1260 | GFP_KERNEL); |
1261 | } |
1262 | |
1263 | #ifdef CONFIG_IOMMU_DEBUGFS |
1264 | extern struct dentry *iommu_debugfs_dir; |
1265 | void iommu_debugfs_setup(void); |
1266 | #else |
1267 | static inline void iommu_debugfs_setup(void) {} |
1268 | #endif |
1269 | |
1270 | #ifdef CONFIG_IOMMU_DMA |
1271 | #include <linux/msi.h> |
1272 | |
1273 | /* Setup call for arch DMA mapping code */ |
1274 | void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit); |
1275 | |
1276 | int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); |
1277 | |
1278 | int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr); |
1279 | void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg); |
1280 | |
1281 | #else /* CONFIG_IOMMU_DMA */ |
1282 | |
1283 | struct msi_desc; |
1284 | struct msi_msg; |
1285 | |
1286 | static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) |
1287 | { |
1288 | } |
1289 | |
1290 | static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) |
1291 | { |
1292 | return -ENODEV; |
1293 | } |
1294 | |
1295 | static inline int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) |
1296 | { |
1297 | return 0; |
1298 | } |
1299 | |
1300 | static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg) |
1301 | { |
1302 | } |
1303 | |
1304 | #endif /* CONFIG_IOMMU_DMA */ |
1305 | |
1306 | /* |
1307 | * Newer generations of Tegra SoCs require devices' stream IDs to be directly programmed into |
1308 | * some registers. These are always paired with a Tegra SMMU or ARM SMMU, for which the contents |
1309 | * of the struct iommu_fwspec are known. Use this helper to formalize access to these internals. |
1310 | */ |
1311 | #define TEGRA_STREAM_ID_BYPASS 0x7f |
1312 | |
1313 | static inline bool tegra_dev_iommu_get_stream_id(struct device *dev, u32 *stream_id) |
1314 | { |
1315 | #ifdef CONFIG_IOMMU_API |
1316 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
1317 | |
1318 | if (fwspec && fwspec->num_ids == 1) { |
1319 | *stream_id = fwspec->ids[0] & 0xffff; |
1320 | return true; |
1321 | } |
1322 | #endif |
1323 | |
1324 | return false; |
1325 | } |
1326 | |
1327 | #ifdef CONFIG_IOMMU_SVA |
1328 | static inline void mm_pasid_init(struct mm_struct *mm) |
1329 | { |
1330 | mm->pasid = IOMMU_PASID_INVALID; |
1331 | } |
1332 | static inline bool mm_valid_pasid(struct mm_struct *mm) |
1333 | { |
1334 | return mm->pasid != IOMMU_PASID_INVALID; |
1335 | } |
1336 | void mm_pasid_drop(struct mm_struct *mm); |
1337 | struct iommu_sva *iommu_sva_bind_device(struct device *dev, |
1338 | struct mm_struct *mm); |
1339 | void iommu_sva_unbind_device(struct iommu_sva *handle); |
1340 | u32 iommu_sva_get_pasid(struct iommu_sva *handle); |
1341 | #else |
1342 | static inline struct iommu_sva * |
1343 | iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) |
1344 | { |
1345 | return NULL; |
1346 | } |
1347 | |
1348 | static inline void iommu_sva_unbind_device(struct iommu_sva *handle) |
1349 | { |
1350 | } |
1351 | |
1352 | static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle) |
1353 | { |
1354 | return IOMMU_PASID_INVALID; |
1355 | } |
1356 | static inline void mm_pasid_init(struct mm_struct *mm) {} |
1357 | static inline bool mm_valid_pasid(struct mm_struct *mm) { return false; } |
1358 | static inline void mm_pasid_drop(struct mm_struct *mm) {} |
1359 | #endif /* CONFIG_IOMMU_SVA */ |
1360 | |
1361 | #endif /* __LINUX_IOMMU_H */ |
1362 | |